diff --git a/src/commands/run-tests-command.ts b/src/commands/run-tests-command.ts index 3f01349..f85385a 100644 --- a/src/commands/run-tests-command.ts +++ b/src/commands/run-tests-command.ts @@ -18,10 +18,8 @@ export class RunCommand { // Convert ConfigLoader to the format expected by TestRunner const configData = this.convertConfigToData(config, options.quick); - // Run tests using the shared TestRunner with axios for backward compatibility - const results = await this.testRunner.runTests(configData, { - useAxios: true, // Use axios for backward compatibility with existing behavior - }); + // Run tests using the shared TestRunner + const results = await this.testRunner.runTests(configData); // Validate before saving if validation option is enabled if (options.validate) { diff --git a/src/models/test-types.ts b/src/models/test-types.ts index d5b907b..b3be859 100644 --- a/src/models/test-types.ts +++ b/src/models/test-types.ts @@ -80,7 +80,6 @@ export interface InternalTestResult { invalid?: 'false' | 'true' | 'semantic' | 'undefined'; expression: string; capability?: CapabilityKV[]; - SkipMessage?: string; } // Schema-compliant TestResult type (strictly matches cql-test-results.schema.json) diff --git a/src/server/test-execution-service.ts b/src/server/test-execution-service.ts index cfdb94a..5ed6ee3 100644 --- a/src/server/test-execution-service.ts +++ b/src/server/test-execution-service.ts @@ -1,217 +1,85 @@ // Author: Preston Lee -import { ConfigLoader } from '../conf/config-loader.js'; -import { CQLEngine } from '../cql-engine/cql-engine.js'; import { CQLTestResults } from '../test-results/cql-test-results.js'; -import { TestLoader } from '../loaders/test-loader.js'; -import { - generateEmptyResults, - generateParametersResource, - Result, -} from '../shared/results-shared.js'; -import { InternalTestResult, Tests } from '../models/test-types.js'; -import { ServerConnectivity } from '../shared/server-connectivity.js'; -import { ResultExtractor } from '../extractors/result-extractor.js'; -import { buildExtractor } from './extractor-builder.js'; -import { createConfigFromData } from './config-utils.js'; -import { ValueMap } from '../extractors/value-map.js'; -import { resultsEqual } from '../shared/results-utils.js'; - -interface ExecutionContext { - config: ConfigLoader; - cqlEngine: CQLEngine; - cvl: any; - tests: Tests[]; - resultExtractor: ResultExtractor; - skipMap: Map; - onlySet: Set; -} - +import { generateEmptyResults, Result } from '../shared/results-shared.js'; +import { createExecutionContext, runTest } from '../shared/run-test-core.js'; + +/** + * Server/MCP-facing runner. Thin wrapper over the shared {@link runTest} core: builds the + * execution context and returns JSON-shaped results. Shares identical execution and + * classification with the CLI {@link TestRunner}. + */ export class TestExecutionService { - /** - * Builds shared execution context from config data (engine, CVL, tests, extractor, skip map). - */ - private async createExecutionContext(configData: any): Promise { - const config = createConfigFromData(configData); - const serverBaseUrl = config.FhirServer.BaseUrl; - const cqlEndpoint = config.CqlEndpoint; - - await ServerConnectivity.verifyServerConnectivity(serverBaseUrl); - - const build = config.Build; - const cqlEngine = new CQLEngine( - serverBaseUrl, - cqlEndpoint, - build?.cqlTranslator ?? '', - build?.cqlTranslatorVersion ?? '', - build?.cqlEngine ?? '', - build?.cqlEngineVersion ?? '' - ); - cqlEngine.cqlVersion = config.Build?.CqlVersion || '1.5'; - - // @ts-expect-error - cvl.mjs has no declaration file - const cvlModule = await import('../../cvl/cvl.mjs'); - const cvl = cvlModule.default; - - const tests = TestLoader.load(); - const resultExtractor = buildExtractor(); - const skipMap = config.skipListMap(); - const onlySet = config.onlyListSet(); - - return { config, cqlEngine, cvl, tests, resultExtractor, skipMap, onlySet }; - } - - /** - * Runs a single test - */ - private async runTest( - result: InternalTestResult, - apiUrl: string, - cvl: any, - resultExtractor: ResultExtractor, - skipMap: Map, - onlySet: Set, - config: ConfigLoader - ): Promise { - const key = `${result.testsName}-${result.groupName}-${result.testName}`; - - if (result.testStatus === 'skip') { - result.SkipMessage = 'Skipped by cql-tests-runner'; - return result; - } else if (onlySet.size > 0 && !onlySet.has(key)) { - result.SkipMessage = 'Skipped by OnlyList filter'; - result.testStatus = 'skip'; - return result; - } else if (skipMap.has(key)) { - const reason = skipMap.get(key) || ''; - result.SkipMessage = `Skipped by config: ${reason}`; - result.testStatus = 'skip'; - return result; - } - - const data = generateParametersResource(result, config.FhirServer.CqlOperation); - - try { - console.log('Running test %s:%s:%s', result.testsName, result.groupName, result.testName); - const response = await fetch(apiUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(data) - }); - - result.responseStatus = response.status; - const responseBody = await response.json(); - const parsedExpected = cvl.parse(result.expected); - result.actual = resultExtractor.extract(responseBody, { - singletonListKeys: ValueMap.singletonListKeysFromExpected(parsedExpected), - }); - const invalid = result.invalid; - - if (invalid === 'true' || invalid === 'semantic') { - result.testStatus = response.status === 200 ? 'fail' : 'pass'; - } else { - if (response.status === 200) { - result.testStatus = resultsEqual(parsedExpected, result.actual) ? 'pass' : 'fail'; - } else { - result.testStatus = 'fail'; - } - } - } catch (error: any) { - result.testStatus = 'error'; - result.error = { - message: error.message, - name: error.name || 'Error', - stack: error.stack - }; - } - - console.log('Test %s:%s:%s status: %s expected: %s actual: %s', - result.testsName, result.groupName, result.testName, result.testStatus, result.expected, result.actual); - - return result; - } - - /** - * Runs all tests based on configuration - */ - async runTests(configData: any): Promise { - const ctx = await this.createExecutionContext(configData); - const { config, cqlEngine, cvl, tests, resultExtractor, skipMap, onlySet } = ctx; - - const quickTest = config.Debug?.QuickTest || false; - const emptyResults = await generateEmptyResults(tests, quickTest); - const results = new CQLTestResults(cqlEngine); - - for (const testFile of emptyResults) { - for (const result of testFile) { - await this.runTest(result, cqlEngine.apiUrl!, cvl, resultExtractor, skipMap, onlySet, config); - results.add(result); - } - } - - return results.toJSON(); - } - - /** - * Runs a single test by identifier - */ - async runSingleTest( - testsName: string, - groupName: string, - testName: string, - configData: any - ): Promise { - const ctx = await this.createExecutionContext(configData); - const { config, cqlEngine, cvl, tests, resultExtractor, skipMap, onlySet } = ctx; - - for (const testSuite of tests) { - if (testSuite.name !== testsName) continue; - for (const group of testSuite.group) { - if (group.name !== groupName || !group.test) continue; - for (const test of group.test) { - if (test.name !== testName) continue; - - const result = new Result(testsName, groupName, test); - await this.runTest(result, cqlEngine.apiUrl!, cvl, resultExtractor, skipMap, onlySet, config); - - const testResults = new CQLTestResults(cqlEngine); - testResults.add(result); - return testResults.toJSON().results[0] ?? null; - } - } - } - - throw new Error(`Test not found: ${testsName}/${groupName}/${testName}`); - } - - /** - * Runs all tests in a group - */ - async runTestGroup( - testsName: string, - groupName: string, - configData: any - ): Promise { - const ctx = await this.createExecutionContext(configData); - const { config, cqlEngine, cvl, tests, resultExtractor, skipMap, onlySet } = ctx; - - const results = new CQLTestResults(cqlEngine); - - for (const testSuite of tests) { - if (testSuite.name !== testsName) continue; - for (const group of testSuite.group) { - if (group.name !== groupName || !group.test) continue; - for (const test of group.test) { - const result = new Result(testsName, groupName, test); - await this.runTest(result, cqlEngine.apiUrl!, cvl, resultExtractor, skipMap, onlySet, config); - results.add(result); - } - return results.toJSON().results; - } - } - - return results.toJSON().results; - } + /** + * Runs all tests based on configuration. + */ + async runTests(configData: any): Promise { + const ctx = await createExecutionContext(configData); + const quickTest = ctx.config.Debug?.QuickTest || false; + const emptyResults = await generateEmptyResults(ctx.tests, quickTest); + const results = new CQLTestResults(ctx.cqlEngine); + + for (const testFile of emptyResults) { + for (const result of testFile) { + await runTest(result, ctx); + results.add(result); + } + } + + return results.toJSON(); + } + + /** + * Runs a single test by identifier. + */ + async runSingleTest( + testsName: string, + groupName: string, + testName: string, + configData: any + ): Promise { + const ctx = await createExecutionContext(configData); + + for (const testSuite of ctx.tests) { + if (testSuite.name !== testsName) continue; + for (const group of testSuite.group) { + if (group.name !== groupName || !group.test) continue; + for (const test of group.test) { + if (test.name !== testName) continue; + + const result = new Result(testsName, groupName, test); + await runTest(result, ctx); + + const testResults = new CQLTestResults(ctx.cqlEngine); + testResults.add(result); + return testResults.toJSON().results[0] ?? null; + } + } + } + + throw new Error(`Test not found: ${testsName}/${groupName}/${testName}`); + } + + /** + * Runs all tests in a group. + */ + async runTestGroup(testsName: string, groupName: string, configData: any): Promise { + const ctx = await createExecutionContext(configData); + const results = new CQLTestResults(ctx.cqlEngine); + + for (const testSuite of ctx.tests) { + if (testSuite.name !== testsName) continue; + for (const group of testSuite.group) { + if (group.name !== groupName || !group.test) continue; + for (const test of group.test) { + const result = new Result(testsName, groupName, test); + await runTest(result, ctx); + results.add(result); + } + return results.toJSON().results; + } + } + + return results.toJSON().results; + } } diff --git a/src/services/test-runner.ts b/src/services/test-runner.ts index b68627d..69f21e8 100644 --- a/src/services/test-runner.ts +++ b/src/services/test-runner.ts @@ -1,93 +1,32 @@ -import { ConfigLoader } from '../conf/config-loader.js'; -import { CQLEngine } from '../cql-engine/cql-engine.js'; -import { TestLoader } from '../loaders/test-loader.js'; import { CQLTestResults } from '../test-results/cql-test-results.js'; -import { generateEmptyResults, generateParametersResource } from '../shared/results-shared.js'; -import { InternalTestResult } from '../models/test-types.js'; -import { ResultExtractor } from '../extractors/result-extractor.js'; -import { ServerConnectivity } from '../shared/server-connectivity.js'; -import { buildExtractor } from '../server/extractor-builder.js'; -import { createConfigFromData } from '../server/config-utils.js'; -import { ValueMap } from '../extractors/value-map.js'; -import { resultsEqual } from '../shared/results-utils.js'; +import { generateEmptyResults } from '../shared/results-shared.js'; +import { createExecutionContext, runTest } from '../shared/run-test-core.js'; export interface TestRunnerOptions { onProgress?: (current: number, total: number, message?: string) => Promise; - useAxios?: boolean; // For backward compatibility with run-tests-command } +/** + * CLI-facing runner. Builds the shared execution context, runs every loaded test through the + * shared {@link runTest}, and reports progress via the optional callback. Returns the + * {@link CQLTestResults} instance so the caller can validate and save it. + */ export class TestRunner { public async runTests( configData: any, options: TestRunnerOptions = {} ): Promise { - // Create a temporary config loader from the provided data - const config = createConfigFromData(configData); - const serverBaseUrl = config.FhirServer.BaseUrl; - const cqlEndpoint = config.CqlEndpoint; - - // Verify server connectivity before proceeding - await ServerConnectivity.verifyServerConnectivity(serverBaseUrl); - - const build = config.Build; - const cqlEngine = new CQLEngine( - serverBaseUrl, - cqlEndpoint, - build.cqlTranslator ?? '', - build.cqlTranslatorVersion ?? '', - build.cqlEngine ?? '', - build.cqlEngineVersion ?? '' - ); - cqlEngine.cqlVersion = '1.5'; //default value - const cqlVersion = config.Build?.CqlVersion; - if (typeof cqlVersion === 'string' && cqlVersion.trim() !== '') { - cqlEngine.cqlVersion = cqlVersion; - } - - // Load CVL using dynamic import - // @ts-ignore - const cvlModule = await import('../../cvl/cvl.mjs'); - const cvl = cvlModule.default; - - const tests = TestLoader.load(); - const quickTest = config.Debug?.QuickTest || false; - const resultExtractor = buildExtractor(); - const emptyResults = await generateEmptyResults(tests, quickTest); - const skipMap = config.skipListMap(); - const onlySet = config.onlyListSet(); - - const results = new CQLTestResults(cqlEngine); + const ctx = await createExecutionContext(configData); + const quickTest = ctx.config.Debug?.QuickTest || false; + const emptyResults = await generateEmptyResults(ctx.tests, quickTest); + const results = new CQLTestResults(ctx.cqlEngine); const totalTests = emptyResults.reduce((sum, testFile) => sum + testFile.length, 0); let completedTests = 0; for (const testFile of emptyResults) { for (const result of testFile) { - if (this.shouldSkipVersionTest(cqlEngine, result)) { - const skipReason = - result.testVersionTo && - this.compareVersions(cqlEngine.cqlVersion, result.testVersionTo) > 0 - ? `test versionTo ${result.testVersionTo} not applicable to engine version ${cqlEngine.cqlVersion}` - : `test version ${result.testVersion} not applicable to engine version ${cqlEngine.cqlVersion}`; - - this.addToSkipList( - skipMap, - result.testsName, - result.groupName, - result.testName, - skipReason - ); - } - await this.runTest( - result, - cqlEngine.apiUrl!, - cvl, - resultExtractor, - skipMap, - onlySet, - config, - options.useAxios - ); + await runTest(result, ctx); results.add(result); completedTests++; @@ -100,176 +39,7 @@ export class TestRunner { } } } - // Return the CQLTestResults instance - return results; - } - - private async runTest( - result: InternalTestResult, - apiUrl: string, - cvl: any, - resultExtractor: ResultExtractor, - skipMap: Map, - onlySet: Set, - config: ConfigLoader, - useAxios: boolean = false - ): Promise { - const key = `${result.testsName}-${result.groupName}-${result.testName}`; - - if (result.testStatus === 'skip') { - if (!result.skipMessage?.trim()) { - result.skipMessage = 'Skipped by cql-tests-runner'; - } - console.log( - 'Test %s:%s:%s status: %s skipMessage: %s', - result.testsName, - result.groupName, - result.testName, - result.testStatus, - result.skipMessage - ); - return result; - } else if (onlySet.size > 0 && !onlySet.has(key)) { - result.SkipMessage = 'Skipped by OnlyList filter'; - result.testStatus = 'skip'; - return result; - } else if (skipMap.has(key)) { - const reason = skipMap.get(key) || ''; - result.skipMessage = `Skipped by config: ${reason}`; - result.testStatus = 'skip'; - console.log( - 'Test %s:%s:%s status: %s skipMessage: %s', - result.testsName, - result.groupName, - result.testName, - result.testStatus, - result.skipMessage - ); - return result; - } - const data = generateParametersResource(result, config.FhirServer.CqlOperation); - - try { - console.log( - 'Running test %s:%s:%s', - result.testsName, - result.groupName, - result.testName - ); - - let response: any; - if (useAxios) { - // Use axios for backward compatibility - const axios = await import('axios'); - const axiosResponse = await axios.default.post(apiUrl, data, { - headers: { - 'Content-Type': 'application/json', - }, - }); - response = { - status: axiosResponse.status, - data: axiosResponse.data, - }; - } else { - // Use fetch (default for new code) - const fetchResponse = await fetch(apiUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(data), - }); - response = { - status: fetchResponse.status, - data: await fetchResponse.json(), - }; - } - - result.responseStatus = response.status; - const responseBody = response.data; - const parsedExpected = cvl.parse(result.expected); - result.actual = resultExtractor.extract(responseBody, { - singletonListKeys: ValueMap.singletonListKeysFromExpected(parsedExpected), - }); - const invalid = result.invalid; - - if (invalid === 'true' || invalid === 'semantic') { - result.testStatus = response.status === 200 ? 'fail' : 'pass'; - } else { - if (response.status === 200) { - result.testStatus = resultsEqual(parsedExpected, result.actual) - ? 'pass' - : 'fail'; - } else { - result.testStatus = 'fail'; - } - } - } catch (error: any) { - result.testStatus = 'error'; - result.error = { - message: error.message, - name: error.name || 'Error', - stack: error.stack, - }; - } - - console.log( - 'Test %s:%s:%s status: %s expected: %s actual: %s', - result.testsName, - result.groupName, - result.testName, - result.testStatus, - result.expected, - result.actual - ); - - return result; - } - - private compareVersions(versionA: string | undefined, versionB: string | undefined): number { - // Split into numeric parts (e.g., "1.5.2" → [1,5,2]) - const partsA = String(versionA ?? '') - .trim() - .split('.') - .map(n => parseInt(n, 10) || 0); - const partsB = String(versionB ?? '') - .trim() - .split('.') - .map(n => parseInt(n, 10) || 0); - - const maxLength = Math.max(partsA.length, partsB.length); - - for (let i = 0; i < maxLength; i++) { - const numA = partsA[i] ?? 0; - const numB = partsB[i] ?? 0; - if (numA !== numB) { - return numA < numB ? -1 : 1; // -1 if A < B, 1 if A > B - } - } - return 0; // versions are equal - } - private shouldSkipVersionTest(cqlEngine: CQLEngine, result: InternalTestResult): boolean { - const engineVersion = cqlEngine?.cqlVersion; - if (!engineVersion) return false; // no version to compare against - // Rule 1: if test.version is set, engine must be >= test.version - if (result.testVersion && this.compareVersions(engineVersion, result.testVersion) < 0) { - return true; - } - // Rule 2: if test.versionTo is set, engine must be <= test.versionTo - if (result.testVersionTo && this.compareVersions(engineVersion, result.testVersionTo) > 0) { - return true; - } - return false; // passes all checks - } - - private addToSkipList( - skipMap: Map, - testsName: string, - groupName: string, - testName: string, - reason: string - ): void { - skipMap.set(`${testsName}-${groupName}-${testName}`, reason); + return results; } } diff --git a/src/shared/results-shared.ts b/src/shared/results-shared.ts index b910d55..04973a5 100644 --- a/src/shared/results-shared.ts +++ b/src/shared/results-shared.ts @@ -48,7 +48,10 @@ export class Result implements InternalTestResult { } else { this.expected = test.output as string; } - } else { + } else if (this.invalid !== 'true' && this.invalid !== 'semantic') { + // No output is expected only when the expression is marked invalid ("true" + // for a run-time error, "semantic" for a translation error) — the test expects + // an error. Otherwise there is nothing to compare against, so skip. this.testStatus = 'skip'; this.skipMessage = 'No output specified'; } @@ -100,6 +103,26 @@ export async function generateEmptyResults( return groupResults; } +/** + * Determines whether a CQL evaluation response represents an error. Used to decide + * whether `invalid="true"`/`invalid="semantic"` tests pass (an error is expected). + * + * The engine does not always signal a run-time error with a non-2xx HTTP status: the + * FHIR `$cql`/`$evaluate` operations typically return HTTP 200 with a `Parameters` + * resource carrying an `evaluation error` parameter (an OperationOutcome). We treat + * both a non-2xx status and the presence of that parameter as an error. + */ +export function responseIndicatesError(status: number | undefined, responseBody: any): boolean { + if (status !== undefined && (status < 200 || status >= 300)) { + return true; + } + const parameters = responseBody?.parameter; + if (Array.isArray(parameters)) { + return parameters.some((p: any) => p?.name === 'evaluation error'); + } + return false; +} + export function generateParametersResource( result: InternalTestResult, cqlEndpoint: string diff --git a/src/shared/run-test-core.ts b/src/shared/run-test-core.ts new file mode 100644 index 0000000..e47e44e --- /dev/null +++ b/src/shared/run-test-core.ts @@ -0,0 +1,214 @@ +import { ConfigLoader } from '../conf/config-loader.js'; +import { CQLEngine } from '../cql-engine/cql-engine.js'; +import { TestLoader } from '../loaders/test-loader.js'; +import { generateParametersResource, responseIndicatesError } from './results-shared.js'; +import { InternalTestResult, Tests } from '../models/test-types.js'; +import { ServerConnectivity } from './server-connectivity.js'; +import { ResultExtractor } from '../extractors/result-extractor.js'; +import { buildExtractor } from '../server/extractor-builder.js'; +import { createConfigFromData } from '../server/config-utils.js'; +import { ValueMap } from '../extractors/value-map.js'; +import { resultsEqual } from './results-utils.js'; + +/** + * Shared execution state for a test run: the resolved config, the engine, the CVL parser, + * the loaded test suites, the value extractor, and the skip/only filters. Built once per + * run and threaded through every {@link runTest} call. + */ +export interface ExecutionContext { + config: ConfigLoader; + cqlEngine: CQLEngine; + cvl: any; + tests: Tests[]; + resultExtractor: ResultExtractor; + skipMap: Map; + onlySet: Set; +} + +/** + * Builds the shared execution context from config data: resolves config, verifies server + * connectivity, constructs the engine, loads the CVL parser and test suites, and builds the + * extractor and skip/only filters. Used by both the CLI ({@link TestRunner}) and server + * ({@link TestExecutionService}) entry points so they run tests identically. + */ +export async function createExecutionContext(configData: any): Promise { + const config = createConfigFromData(configData); + const serverBaseUrl = config.FhirServer.BaseUrl; + const cqlEndpoint = config.CqlEndpoint; + + await ServerConnectivity.verifyServerConnectivity(serverBaseUrl); + + const build = config.Build; + const cqlEngine = new CQLEngine( + serverBaseUrl, + cqlEndpoint, + build?.cqlTranslator ?? '', + build?.cqlTranslatorVersion ?? '', + build?.cqlEngine ?? '', + build?.cqlEngineVersion ?? '' + ); + cqlEngine.cqlVersion = config.Build?.CqlVersion || '1.5'; + + // @ts-expect-error - cvl.mjs has no declaration file + const cvlModule = await import('../../cvl/cvl.mjs'); + const cvl = cvlModule.default; + + const tests = TestLoader.load(); + const resultExtractor = buildExtractor(); + const skipMap = config.skipListMap(); + const onlySet = config.onlyListSet(); + + return { config, cqlEngine, cvl, tests, resultExtractor, skipMap, onlySet }; +} + +/** + * Compares two dotted version strings (e.g. "1.5.2"). Returns -1 if a < b, 1 if a > b, 0 if equal. + * Missing/blank segments are treated as 0. + */ +export function compareVersions(versionA: string | undefined, versionB: string | undefined): number { + const partsA = String(versionA ?? '') + .trim() + .split('.') + .map(n => parseInt(n, 10) || 0); + const partsB = String(versionB ?? '') + .trim() + .split('.') + .map(n => parseInt(n, 10) || 0); + + const maxLength = Math.max(partsA.length, partsB.length); + for (let i = 0; i < maxLength; i++) { + const numA = partsA[i] ?? 0; + const numB = partsB[i] ?? 0; + if (numA !== numB) { + return numA < numB ? -1 : 1; + } + } + return 0; +} + +/** + * Returns a human-readable reason if the test is out of scope for the engine's CQL version + * (engine older than the test's `version`, or newer than its `versionTo`), or null if it applies. + */ +export function versionSkipReason( + engineVersion: string | null | undefined, + result: InternalTestResult +): string | null { + if (!engineVersion) return null; + if (result.testVersion && compareVersions(engineVersion, result.testVersion) < 0) { + return `test version ${result.testVersion} not applicable to engine version ${engineVersion}`; + } + if (result.testVersionTo && compareVersions(engineVersion, result.testVersionTo) > 0) { + return `test versionTo ${result.testVersionTo} not applicable to engine version ${engineVersion}`; + } + return null; +} + +function logSkip(result: InternalTestResult): void { + console.log( + 'Test %s:%s:%s status: %s skipMessage: %s', + result.testsName, + result.groupName, + result.testName, + result.testStatus, + result.skipMessage + ); +} + +/** + * Runs a single test against the engine and records its outcome on `result`. Applies skip + * precedence (pre-marked skip → OnlyList → config SkipList → version gating), then POSTs the + * expression, extracts the actual value, and classifies pass/fail/error. Errors expected by + * `invalid="true"/"semantic"` tests pass only when the engine actually erred. + * + * This is the single implementation shared by the CLI and server runners — both use `fetch` and + * identical classification, so a test scores the same regardless of how it is invoked. + */ +export async function runTest( + result: InternalTestResult, + ctx: ExecutionContext +): Promise { + const { config, cqlEngine, cvl, resultExtractor, skipMap, onlySet } = ctx; + const apiUrl = cqlEngine.apiUrl!; + const key = `${result.testsName}-${result.groupName}-${result.testName}`; + + // Skip precedence. + if (result.testStatus === 'skip') { + if (!result.skipMessage?.trim()) { + result.skipMessage = 'Skipped by cql-tests-runner'; + } + logSkip(result); + return result; + } else if (onlySet.size > 0 && !onlySet.has(key)) { + result.testStatus = 'skip'; + result.skipMessage = 'Skipped by OnlyList filter'; + logSkip(result); + return result; + } else if (skipMap.has(key)) { + result.testStatus = 'skip'; + result.skipMessage = `Skipped by config: ${skipMap.get(key) || ''}`; + logSkip(result); + return result; + } + + // Version gating — applies to both the CLI and server paths. + const versionSkip = versionSkipReason(cqlEngine.cqlVersion, result); + if (versionSkip) { + result.testStatus = 'skip'; + result.skipMessage = `Skipped: ${versionSkip}`; + logSkip(result); + return result; + } + + const data = generateParametersResource(result, config.FhirServer.CqlOperation); + + try { + console.log('Running test %s:%s:%s', result.testsName, result.groupName, result.testName); + + const response = await fetch(apiUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(data), + }); + + result.responseStatus = response.status; + const responseBody = await response.json(); + const parsedExpected = + result.expected !== undefined ? cvl.parse(result.expected) : undefined; + result.actual = resultExtractor.extract(responseBody, { + singletonListKeys: ValueMap.singletonListKeysFromExpected(parsedExpected), + }); + const invalid = result.invalid; + const erroredOut = responseIndicatesError(response.status, responseBody); + + if (invalid === 'true' || invalid === 'semantic') { + // The expression is expected to error; it passes only if the engine erred. + result.testStatus = erroredOut ? 'pass' : 'fail'; + } else if (!erroredOut) { + result.testStatus = resultsEqual(parsedExpected, result.actual) ? 'pass' : 'fail'; + } else { + result.testStatus = 'fail'; + } + } catch (error: any) { + result.testStatus = 'error'; + result.error = { + message: error.message, + name: error.name || 'Error', + stack: error.stack, + }; + } + + console.log( + 'Test %s:%s:%s status: %s expected: %s actual: %s', + result.testsName, + result.groupName, + result.testName, + result.testStatus, + result.expected, + result.actual + ); + + return result; +} diff --git a/test/run-tests.test.ts b/test/run-tests.test.ts index 4f0d7a2..4932db5 100644 --- a/test/run-tests.test.ts +++ b/test/run-tests.test.ts @@ -16,7 +16,8 @@ const makeResult = (testsName: string, groupName: string, testName: string) => ( capability: [], }); -vi.mock('../src/shared/results-shared', () => ({ +vi.mock('../src/shared/results-shared', async orig => ({ + ...(await orig()), generateEmptyResults: vi .fn() .mockImplementation(async () => [