Skip to content

Dedupe runners for cli and server paths #108

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bryantaustin13 wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Dedupe runners for cli and server paths #108

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fe9ec27
Fix error-expecting test handling in CLI and server runners
bryantaustin13 Jul 22, 2026
1d69d38
Deduplicate CLI and server runners into a shared run-test core
bryantaustin13 Jul 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions src/commands/run-tests-command.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,8 @@ export class RunCommand {
// Convert ConfigLoader to the format expected by TestRunner
const configData = this.convertConfigToData(config, options.quick);

// Run tests using the shared TestRunner with axios for backward compatibility
const results = await this.testRunner.runTests(configData, {
useAxios: true, // Use axios for backward compatibility with existing behavior
});
// Run tests using the shared TestRunner
const results = await this.testRunner.runTests(configData);

// Validate before saving if validation option is enabled
if (options.validate) {
Expand Down
1 change: 0 additions & 1 deletion src/models/test-types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,6 @@ export interface InternalTestResult {
invalid?: 'false' | 'true' | 'semantic' | 'undefined';
expression: string;
capability?: CapabilityKV[];
SkipMessage?: string;
}

// Schema-compliant TestResult type (strictly matches cql-test-results.schema.json)
Expand Down
292 changes: 80 additions & 212 deletions src/server/test-execution-service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,217 +1,85 @@
// Author: Preston Lee

import { ConfigLoader } from '../conf/config-loader.js';
import { CQLEngine } from '../cql-engine/cql-engine.js';
import { CQLTestResults } from '../test-results/cql-test-results.js';
import { TestLoader } from '../loaders/test-loader.js';
import {
generateEmptyResults,
generateParametersResource,
Result,
} from '../shared/results-shared.js';
import { InternalTestResult, Tests } from '../models/test-types.js';
import { ServerConnectivity } from '../shared/server-connectivity.js';
import { ResultExtractor } from '../extractors/result-extractor.js';
import { buildExtractor } from './extractor-builder.js';
import { createConfigFromData } from './config-utils.js';
import { ValueMap } from '../extractors/value-map.js';
import { resultsEqual } from '../shared/results-utils.js';

interface ExecutionContext {
config: ConfigLoader;
cqlEngine: CQLEngine;
cvl: any;
tests: Tests[];
resultExtractor: ResultExtractor;
skipMap: Map<string, string>;
onlySet: Set<string>;
}

import { generateEmptyResults, Result } from '../shared/results-shared.js';
import { createExecutionContext, runTest } from '../shared/run-test-core.js';

/**
* Server/MCP-facing runner. Thin wrapper over the shared {@link runTest} core: builds the
* execution context and returns JSON-shaped results. Shares identical execution and
* classification with the CLI {@link TestRunner}.
*/
export class TestExecutionService {
/**
* Builds shared execution context from config data (engine, CVL, tests, extractor, skip map).
*/
private async createExecutionContext(configData: any): Promise<ExecutionContext> {
const config = createConfigFromData(configData);
const serverBaseUrl = config.FhirServer.BaseUrl;
const cqlEndpoint = config.CqlEndpoint;

await ServerConnectivity.verifyServerConnectivity(serverBaseUrl);

const build = config.Build;
const cqlEngine = new CQLEngine(
serverBaseUrl,
cqlEndpoint,
build?.cqlTranslator ?? '',
build?.cqlTranslatorVersion ?? '',
build?.cqlEngine ?? '',
build?.cqlEngineVersion ?? ''
);
cqlEngine.cqlVersion = config.Build?.CqlVersion || '1.5';

// @ts-expect-error - cvl.mjs has no declaration file
const cvlModule = await import('../../cvl/cvl.mjs');
const cvl = cvlModule.default;

const tests = TestLoader.load();
const resultExtractor = buildExtractor();
const skipMap = config.skipListMap();
const onlySet = config.onlyListSet();

return { config, cqlEngine, cvl, tests, resultExtractor, skipMap, onlySet };
}

/**
* Runs a single test
*/
private async runTest(
result: InternalTestResult,
apiUrl: string,
cvl: any,
resultExtractor: ResultExtractor,
skipMap: Map<string, string>,
onlySet: Set<string>,
config: ConfigLoader
): Promise<InternalTestResult> {
const key = `${result.testsName}-${result.groupName}-${result.testName}`;

if (result.testStatus === 'skip') {
result.SkipMessage = 'Skipped by cql-tests-runner';
return result;
} else if (onlySet.size > 0 && !onlySet.has(key)) {
result.SkipMessage = 'Skipped by OnlyList filter';
result.testStatus = 'skip';
return result;
} else if (skipMap.has(key)) {
const reason = skipMap.get(key) || '';
result.SkipMessage = `Skipped by config: ${reason}`;
result.testStatus = 'skip';
return result;
}

const data = generateParametersResource(result, config.FhirServer.CqlOperation);

try {
console.log('Running test %s:%s:%s', result.testsName, result.groupName, result.testName);
const response = await fetch(apiUrl, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(data)
});

result.responseStatus = response.status;
const responseBody = await response.json();
const parsedExpected = cvl.parse(result.expected);
result.actual = resultExtractor.extract(responseBody, {
singletonListKeys: ValueMap.singletonListKeysFromExpected(parsedExpected),
});
const invalid = result.invalid;

if (invalid === 'true' || invalid === 'semantic') {
result.testStatus = response.status === 200 ? 'fail' : 'pass';
} else {
if (response.status === 200) {
result.testStatus = resultsEqual(parsedExpected, result.actual) ? 'pass' : 'fail';
} else {
result.testStatus = 'fail';
}
}
} catch (error: any) {
result.testStatus = 'error';
result.error = {
message: error.message,
name: error.name || 'Error',
stack: error.stack
};
}

console.log('Test %s:%s:%s status: %s expected: %s actual: %s',
result.testsName, result.groupName, result.testName, result.testStatus, result.expected, result.actual);

return result;
}

/**
* Runs all tests based on configuration
*/
async runTests(configData: any): Promise<any> {
const ctx = await this.createExecutionContext(configData);
const { config, cqlEngine, cvl, tests, resultExtractor, skipMap, onlySet } = ctx;

const quickTest = config.Debug?.QuickTest || false;
const emptyResults = await generateEmptyResults(tests, quickTest);
const results = new CQLTestResults(cqlEngine);

for (const testFile of emptyResults) {
for (const result of testFile) {
await this.runTest(result, cqlEngine.apiUrl!, cvl, resultExtractor, skipMap, onlySet, config);
results.add(result);
}
}

return results.toJSON();
}

/**
* Runs a single test by identifier
*/
async runSingleTest(
testsName: string,
groupName: string,
testName: string,
configData: any
): Promise<any> {
const ctx = await this.createExecutionContext(configData);
const { config, cqlEngine, cvl, tests, resultExtractor, skipMap, onlySet } = ctx;

for (const testSuite of tests) {
if (testSuite.name !== testsName) continue;
for (const group of testSuite.group) {
if (group.name !== groupName || !group.test) continue;
for (const test of group.test) {
if (test.name !== testName) continue;

const result = new Result(testsName, groupName, test);
await this.runTest(result, cqlEngine.apiUrl!, cvl, resultExtractor, skipMap, onlySet, config);

const testResults = new CQLTestResults(cqlEngine);
testResults.add(result);
return testResults.toJSON().results[0] ?? null;
}
}
}

throw new Error(`Test not found: ${testsName}/${groupName}/${testName}`);
}

/**
* Runs all tests in a group
*/
async runTestGroup(
testsName: string,
groupName: string,
configData: any
): Promise<any[]> {
const ctx = await this.createExecutionContext(configData);
const { config, cqlEngine, cvl, tests, resultExtractor, skipMap, onlySet } = ctx;

const results = new CQLTestResults(cqlEngine);

for (const testSuite of tests) {
if (testSuite.name !== testsName) continue;
for (const group of testSuite.group) {
if (group.name !== groupName || !group.test) continue;
for (const test of group.test) {
const result = new Result(testsName, groupName, test);
await this.runTest(result, cqlEngine.apiUrl!, cvl, resultExtractor, skipMap, onlySet, config);
results.add(result);
}
return results.toJSON().results;
}
}

return results.toJSON().results;
}
/**
* Runs all tests based on configuration.
*/
async runTests(configData: any): Promise<any> {
const ctx = await createExecutionContext(configData);
const quickTest = ctx.config.Debug?.QuickTest || false;
const emptyResults = await generateEmptyResults(ctx.tests, quickTest);
const results = new CQLTestResults(ctx.cqlEngine);

for (const testFile of emptyResults) {
for (const result of testFile) {
await runTest(result, ctx);
results.add(result);
}
}

return results.toJSON();
}

/**
* Runs a single test by identifier.
*/
async runSingleTest(
testsName: string,
groupName: string,
testName: string,
configData: any
): Promise<any> {
const ctx = await createExecutionContext(configData);

for (const testSuite of ctx.tests) {
if (testSuite.name !== testsName) continue;
for (const group of testSuite.group) {
if (group.name !== groupName || !group.test) continue;
for (const test of group.test) {
if (test.name !== testName) continue;

const result = new Result(testsName, groupName, test);
await runTest(result, ctx);

const testResults = new CQLTestResults(ctx.cqlEngine);
testResults.add(result);
return testResults.toJSON().results[0] ?? null;
}
}
}

throw new Error(`Test not found: ${testsName}/${groupName}/${testName}`);
}

/**
* Runs all tests in a group.
*/
async runTestGroup(testsName: string, groupName: string, configData: any): Promise<any[]> {
const ctx = await createExecutionContext(configData);
const results = new CQLTestResults(ctx.cqlEngine);

for (const testSuite of ctx.tests) {
if (testSuite.name !== testsName) continue;
for (const group of testSuite.group) {
if (group.name !== groupName || !group.test) continue;
for (const test of group.test) {
const result = new Result(testsName, groupName, test);
await runTest(result, ctx);
results.add(result);
}
return results.toJSON().results;
}
}

return results.toJSON().results;
}
}
Loading