Skip to content

Commit bd79920

Browse files
aaronziCopilot
aaronzi
and
Copilot
authored
Improves assertion errors (#535)
* Improves assertion errors Co-authored-by: Copilot <copilot@github.com> * Fixes formatting issues * fixes TS errors Co-authored-by: Copilot <copilot@github.com> * Adds missing status assertions Co-authored-by: Copilot <copilot@github.com> * Adds further status code assertions Co-authored-by: Copilot <copilot@github.com> --------- Co-authored-by: Copilot <copilot@github.com>
1 parent 968b00e commit bd79920

11 files changed

Lines changed: 918 additions & 462 deletions

‎scripts/test-engine.mjs‎

Lines changed: 58 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,40 @@ function escapeXml(value) {
251251
.replaceAll("'", '&apos;');
252252
}
253253

254+
function normalizeFailureMessage(message) {
255+
if (!message) {
256+
return '';
257+
}
258+
259+
return String(message).trim();
260+
}
261+
262+
function sanitizeFailureMessages(messages) {
263+
if (!Array.isArray(messages)) {
264+
return [];
265+
}
266+
267+
return messages.map((message) => normalizeFailureMessage(message)).filter(Boolean);
268+
}
269+
270+
function compactFailureReason(message) {
271+
const normalized = normalizeFailureMessage(message);
272+
if (!normalized) {
273+
return '';
274+
}
275+
276+
const firstLine = normalized
277+
.split('\n')
278+
.map((line) => line.trim())
279+
.find(Boolean);
280+
281+
return firstLine || normalized;
282+
}
283+
284+
function formatFailureDetailLine(message) {
285+
return normalizeFailureMessage(message).replace(/\s+/g, ' ').trim();
286+
}
287+
254288
function parseVitestResults(vitestReport) {
255289
const passedChecks = [];
256290
const failedChecks = [];
@@ -260,12 +294,14 @@ function parseVitestResults(vitestReport) {
260294
let suiteHasFailedAssertion = false;
261295

262296
for (const assertion of suite.assertionResults ?? []) {
297+
const failureMessages = sanitizeFailureMessages(assertion.failureMessages);
263298
const check = {
264299
checkType: 'integration-test',
265300
suite: suite.name,
266301
name: assertion.fullName || assertion.title,
267302
durationMs: assertion.duration,
268-
reason: assertion.failureMessages?.[0] || '',
303+
reason: failureMessages[0] || '',
304+
failureMessages,
269305
status: assertion.status,
270306
};
271307

@@ -284,12 +320,14 @@ function parseVitestResults(vitestReport) {
284320
}
285321

286322
if (suite.status === 'failed' && !suiteHasFailedAssertion) {
323+
const suiteMessage = normalizeFailureMessage(suite.message);
287324
failedChecks.push({
288325
checkType: 'integration-suite',
289326
suite: suite.name,
290327
name: `Suite failure: ${suite.name}`,
291328
durationMs: 0,
292-
reason: suite.message || 'Suite failed before assertions (setup hook or runtime error).',
329+
reason: suiteMessage || 'Suite failed before assertions (setup hook or runtime error).',
330+
failureMessages: suiteMessage ? [suiteMessage] : [],
293331
status: 'failed',
294332
});
295333
}
@@ -449,7 +487,15 @@ function writeMarkdownReport(report, outputPath) {
449487
if (failure.suite) {
450488
lines.push(` - suite: ${failure.suite}`);
451489
}
452-
lines.push(` - reason: ${failure.reason || 'No failure reason provided'}`);
490+
491+
const primaryReason = failure.reason || failure.failureMessages?.[0] || 'No failure reason provided';
492+
lines.push(` - reason: ${primaryReason}`);
493+
494+
if (Array.isArray(failure.failureMessages) && failure.failureMessages.length > 1) {
495+
for (const extraMessage of failure.failureMessages.slice(1, 5)) {
496+
lines.push(` - detail: ${formatFailureDetailLine(extraMessage)}`);
497+
}
498+
}
453499
}
454500
}
455501
lines.push('');
@@ -498,7 +544,7 @@ function writeJunitReport(report, outputPath) {
498544
name: check.name,
499545
status: 'failed',
500546
durationMs: check.durationMs ?? 0,
501-
message: check.reason || 'No failure message',
547+
message: compactFailureReason(check.reason || check.failureMessages?.[0] || 'No failure message'),
502548
});
503549
}
504550

@@ -550,7 +596,14 @@ function printConsoleSummary(report, artifactPaths) {
550596
console.log('Failed checks:');
551597
for (const failure of report.failedChecks) {
552598
console.log(`- [${failure.checkType}] ${failure.name}`);
553-
console.log(` reason: ${failure.reason || 'No failure reason provided'}`);
599+
const primaryReason = failure.reason || failure.failureMessages?.[0] || 'No failure reason provided';
600+
console.log(` reason: ${primaryReason}`);
601+
602+
if (Array.isArray(failure.failureMessages) && failure.failureMessages.length > 1) {
603+
for (const extraMessage of failure.failureMessages.slice(1, 3)) {
604+
console.log(` detail: ${formatFailureDetailLine(extraMessage)}`);
605+
}
606+
}
554607
}
555608
}
556609

‎src/integration-tests/aasDiscovery.integration.test.ts‎

Lines changed: 18 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { AasDiscoveryClient } from '../clients/AasDiscoveryClient';
33
import { Configuration } from '../generated';
44
import { base64Encode } from '../lib/base64Url';
55
import { createTestSpecificAssetId1, createTestSpecificAssetId2 } from './fixtures/aasDiscoveryFixtures';
6+
import { assertApiFailure, assertApiResult } from './fixtures/assertionHelpers';
67
import { createPerTestCleanupRunner } from './fixtures/testCleanup';
78
import { getIntegrationBasePath } from './testEngineConfig';
89

@@ -55,10 +56,7 @@ describe('AAS Discovery Integration Tests', () => {
5556
specificAssetId: [specificAssetId1, specificAssetId2],
5657
});
5758

58-
expect(createResponse.success).toBe(true);
59-
if (!createResponse.success) {
60-
throw new Error('Failed to seed AAS discovery asset links fixture');
61-
}
59+
assertApiResult(createResponse, 'Seed AAS discovery asset links fixture');
6260

6361
registerAssetLinkCleanup(aasIdentifier);
6462
return { aasIdentifier, specificAssetId1, specificAssetId2 };
@@ -79,7 +77,7 @@ describe('AAS Discovery Integration Tests', () => {
7977
specificAssetId: [specificAssetId1, specificAssetId2],
8078
});
8179

82-
expect(response.success).toBe(true);
80+
assertApiResult(response);
8381
if (response.success) {
8482
registerAssetLinkCleanup(aasIdentifier);
8583
expect(response.statusCode).toBe(201);
@@ -100,7 +98,7 @@ describe('AAS Discovery Integration Tests', () => {
10098
aasIdentifier,
10199
});
102100

103-
expect(response.success).toBe(true);
101+
assertApiResult(response);
104102
if (response.success) {
105103
expect(response.statusCode).toBe(200);
106104
expect(response.data).toBeDefined();
@@ -118,7 +116,7 @@ describe('AAS Discovery Integration Tests', () => {
118116
configuration,
119117
aasIdentifier: nonExistingId,
120118
});
121-
expect(response.success).toBe(false);
119+
assertApiFailure(response);
122120
if (!response.success) {
123121
expect(response.statusCode).toBe(404);
124122
expect(response.error).toBeDefined();
@@ -153,7 +151,7 @@ describe('AAS Discovery Integration Tests', () => {
153151
assetIds: [specificAssetId1, specificAssetId2],
154152
});
155153

156-
expect(response.success).toBe(true);
154+
assertApiResult(response);
157155
if (response.success) {
158156
expect(response.statusCode).toBe(200);
159157
expect(response.data).toBeDefined();
@@ -176,7 +174,7 @@ describe('AAS Discovery Integration Tests', () => {
176174
limit: -1,
177175
});
178176

179-
expect(response.success).toBe(false);
177+
assertApiFailure(response);
180178
if (!response.success) {
181179
expect(response.statusCode).toBe(400);
182180
expect(response.error.messages?.[0]?.code).toBe('400');
@@ -197,7 +195,7 @@ describe('AAS Discovery Integration Tests', () => {
197195
cursor: unavailableCursor(),
198196
});
199197

200-
expect(response.success).toBe(true);
198+
assertApiResult(response);
201199
if (response.success) {
202200
expect(response.statusCode).toBe(200);
203201
expect(response.data.pagedResult).toEqual({});
@@ -217,7 +215,7 @@ describe('AAS Discovery Integration Tests', () => {
217215
assetLink: [specificAssetId1, specificAssetId2],
218216
});
219217

220-
expect(response.success).toBe(true);
218+
assertApiResult(response);
221219
if (response.success) {
222220
expect(response.statusCode).toBe(200);
223221
expect(response.data.result).toContainEqual(aasIdentifier);
@@ -234,7 +232,7 @@ describe('AAS Discovery Integration Tests', () => {
234232
assetLink: [{ name: 'globalAssetId' } as unknown as CoreSpecificAssetId],
235233
});
236234

237-
expect(response.success).toBe(false);
235+
assertApiFailure(response);
238236
if (!response.success) {
239237
expect(response.statusCode).toBe(400);
240238
expect(response.error.messages?.[0]?.code).toBe('400');
@@ -255,7 +253,7 @@ describe('AAS Discovery Integration Tests', () => {
255253
cursor: unavailableCursor(),
256254
});
257255

258-
expect(response.success).toBe(true);
256+
assertApiResult(response);
259257
if (response.success) {
260258
expect(response.statusCode).toBe(200);
261259
expect(response.data.pagedResult).toEqual({});
@@ -277,7 +275,7 @@ describe('AAS Discovery Integration Tests', () => {
277275
specificAssetId: [invalidSpecificAssetId],
278276
});
279277

280-
expect(response.success).toBe(false);
278+
assertApiFailure(response);
281279
if (!response.success) {
282280
expect(response.statusCode).toBe(400);
283281
expect(response.error.messages?.[0]?.code).toBe('400');
@@ -297,7 +295,7 @@ describe('AAS Discovery Integration Tests', () => {
297295
specificAssetId: [specificAssetId1],
298296
});
299297

300-
expect(response.success).toBe(false);
298+
assertApiFailure(response);
301299
if (!response.success) {
302300
expect(response.statusCode).toBe(405);
303301
expect(response.error.messages?.[0]?.code).toBe('405');
@@ -317,7 +315,7 @@ describe('AAS Discovery Integration Tests', () => {
317315
specificAssetId: [specificAssetId1],
318316
});
319317

320-
expect(response.success).toBe(false);
318+
assertApiFailure(response);
321319
if (!response.success) {
322320
expect(response.statusCode).toBe(404);
323321
expect(response.error.messages?.[0]?.code).toBe('404');
@@ -358,7 +356,7 @@ describe('AAS Discovery Integration Tests', () => {
358356
aasIdentifier,
359357
});
360358

361-
expect(response.success).toBe(true);
359+
assertApiResult(response);
362360
if (response.success) {
363361
expect(response.statusCode).toBe(204);
364362
expect(response.data).toBeUndefined();
@@ -377,7 +375,7 @@ describe('AAS Discovery Integration Tests', () => {
377375
aasIdentifier: nonExistingAasIdentifier,
378376
});
379377

380-
expect(response.success).toBe(false);
378+
assertApiFailure(response);
381379
if (!response.success) {
382380
expect(response.statusCode).toBe(404);
383381
expect(response.error.messages?.[0]?.code).toBe('404');
@@ -394,7 +392,7 @@ describe('AAS Discovery Integration Tests', () => {
394392
aasIdentifier: '',
395393
});
396394

397-
expect(response.success).toBe(false);
395+
assertApiFailure(response);
398396
if (!response.success) {
399397
expect(response.statusCode).toBe(405);
400398
expect(response.error.messages?.[0]?.code).toBe('405');
@@ -410,7 +408,7 @@ describe('AAS Discovery Integration Tests', () => {
410408
configuration,
411409
});
412410

413-
expect(response.success).toBe(true);
411+
assertApiResult(response);
414412
if (response.success) {
415413
expect(response.statusCode).toBe(200);
416414
expect(response.data).toBeDefined();

0 commit comments

Comments
 (0)