Skip to content

Commit 955c91c

Browse files
RomanHotsiyRomanHotsiy
authored
feat: add junit output support to scorecard-classic (#2727)
* Add junit output for scorecard-classic * Refine scorecard formatter helpers * Tighten junit scorecard output * Simplify scorecard output format type * Align junit formatter with json output * Trim redundant junit formatter tests * Fix packaged smoke regression and junit metadata * Revert scorecard config fallback workaround * chore: move strip-ansi-codes to utils
1 parent 1076252 commit 955c91c

10 files changed

Lines changed: 344 additions & 12 deletions

File tree

‎.changeset/flat-dingos-play.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@redocly/cli": minor
3+
---
4+
5+
Added support for `junit` output in the `scorecard-classic` command.

‎docs/@v2/commands/scorecard-classic.md‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ The `scorecard-classic` command requires a scorecard configuration in your Redoc
1515
redocly scorecard-classic <api> --project-url=<url>
1616
redocly scorecard-classic <api> --config=<path>
1717
redocly scorecard-classic <api> --format=json
18+
redocly scorecard-classic <api> --format=junit
1819
redocly scorecard-classic <api> --target-level=<level>
1920
redocly scorecard-classic <api> --verbose
2021
```
@@ -25,7 +26,7 @@ redocly scorecard-classic <api> --verbose
2526
| -------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
2627
| api | string | Path to the API description filename or alias that you want to evaluate. See [the API section](#specify-api) for more details. |
2728
| --config | string | Specify path to the [configuration file](#use-alternative-configuration-file). |
28-
| --format | string | Format for the output.<br />**Possible values:** `stylish`, `json`, `checkstyle`. Default value is `stylish`. |
29+
| --format | string | Format for the output.<br />**Possible values:** `stylish`, `json`, `checkstyle`, `junit`. Default value is `stylish`. |
2930
| --help | boolean | Show help. |
3031
| --project-url | string | URL to the project scorecard configuration. Required if not configured in the Redocly configuration file. Example: `https://app.cloud.redocly.com/org/my-org/projects/my-project/scorecard-classic`. |
3132
| --target-level | string | Target scorecard level to achieve. The command validates that the API meets this level and all preceding levels without errors. Exits with an error if the target level is not achieved. |
@@ -89,6 +90,21 @@ The JSON output is grouped by scorecard level and includes:
8990
- location information (file path, line/column range, and JSON pointer)
9091
- descriptive message about the violation
9192

93+
### Use JUnit output format
94+
95+
To generate JUnit XML for test reporting integrations, use the JUnit format:
96+
97+
```bash
98+
redocly scorecard-classic openapi/openapi.yaml --format=junit
99+
```
100+
101+
The JUnit output includes:
102+
103+
- one test suite for the `scorecard-classic` run
104+
- one test case per reported issue
105+
- scorecard level, rule, severity, and location details for each issue
106+
- warnings as skipped test cases and errors as failed test cases
107+
92108
### Validate against a target level
93109

94110
Use the `--target-level` option to ensure your API meets a specific quality level. The command validates that your API satisfies the target level and all preceding levels without errors:
@@ -149,7 +165,7 @@ The scorecard evaluation categorizes issues into multiple levels based on your p
149165
Each issue is associated with a specific scorecard level, allowing you to prioritize improvements.
150166

151167
The command displays the achieved scorecard level, which is the highest level your API meets without errors.
152-
The achieved level is shown in both stylish, JSON and checkstyle output formats.
168+
The achieved level is shown in stylish, JSON, checkstyle, and JUnit output formats.
153169

154170
When all checks pass, the command displays a success message:
155171

‎packages/cli/src/commands/scorecard-classic/__tests__/json-formatter.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,7 @@ describe('printScorecardResultsAsJson', () => {
188188
it('should strip ANSI codes from messages', () => {
189189
const problems: ScorecardProblem[] = [
190190
{
191-
message: '\u001b[31mError message with color\u001b[0m',
191+
message: '\u001b[1;31mError message with color\u001b[0m',
192192
ruleId: 'test-rule',
193193
severity: 'error',
194194
suggest: [],

‎packages/cli/src/commands/scorecard-classic/__tests__/junit-formatter.test.ts‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
import * as openapiCore from '@redocly/openapi-core';
2+
3+
import { printScorecardResultsAsJunit } from '../formatters/junit-formatter.js';
4+
import type { ScorecardProblem } from '../types.js';
5+
6+
const createMockSource = (absoluteRef: string) => ({
7+
absoluteRef,
8+
getAst: () => ({}),
9+
getRootAst: () => ({}),
10+
getLineColLocation: () => ({ line: 1, col: 1 }),
11+
});
12+
13+
describe('printScorecardResultsAsJunit', () => {
14+
beforeEach(() => {
15+
vi.spyOn(openapiCore.logger, 'output').mockImplementation(() => {});
16+
vi.spyOn(openapiCore.logger, 'info').mockImplementation(() => {});
17+
});
18+
19+
const getOutput = () =>
20+
(openapiCore.logger.output as any).mock.calls.map((call: any) => call[0]).join('');
21+
22+
it('outputs an empty junit suite when there are no problems', () => {
23+
printScorecardResultsAsJunit('/api/openapi.yaml', [], 'Gold', true);
24+
25+
expect(getOutput()).toMatchInlineSnapshot(`
26+
"<?xml version="1.0" encoding="UTF-8"?>
27+
<testsuites name="redocly scorecard-classic" tests="0" failures="0" errors="0" skipped="0">
28+
<testsuite name="scorecard-classic" tests="0" failures="0" errors="0" skipped="0">
29+
<properties>
30+
<property name="api" value="/api/openapi.yaml" />
31+
<property name="achievedLevel" value="Gold" />
32+
</properties>
33+
</testsuite>
34+
</testsuites>
35+
36+
"
37+
`);
38+
});
39+
40+
it('maps errors to failures and warnings to skipped test cases', () => {
41+
const problems: ScorecardProblem[] = [
42+
{
43+
message: 'Missing summary',
44+
ruleId: 'operation-summary',
45+
severity: 'error',
46+
suggest: [],
47+
location: [
48+
{
49+
source: createMockSource('/api/openapi.yaml') as any,
50+
pointer: '#/paths/~1pets/get/summary',
51+
reportOnKey: false,
52+
},
53+
],
54+
scorecardLevel: 'Gold',
55+
},
56+
{
57+
message: 'Missing description',
58+
ruleId: 'operation-description',
59+
severity: 'warn',
60+
suggest: [],
61+
location: [
62+
{
63+
source: createMockSource('/api/openapi.yaml') as any,
64+
pointer: '#/info',
65+
reportOnKey: false,
66+
},
67+
],
68+
scorecardLevel: 'Silver',
69+
},
70+
];
71+
72+
printScorecardResultsAsJunit('/api/openapi.yaml', problems, 'Silver', false);
73+
74+
expect(getOutput()).toMatchInlineSnapshot(`
75+
"<?xml version="1.0" encoding="UTF-8"?>
76+
<testsuites name="redocly scorecard-classic" tests="2" failures="1" errors="0" skipped="1">
77+
<testsuite name="scorecard-classic" tests="2" failures="1" errors="0" skipped="1">
78+
<properties>
79+
<property name="api" value="/api/openapi.yaml" />
80+
</properties>
81+
<testcase classname="Gold" name="operation-summary" file="/api/openapi.yaml" line="1">
82+
<failure message="Missing summary" type="operation-summary">Level: Gold
83+
Rule: operation-summary
84+
Severity: error
85+
File: /api/openapi.yaml
86+
Line: 1
87+
Column: 1
88+
Pointer: #/paths/~1pets/get/summary
89+
Message: Missing summary</failure>
90+
</testcase>
91+
<testcase classname="Silver" name="operation-description" file="/api/openapi.yaml" line="1">
92+
<skipped message="Missing description">Level: Silver
93+
Rule: operation-description
94+
Severity: warn
95+
File: /api/openapi.yaml
96+
Line: 1
97+
Column: 1
98+
Pointer: #/info
99+
Message: Missing description</skipped>
100+
</testcase>
101+
</testsuite>
102+
</testsuites>
103+
104+
"
105+
`);
106+
});
107+
108+
it('XML-escapes details and strips ANSI sequences from messages', () => {
109+
const problems: ScorecardProblem[] = [
110+
{
111+
message: '\u001b[1;31mValue must be < 5 & > 0\u001b[0m',
112+
ruleId: 'custom/my-rule',
113+
severity: 'error',
114+
suggest: [],
115+
location: [
116+
{
117+
source: createMockSource('/api/openapi.yaml') as any,
118+
pointer: '#/info',
119+
reportOnKey: false,
120+
},
121+
],
122+
scorecardLevel: 'Gold',
123+
},
124+
];
125+
126+
printScorecardResultsAsJunit('/api/openapi.yaml', problems, 'Gold', false);
127+
128+
expect(getOutput()).toMatchInlineSnapshot(`
129+
"<?xml version="1.0" encoding="UTF-8"?>
130+
<testsuites name="redocly scorecard-classic" tests="1" failures="1" errors="0" skipped="0">
131+
<testsuite name="scorecard-classic" tests="1" failures="1" errors="0" skipped="0">
132+
<properties>
133+
<property name="api" value="/api/openapi.yaml" />
134+
</properties>
135+
<testcase classname="Gold" name="custom/my-rule" file="/api/openapi.yaml" line="1">
136+
<failure message="Value must be &lt; 5 &amp; &gt; 0" type="custom/my-rule">Level: Gold
137+
Rule: custom/my-rule
138+
Severity: error
139+
File: /api/openapi.yaml
140+
Line: 1
141+
Column: 1
142+
Pointer: #/info
143+
Message: Value must be &lt; 5 &amp; &gt; 0</failure>
144+
</testcase>
145+
</testsuite>
146+
</testsuites>
147+
148+
"
149+
`);
150+
});
151+
152+
it('falls back to the api path and zero coordinates when a problem has no location', () => {
153+
const problems: ScorecardProblem[] = [
154+
{
155+
message: 'No location error',
156+
ruleId: 'some-rule',
157+
severity: 'error',
158+
suggest: [],
159+
location: [],
160+
scorecardLevel: undefined,
161+
},
162+
];
163+
164+
printScorecardResultsAsJunit('/api/openapi.yaml', problems, 'Non Conformant', false);
165+
166+
expect(getOutput()).toMatchInlineSnapshot(`
167+
"<?xml version="1.0" encoding="UTF-8"?>
168+
<testsuites name="redocly scorecard-classic" tests="1" failures="1" errors="0" skipped="0">
169+
<testsuite name="scorecard-classic" tests="1" failures="1" errors="0" skipped="0">
170+
<properties>
171+
<property name="api" value="/api/openapi.yaml" />
172+
</properties>
173+
<testcase classname="Unknown" name="some-rule" file="/api/openapi.yaml" line="0">
174+
<failure message="No location error" type="some-rule">Level: Unknown
175+
Rule: some-rule
176+
Severity: error
177+
File: /api/openapi.yaml
178+
Line: 0
179+
Column: 0
180+
Message: No location error</failure>
181+
</testcase>
182+
</testsuite>
183+
</testsuites>
184+
185+
"
186+
`);
187+
});
188+
});

‎packages/cli/src/commands/scorecard-classic/formatters/json-formatter.ts‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { logger, getLineColLocation } from '@redocly/openapi-core';
22

3+
import { stripAnsiCodes } from '../../../utils/strip-ansi-codes.js';
34
import type { ScorecardProblem } from '../types.js';
45

56
type ScorecardLevel = {
@@ -46,11 +47,6 @@ function getRuleUrl(ruleId: string): string | undefined {
4647
return undefined;
4748
}
4849

49-
function stripAnsiCodes(text: string): string {
50-
// eslint-disable-next-line no-control-regex
51-
return text.replace(/\u001b\[\d+m/g, '');
52-
}
53-
5450
export function printScorecardResultsAsJson(
5551
problems: ScorecardProblem[],
5652
achievedLevel: string,

‎packages/cli/src/commands/scorecard-classic/formatters/junit-formatter.ts‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
import { getLineColLocation, logger, xmlEscape } from '@redocly/openapi-core';
2+
3+
import { stripAnsiCodes } from '../../../utils/strip-ansi-codes.js';
4+
import type { ScorecardProblem } from '../types.js';
5+
6+
type ProblemLocation = {
7+
file: string;
8+
line: number;
9+
column: number;
10+
pointer?: string;
11+
};
12+
13+
function formatDetail(label: string, value: string | number): string {
14+
return `${label}: ${xmlEscape(String(value))}`;
15+
}
16+
17+
function getProblemLocation(problem: ScorecardProblem, apiPath: string): ProblemLocation {
18+
const location = problem.location[0];
19+
20+
if (!location) {
21+
return {
22+
file: apiPath,
23+
line: 0,
24+
column: 0,
25+
};
26+
}
27+
28+
const lineColLocation = getLineColLocation(location);
29+
30+
return {
31+
file: location.source.absoluteRef,
32+
line: lineColLocation.start.line,
33+
column: lineColLocation.start.col,
34+
pointer: location.pointer,
35+
};
36+
}
37+
38+
function formatProblemDetails(problem: ScorecardProblem, location: ProblemLocation): string {
39+
const details = [
40+
formatDetail('Level', problem.scorecardLevel || 'Unknown'),
41+
formatDetail('Rule', problem.ruleId),
42+
formatDetail('Severity', problem.severity),
43+
formatDetail('File', location.file),
44+
formatDetail('Line', location.line),
45+
formatDetail('Column', location.column),
46+
];
47+
48+
if (location.pointer) {
49+
details.push(formatDetail('Pointer', location.pointer));
50+
}
51+
52+
details.push(formatDetail('Message', stripAnsiCodes(problem.message)));
53+
54+
return details.join('\n');
55+
}
56+
57+
export function printScorecardResultsAsJunit(
58+
path: string,
59+
problems: ScorecardProblem[],
60+
achievedLevel: string,
61+
targetLevelAchieved: boolean
62+
): void {
63+
const failures = problems.filter((problem) => problem.severity === 'error').length;
64+
const skipped = problems.filter((problem) => problem.severity === 'warn').length;
65+
66+
logger.output('<?xml version="1.0" encoding="UTF-8"?>\n');
67+
logger.output(
68+
`<testsuites name="redocly scorecard-classic" tests="${problems.length}" failures="${failures}" errors="0" skipped="${skipped}">\n`
69+
);
70+
logger.output(
71+
`<testsuite name="scorecard-classic" tests="${problems.length}" failures="${failures}" errors="0" skipped="${skipped}">\n`
72+
);
73+
logger.output('<properties>\n');
74+
logger.output(`<property name="api" value="${xmlEscape(path)}" />\n`);
75+
if (targetLevelAchieved) {
76+
logger.output(`<property name="achievedLevel" value="${xmlEscape(achievedLevel)}" />\n`);
77+
}
78+
logger.output('</properties>\n');
79+
80+
for (const problem of problems) {
81+
const location = getProblemLocation(problem, path);
82+
const level = problem.scorecardLevel || 'Unknown';
83+
const message = stripAnsiCodes(problem.message);
84+
const details = formatProblemDetails(problem, location);
85+
86+
logger.output(
87+
`<testcase classname="${xmlEscape(level)}" name="${xmlEscape(
88+
problem.ruleId
89+
)}" file="${xmlEscape(location.file)}" line="${location.line}">\n`
90+
);
91+
92+
if (problem.severity === 'warn') {
93+
logger.output(`<skipped message="${xmlEscape(message)}">${details}</skipped>\n`);
94+
} else {
95+
logger.output(
96+
`<failure message="${xmlEscape(message)}" type="${xmlEscape(
97+
problem.ruleId
98+
)}">${details}</failure>\n`
99+
);
100+
}
101+
102+
logger.output('</testcase>\n');
103+
}
104+
105+
logger.output('</testsuite>\n');
106+
logger.output('</testsuites>\n\n');
107+
}

‎packages/cli/src/commands/scorecard-classic/index.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import type { CommandArgs } from '../../wrapper.js';
1212
import { handleLoginAndFetchToken } from './auth/login-handler.js';
1313
import { printScorecardResultsAsCheckstyle } from './formatters/checkstyle-formatter.js';
1414
import { printScorecardResultsAsJson } from './formatters/json-formatter.js';
15+
import { printScorecardResultsAsJunit } from './formatters/junit-formatter.js';
1516
import { printScorecardResults } from './formatters/stylish-formatter.js';
1617
import { fetchRemoteScorecardAndPlugins } from './remote/fetch-scorecard.js';
1718
import { getTarget } from './targets-handler/targets-handler.js';
@@ -156,6 +157,8 @@ export async function handleScorecardClassic({
156157
printScorecardResultsAsJson(result, achievedLevel, targetLevelAchieved, version);
157158
} else if (argv.format === 'checkstyle') {
158159
printScorecardResultsAsCheckstyle(path, result, achievedLevel, targetLevelAchieved);
160+
} else if (argv.format === 'junit') {
161+
printScorecardResultsAsJunit(path, result, achievedLevel, targetLevelAchieved);
159162
} else {
160163
printScorecardResults(result, achievedLevel, targetLevelAchieved);
161164
}

0 commit comments

Comments
 (0)