Skip to content

Commit 34ee509

Browse files
ludovicianulludovicianul
committed
feat: Enhance replay command to replay all --errros and/or --warnings from a previous run
1 parent 6507b16 commit 34ee509

3 files changed

Lines changed: 331 additions & 33 deletions

File tree

‎src/main/java/dev/dochia/cli/core/command/ReplayCommand.java‎

Lines changed: 210 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@
66
import dev.dochia.cli.core.io.ServiceCaller;
77
import dev.dochia.cli.core.model.HttpResponse;
88
import dev.dochia.cli.core.model.TestCase;
9+
import dev.dochia.cli.core.report.TestCaseExporter;
910
import dev.dochia.cli.core.report.TestCaseListener;
10-
import dev.dochia.cli.core.util.CommonUtils;
1111
import dev.dochia.cli.core.util.JsonUtils;
1212
import dev.dochia.cli.core.util.KeyValuePair;
1313
import dev.dochia.cli.core.util.VersionProvider;
@@ -20,8 +20,15 @@
2020

2121
import java.io.IOException;
2222
import java.nio.file.Files;
23+
import java.nio.file.Path;
2324
import java.nio.file.Paths;
24-
import java.util.*;
25+
import java.util.ArrayList;
26+
import java.util.Arrays;
27+
import java.util.Collections;
28+
import java.util.HashMap;
29+
import java.util.List;
30+
import java.util.Map;
31+
import java.util.Optional;
2532

2633
/**
2734
* This will replay a given list of tests solely based on the information received in the test case file(s).
@@ -41,7 +48,11 @@
4148
footer = {" Replay Test 1 from the default reporting folder:",
4249
" dochia replay Test1",
4350
"", " Replay Test 1 from the default reporting folder and write the new output in another folder",
44-
" dochia replay Test1 --output path/to/new/folder"},
51+
" dochia replay Test1 --output path/to/new/folder",
52+
"", " Retry all failed (error) tests from the default dochia-report folder:",
53+
" dochia replay --errors",
54+
"", " Retry all failed tests including warnings:",
55+
" dochia replay --errors --warnings"},
4556
versionProvider = VersionProvider.class)
4657
@Unremovable
4758
public class ReplayCommand implements Runnable {
@@ -61,7 +72,7 @@ public class ReplayCommand implements Runnable {
6172
@CommandLine.Mixin
6273
HelpFullOption helpFullOption;
6374

64-
@CommandLine.Option(names = {"-v"},
75+
@CommandLine.Option(names = {"-v", "--verbose"},
6576
description = "Prints verbose information about the execution")
6677
private boolean verbose;
6778

@@ -77,6 +88,18 @@ public class ReplayCommand implements Runnable {
7788
description = "If supplied, it will create TestXXX.json files within the given folder with the updated responses received when replaying the tests")
7889
private String outputReportFolder;
7990

91+
@CommandLine.Option(names = {"--errors"},
92+
description = "Retry all tests with error results from the dochia-summary-report.json in the report folder")
93+
private boolean errors;
94+
95+
@CommandLine.Option(names = {"--warnings"},
96+
description = "Retry all tests with warning results from the dochia-summary-report.json in the report folder")
97+
private boolean warnings;
98+
99+
@CommandLine.Option(names = {"--report-folder", "-r"},
100+
description = "The folder containing the dochia-summary-report.json file when using --errors/--warnings. Default: @|bold,underline dochia-report|@")
101+
private String reportFolder = "dochia-report";
102+
80103

81104
/**
82105
* Constructs a new instance of the {@code ReplayCommand} class.
@@ -91,13 +114,102 @@ public ReplayCommand(ServiceCaller serviceCaller, TestCaseListener testCaseListe
91114
}
92115

93116
private List<String> parseTestCases() {
94-
return Arrays.stream(tests)
95-
.map(testCase -> testCase.trim().strip())
96-
.map(testCase -> testCase.endsWith(".json") ? testCase : "dochia-report/" + testCase + ".json")
97-
.toList();
117+
List<String> testCaseFiles = new ArrayList<>();
118+
119+
// Add tests from retry options (--errors, --warnings)
120+
if (errors || warnings) {
121+
testCaseFiles.addAll(loadTestIdsFromSummaryReport());
122+
}
123+
124+
// Add explicitly provided test cases
125+
if (tests != null && tests.length > 0) {
126+
testCaseFiles.addAll(Arrays.stream(tests)
127+
.map(testCase -> testCase.trim().strip())
128+
.map(testCase -> testCase.endsWith(".json") ? testCase : reportFolder + "/" + testCase + ".json")
129+
.toList());
130+
}
131+
132+
return testCaseFiles;
133+
}
134+
135+
private List<String> loadTestIdsFromSummaryReport() {
136+
Path summaryPath = Paths.get(reportFolder, TestCaseExporter.REPORT_JS);
137+
if (!Files.exists(summaryPath)) {
138+
logger.error("Summary report not found at: {}", summaryPath);
139+
return Collections.emptyList();
140+
}
141+
142+
try {
143+
String content = Files.readString(summaryPath);
144+
SummaryReport report = JsonUtils.GSON.fromJson(content, SummaryReport.class);
145+
146+
if (report == null || report.testCases == null) {
147+
logger.error("Invalid summary report format");
148+
return Collections.emptyList();
149+
}
150+
151+
List<String> failedIds = new ArrayList<>();
152+
for (TestCaseSummaryEntry entry : report.testCases) {
153+
if (shouldRetryTest(entry)) {
154+
String testId = entry.id.replace(" ", "");
155+
failedIds.add(reportFolder + "/" + testId + ".json");
156+
}
157+
}
158+
159+
if (failedIds.isEmpty()) {
160+
logger.info("No failed tests found to retry");
161+
} else {
162+
logger.info("Found {} failed test(s) to retry", failedIds.size());
163+
}
164+
165+
return failedIds;
166+
} catch (IOException e) {
167+
logger.error("Failed to read summary report: {}", e.getMessage());
168+
logger.debug("Stacktrace:", e);
169+
return Collections.emptyList();
170+
} catch (Exception e) {
171+
logger.error("Failed to parse summary report: {}", e.getMessage());
172+
logger.debug("Stacktrace:", e);
173+
return Collections.emptyList();
174+
}
175+
}
176+
177+
private boolean shouldRetryTest(TestCaseSummaryEntry entry) {
178+
if (entry.result == null) {
179+
return false;
180+
}
181+
boolean isError = errors && "error".equalsIgnoreCase(entry.result);
182+
boolean isWarning = warnings && "warn".equalsIgnoreCase(entry.result);
183+
return isError || isWarning;
184+
}
185+
186+
/**
187+
* Internal class for deserializing the summary report.
188+
*/
189+
static class SummaryReport {
190+
List<TestCaseSummaryEntry> testCases;
191+
}
192+
193+
/**
194+
* Internal class for deserializing individual test case entries from the summary.
195+
*/
196+
static class TestCaseSummaryEntry {
197+
String id;
198+
String result;
199+
}
200+
201+
/**
202+
* Tracks replay statistics for summary display.
203+
*/
204+
static class ReplayStats {
205+
int initialErrors;
206+
int initialWarnings;
207+
int unchanged;
208+
int improved;
209+
int regressed;
98210
}
99211

100-
private void executeTestCase(String testCaseFileName) throws IOException {
212+
private void executeTestCase(String testCaseFileName, ReplayStats stats) throws IOException {
101213
TestCase testCase = this.loadTestCaseFile(testCaseFileName);
102214
logger.start("Calling service endpoint: {}", testCase.getRequest().getUrl());
103215
this.loadHeadersIfSupplied(testCase);
@@ -114,20 +226,46 @@ private void executeTestCase(String testCaseFileName) throws IOException {
114226
.build();
115227
}
116228

117-
logger.complete("Response body: \n{}", response.getBody());
229+
if (verbose) {
230+
logger.complete("Response body: \n{}", response.getBody());
231+
}
118232
this.writeTestJsonsIfSupplied(testCase, response);
119233
this.showResponseCodesDifferences(testCase, response);
234+
this.updateStats(testCase, response, stats);
235+
}
236+
237+
private void updateStats(TestCase testCase, HttpResponse response, ReplayStats stats) {
238+
if (stats == null) {
239+
return;
240+
}
241+
int oldCode = testCase.getResponse().getResponseCode();
242+
int newCode = response.getResponseCode();
243+
244+
boolean wasError = oldCode >= 500 || oldCode == 0;
245+
boolean isNowError = newCode >= 500 || newCode == 0;
246+
boolean isNowClientError = newCode >= 400 && newCode < 500;
247+
boolean isNowSuccess = newCode >= 200 && newCode < 300;
248+
249+
if (oldCode == newCode) {
250+
stats.unchanged++;
251+
} else if (isNowSuccess || (wasError && isNowClientError)) {
252+
stats.improved++;
253+
} else if (isNowError) {
254+
stats.regressed++;
255+
}
120256
}
121257

122258
void showResponseCodesDifferences(TestCase testCase, HttpResponse response) {
123259
logger.noFormat("");
124260
logger.star("Old response code: {}", testCase.getResponse().getResponseCode());
125261
logger.star("New response code: {}", response.getResponseCode());
126-
127-
logger.noFormat("");
128-
logger.star("Old response body: {}", testCase.getResponse().getJsonBody());
129-
logger.star("New response body: {}", response.getJsonBody());
130262
logger.noFormat("");
263+
264+
if (verbose) {
265+
logger.star("Old response body: {}", testCase.getResponse().getJsonBody());
266+
logger.star("New response body: {}", response.getJsonBody());
267+
logger.noFormat("");
268+
}
131269
}
132270

133271
void writeTestJsonsIfSupplied(TestCase testCase, HttpResponse response) {
@@ -142,19 +280,17 @@ void writeTestJsonsIfSupplied(TestCase testCase, HttpResponse response) {
142280
private void loadHeadersIfSupplied(TestCase testCase) {
143281
List<KeyValuePair<String, Object>> headersFromFile = new java.util.ArrayList<>(Optional.ofNullable(testCase.getRequest().getHeaders()).orElse(Collections.emptyList()));
144282

145-
//remove old headers
146283
headersFromFile.removeIf(header -> headersMap.containsKey(header.getKey()));
147-
148-
//add new headers
149284
headersFromFile.addAll(headersMap.entrySet().stream().map(entry -> new KeyValuePair<>(entry.getKey(), entry.getValue())).toList());
150285

151-
//see if any header is dynamic and it needs a parser
152286
headersFromFile.forEach(header -> header.setValue(DSLParser.parseAndGetResult(header.getValue().toString(), authArgs.getAuthScriptAsMap())));
153287
}
154288

155289
private TestCase loadTestCaseFile(String testCaseFileName) throws IOException {
156290
String testCaseFile = Files.readString(Paths.get(testCaseFileName));
157-
logger.config("Loaded content: \n" + testCaseFile);
291+
if (verbose) {
292+
logger.config("Loaded content: \n" + testCaseFile);
293+
}
158294
TestCase testCase = JsonUtils.GSON.fromJson(testCaseFile, TestCase.class);
159295
testCase.updateServer(server);
160296
return testCase;
@@ -177,21 +313,70 @@ private void initReportingPath() {
177313

178314
@Override
179315
public void run() {
180-
if (verbose) {
181-
CommonUtils.setDochiaLogLevel("ALL");
182-
logger.fav("Setting dochia log level to ALL!");
316+
List<String> testCases = this.parseTestCases();
317+
if (testCases.isEmpty()) {
318+
logger.warning("No tests to replay. Provide test names as arguments or use --errors/--warnings");
319+
return;
183320
}
321+
184322
this.initReportingPath();
185-
for (String testCaseFileName : this.parseTestCases()) {
323+
ReplayStats stats = (errors || warnings) ? createInitialStats() : null;
324+
325+
for (String testCaseFileName : testCases) {
186326
try {
327+
logger.noFormat("");
187328
logger.start("Executing {}", testCaseFileName);
188-
this.executeTestCase(testCaseFileName);
329+
this.executeTestCase(testCaseFileName, stats);
189330
logger.complete("Finish executing {}", testCaseFileName);
190331
} catch (IOException e) {
191332
logger.debug("Exception while replaying test!", e);
192333
logger.error("Something went wrong while replaying {}. If the test name ends with .json it is searched as a full path. " +
193-
"If it doesn't have an extension it will be searched in dochia-report/ folder. Error message: {}", testCaseFileName, e.toString());
334+
"If it doesn't have an extension it will be searched in the {} folder. Error message: {}", testCaseFileName, reportFolder, e.toString());
335+
}
336+
}
337+
338+
if (stats != null) {
339+
printSummary(stats, testCases.size());
340+
}
341+
}
342+
343+
private ReplayStats createInitialStats() {
344+
ReplayStats stats = new ReplayStats();
345+
Path summaryPath = Paths.get(reportFolder, TestCaseExporter.REPORT_JS);
346+
try {
347+
String content = Files.readString(summaryPath);
348+
SummaryReport report = JsonUtils.GSON.fromJson(content, SummaryReport.class);
349+
if (report != null && report.testCases != null) {
350+
for (TestCaseSummaryEntry entry : report.testCases) {
351+
if ("error".equalsIgnoreCase(entry.result)) {
352+
stats.initialErrors++;
353+
} else if ("warn".equalsIgnoreCase(entry.result)) {
354+
stats.initialWarnings++;
355+
}
356+
}
194357
}
358+
} catch (Exception e) {
359+
logger.debug("Could not read initial stats: {}", e.getMessage());
360+
}
361+
return stats;
362+
}
363+
364+
private void printSummary(ReplayStats stats, int totalReplayed) {
365+
logger.noFormat("");
366+
logger.noFormat("─".repeat(60));
367+
logger.info("Replay Summary");
368+
logger.noFormat("─".repeat(60));
369+
logger.star("Total tests replayed: {}", totalReplayed);
370+
logger.star("Initial errors in report: {}", stats.initialErrors);
371+
logger.star("Initial warnings in report: {}", stats.initialWarnings);
372+
logger.noFormat("");
373+
logger.star("Unchanged (same response code): {}", stats.unchanged);
374+
logger.complete("Improved (better response): {}", stats.improved);
375+
if (stats.regressed > 0) {
376+
logger.error("Regressed (worse response): {}", stats.regressed);
377+
} else {
378+
logger.star("Regressed (worse response): {}", stats.regressed);
195379
}
380+
logger.noFormat("─".repeat(60));
196381
}
197382
}

‎src/main/java/dev/dochia/cli/core/report/TestCaseExporter.java‎

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,13 @@
88
import com.google.gson.Strictness;
99
import dev.dochia.cli.core.args.ReportingArguments;
1010
import dev.dochia.cli.core.context.GlobalContext;
11-
import dev.dochia.cli.core.model.*;
11+
import dev.dochia.cli.core.model.DochiaConfiguration;
12+
import dev.dochia.cli.core.model.TestCase;
13+
import dev.dochia.cli.core.model.TestCaseExecutionSummary;
14+
import dev.dochia.cli.core.model.TestCaseSummary;
15+
import dev.dochia.cli.core.model.TestReport;
16+
import dev.dochia.cli.core.model.TimeExecution;
17+
import dev.dochia.cli.core.model.TimeExecutionDetails;
1218
import dev.dochia.cli.core.model.ann.ExcludeTestCaseStrategy;
1319
import dev.dochia.cli.core.playbook.api.DryRun;
1420
import dev.dochia.cli.core.util.ConsoleUtils;
@@ -24,17 +30,30 @@
2430
import org.eclipse.microprofile.config.inject.ConfigProperty;
2531
import org.fusesource.jansi.Ansi;
2632

27-
import java.io.*;
33+
import java.io.File;
34+
import java.io.IOException;
35+
import java.io.InputStream;
36+
import java.io.StringWriter;
37+
import java.io.Writer;
2838
import java.nio.charset.StandardCharsets;
29-
import java.nio.file.*;
39+
import java.nio.file.Files;
40+
import java.nio.file.Path;
41+
import java.nio.file.Paths;
42+
import java.nio.file.StandardCopyOption;
43+
import java.nio.file.StandardOpenOption;
3044
import java.text.DecimalFormat;
3145
import java.text.DecimalFormatSymbols;
3246
import java.text.NumberFormat;
3347
import java.time.Duration;
3448
import java.time.OffsetDateTime;
3549
import java.time.ZoneId;
3650
import java.time.format.DateTimeFormatter;
37-
import java.util.*;
51+
import java.util.Comparator;
52+
import java.util.HashMap;
53+
import java.util.List;
54+
import java.util.Locale;
55+
import java.util.Map;
56+
import java.util.Objects;
3857
import java.util.stream.Collectors;
3958
import java.util.zip.ZipEntry;
4059
import java.util.zip.ZipInputStream;
@@ -50,6 +69,7 @@ public abstract class TestCaseExporter {
5069
static final String REPORT_HTML = "index.html";
5170
static final MustacheFactory mustacheFactory = new DefaultMustacheFactory();
5271
static final Mustache SUMMARY_MUSTACHE = mustacheFactory.compile("summary.mustache");
72+
public static final String REPORT_JS = "dochia-summary-report.json";
5373
private static final String HTML = ".html";
5474
private static final String JSON = ".json";
5575
private static final Mustache TEST_CASE_MUSTACHE = mustacheFactory.compile("test-case.mustache");
@@ -308,10 +328,10 @@ public void writeSummary(List<TestCaseSummary> summaries, ExecutionStatisticsLis
308328

309329
try {
310330
writer.flush();
311-
String content = writer.toString();
312-
byte[] bytes = content.getBytes(StandardCharsets.UTF_8);
313331
Files.write(Paths.get(reportingPath.toFile().getAbsolutePath(), this.getSummaryReportTitle()),
314-
bytes, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
332+
writer.toString().getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
333+
Files.write(Paths.get(reportingPath.toFile().getAbsolutePath(), REPORT_JS), maskingSerializer.toJson(report).getBytes(StandardCharsets.UTF_8));
334+
315335
} catch (IOException e) {
316336
logger.error("There was an error writing the report summary: {}. Please check if dochia has proper right to write in the report location: {}",
317337
e.getMessage(), reportingPath.toFile().getAbsolutePath());

0 commit comments

Comments
 (0)