Skip to content

Commit 7efb4e3

Browse files
davidgamezdavidgamez
authored
ci: Add INFO level notices to acceptance tests (#2148)
1 parent bd118bf commit 7efb4e3

7 files changed

Lines changed: 126 additions & 3 deletions

File tree

‎output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/cli/ValidationReportComparator.java‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,16 @@ public Result compareValidationRuns(
5757
SeverityLevel.WARNING,
5858
args.getNewErrorThreshold(),
5959
args.getPercentInvalidDatasetsThreshold());
60+
ChangedNoticesCollector newInfoNotices =
61+
new ChangedNoticesCollector(
62+
SeverityLevel.INFO,
63+
args.getNewErrorThreshold(),
64+
args.getPercentInvalidDatasetsThreshold());
65+
ChangedNoticesCollector droppedInfoNotices =
66+
new ChangedNoticesCollector(
67+
SeverityLevel.INFO,
68+
args.getNewErrorThreshold(),
69+
args.getPercentInvalidDatasetsThreshold());
6070
CorruptedSourcesCollector corruptedSources =
6171
new CorruptedSourcesCollector(args.getPercentCorruptedSourcesThreshold());
6272
OutOfMemorySourcesCollector oomSources = new OutOfMemorySourcesCollector();
@@ -135,6 +145,9 @@ public Result compareValidationRuns(
135145
droppedErrors.compareValidationReports(sourceId, sourceUrl, latestReport, referenceReport);
136146
newWarnings.compareValidationReports(sourceId, sourceUrl, referenceReport, latestReport);
137147
droppedWarnings.compareValidationReports(sourceId, sourceUrl, latestReport, referenceReport);
148+
newInfoNotices.compareValidationReports(sourceId, sourceUrl, referenceReport, latestReport);
149+
droppedInfoNotices.compareValidationReports(
150+
sourceId, sourceUrl, latestReport, referenceReport);
138151
validationPerformanceCollector.compareValidationReports(
139152
sourceId, referenceReport, latestReport);
140153
}
@@ -145,6 +158,8 @@ public Result compareValidationRuns(
145158
droppedErrors.getChangedNotices(),
146159
newWarnings.getChangedNotices(),
147160
droppedWarnings.getChangedNotices(),
161+
newInfoNotices.getChangedNotices(),
162+
droppedInfoNotices.getChangedNotices(),
148163
corruptedSources.toReport(),
149164
validationPerformanceCollector.toReport());
150165

@@ -153,6 +168,8 @@ public Result compareValidationRuns(
153168
|| droppedErrors.isAboveThreshold()
154169
|| newWarnings.isAboveThreshold()
155170
|| droppedWarnings.isAboveThreshold()
171+
|| newInfoNotices.isAboveThreshold()
172+
|| droppedInfoNotices.isAboveThreshold()
156173
|| corruptedSources.isAboveThreshold();
157174

158175
String reportSummaryString =
@@ -162,6 +179,8 @@ public Result compareValidationRuns(
162179
droppedErrors,
163180
newWarnings,
164181
droppedWarnings,
182+
newInfoNotices,
183+
droppedInfoNotices,
165184
corruptedSources,
166185
oomSources,
167186
validationPerformanceCollector,
@@ -220,6 +239,8 @@ private static String generateReportSummaryString(
220239
ChangedNoticesCollector droppedErrors,
221240
ChangedNoticesCollector newWarnings,
222241
ChangedNoticesCollector droppedWarnings,
242+
ChangedNoticesCollector newInfoNotices,
243+
ChangedNoticesCollector droppedInfoNotices,
223244
CorruptedSourcesCollector corruptedSources,
224245
OutOfMemorySourcesCollector oomSources,
225246
ValidationPerformanceCollector validationPerformanceCollector,
@@ -247,7 +268,9 @@ private static String generateReportSummaryString(
247268
b.append(newErrors.generateLogString("New Errors")).append('\n');
248269
b.append(droppedErrors.generateLogString("Dropped Errors")).append('\n');
249270
b.append(newWarnings.generateLogString("New Warnings")).append('\n');
250-
b.append(droppedWarnings.generateLogString("Dropped Warnings")).append("\n\n");
271+
b.append(droppedWarnings.generateLogString("Dropped Warnings")).append('\n');
272+
b.append(newInfoNotices.generateLogString("New Info Notices")).append('\n');
273+
b.append(droppedInfoNotices.generateLogString("Dropped Info Notices")).append("\n\n");
251274
b.append(corruptedSources.generateLogString()).append("\n").append("\n");
252275
b.append(oomSources.generateLogString()).append("\n").append("\n");
253276
b.append(validationPerformanceCollector.generateLogString()).append("\n");

‎output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/model/report/AcceptanceReport.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,10 @@ public abstract class AcceptanceReport {
1515

1616
public abstract List<ChangedNotice> droppedWarnings();
1717

18+
public abstract List<ChangedNotice> newInfoNotices();
19+
20+
public abstract List<ChangedNotice> droppedInfoNotices();
21+
1822
public abstract CorruptedSources corruptedSources();
1923

2024
public abstract List<ValidationPerformance> validationPerformance();
@@ -24,13 +28,17 @@ public static AcceptanceReport create(
2428
List<ChangedNotice> droppedErrors,
2529
List<ChangedNotice> newWarnings,
2630
List<ChangedNotice> droppedWarnings,
31+
List<ChangedNotice> newInfoNotices,
32+
List<ChangedNotice> droppedInfoNotices,
2733
CorruptedSources corruptedSources,
2834
List<ValidationPerformance> validationPerformance) {
2935
return new AutoValue_AcceptanceReport(
3036
newErrors,
3137
droppedErrors,
3238
newWarnings,
3339
droppedWarnings,
40+
newInfoNotices,
41+
droppedInfoNotices,
3442
corruptedSources,
3543
validationPerformance);
3644
}

‎output-comparator/src/test/java/org/mobilitydata/gtfsvalidator/outputcomparator/cli/ValidationReportComparatorTest.java‎

Lines changed: 86 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
import org.mobilitydata.gtfsvalidator.notice.MissingRecommendedFileNotice;
1717
import org.mobilitydata.gtfsvalidator.notice.MissingRequiredFileNotice;
1818
import org.mobilitydata.gtfsvalidator.notice.NoticeContainer;
19+
import org.mobilitydata.gtfsvalidator.notice.UnknownColumnNotice;
20+
import org.mobilitydata.gtfsvalidator.notice.UnknownFileNotice;
1921
import org.mobilitydata.gtfsvalidator.notice.ValidationNotice;
2022
import org.mobilitydata.gtfsvalidator.outputcomparator.cli.ValidationReportComparator.Result;
2123
import org.mobilitydata.gtfsvalidator.outputcomparator.model.SourceUrlContainer;
@@ -48,13 +50,23 @@ public void acceptanceReport() throws IOException {
4850
ImmutableList.of(
4951
new MissingRequiredFileNotice("a.txt"), new MissingRequiredFileNotice("c.txt")),
5052
ImmutableList.of(new MissingRecommendedFileNotice("b.txt"))),
53+
// Drops an info notice (unknown_file) and adds a different info notice
54+
// (unknown_column).
55+
constructBeforeAndAfterReports(
56+
"feed-id-d",
57+
ImmutableList.of(new UnknownFileNotice("old.txt")),
58+
ImmutableList.of(new UnknownColumnNotice("f.txt", "col", 1))),
5159
// A "corrupted" source directory with no feeds.
5260
new File(reportsDir.getRoot(), "feed-id-c"));
5361

5462
// Note that we don't include feed-id-c here.
5563
SourceUrlContainer sourceUrlContainer =
5664
new SourceUrlContainer(
57-
ImmutableMap.of("feed-id-a", "url-a", "feed-id-b", "url-b", "feed-id-c", "url-c"));
65+
ImmutableMap.of(
66+
"feed-id-a", "url-a",
67+
"feed-id-b", "url-b",
68+
"feed-id-c", "url-c",
69+
"feed-id-d", "url-d"));
5870

5971
ValidationReportComparator comparator = new ValidationReportComparator();
6072
Result result = comparator.compareValidationRuns(args, reportDirs, sourceUrlContainer);
@@ -77,9 +89,17 @@ public void acceptanceReport() throws IOException {
7789
.containsExactly(
7890
new ChangedNotice("missing_recommended_file")
7991
.addAffectedSource(AffectedSource.create("feed-id-a", "url-a", 1)));
92+
assertThat(report.newInfoNotices())
93+
.containsExactly(
94+
new ChangedNotice("unknown_column")
95+
.addAffectedSource(AffectedSource.create("feed-id-d", "url-d", 1)));
96+
assertThat(report.droppedInfoNotices())
97+
.containsExactly(
98+
new ChangedNotice("unknown_file")
99+
.addAffectedSource(AffectedSource.create("feed-id-d", "url-d", 1)));
80100

81101
CorruptedSources corruptedSources = report.corruptedSources();
82-
assertThat(corruptedSources.sourceIdCount()).isEqualTo(3);
102+
assertThat(corruptedSources.sourceIdCount()).isEqualTo(4);
83103
assertThat(corruptedSources.corruptedSourcesCount()).isEqualTo(1);
84104
assertThat(corruptedSources.corruptedSources()).containsExactly("feed-id-c");
85105
}
@@ -131,6 +151,14 @@ public void addedErrorNotice_summaryString() throws Exception {
131151
+ "<summary><strong>Dropped Warnings</strong> (0 out of 2 datasets, ~0%) ✅</summary>\n"
132152
+ "<p>No changes were detected due to the code change.</p>\n"
133153
+ "</details>\n"
154+
+ "<details>\n"
155+
+ "<summary><strong>New Info Notices</strong> (0 out of 2 datasets, ~0%) ✅</summary>\n"
156+
+ "<p>No changes were detected due to the code change.</p>\n"
157+
+ "</details>\n"
158+
+ "<details>\n"
159+
+ "<summary><strong>Dropped Info Notices</strong> (0 out of 2 datasets, ~0%) ✅</summary>\n"
160+
+ "<p>No changes were detected due to the code change.</p>\n"
161+
+ "</details>\n"
134162
+ "\n"
135163
+ "### \uD83D\uDEE1\uFE0F Corruption Check\n"
136164
+ "0 out of 2 sources (~0 %) are corrupted.\n\n\n\n"
@@ -330,6 +358,62 @@ public void droppedWarningNotice_aboveThresholds_failure() throws Exception {
330358
assertThat(result.failure()).isTrue();
331359
}
332360

361+
@Test
362+
public void addedInfoNotice_aboveThresholds_failure() throws Exception {
363+
Arguments args = defaultArgs();
364+
args.setNewErrorThreshold(1);
365+
args.setPercentInvalidDatasetsThreshold(25);
366+
367+
List<File> reportDirs =
368+
ImmutableList.of(
369+
constructBeforeAndAfterReports(
370+
"feed-id-a",
371+
ImmutableList.of(new MissingRequiredFileNotice("a.txt")),
372+
ImmutableList.of(new MissingRequiredFileNotice("a.txt"))),
373+
// We have a single additional info notice in feed b.
374+
constructBeforeAndAfterReports(
375+
"feed-id-b", ImmutableList.of(), ImmutableList.of(new UnknownFileNotice("x.txt"))));
376+
377+
ValidationReportComparator comparator = new ValidationReportComparator();
378+
Result result = comparator.compareValidationRuns(args, reportDirs, constructSourceUrls());
379+
380+
// 1 additional info notice is enough to invalidate one dataset, for an invalid dataset
381+
// threshold of 50%.
382+
assertThat(result.failure()).isTrue();
383+
384+
assertThat(result.report().newInfoNotices())
385+
.containsExactly(
386+
new ChangedNotice("unknown_file")
387+
.addAffectedSource(AffectedSource.create("feed-id-b", "url", 1)));
388+
assertThat(result.report().droppedInfoNotices()).isEmpty();
389+
assertThat(result.report().newErrors()).isEmpty();
390+
assertThat(result.report().newWarnings()).isEmpty();
391+
}
392+
393+
@Test
394+
public void droppedInfoNotice_aboveThresholds_failure() throws Exception {
395+
Arguments args = defaultArgs();
396+
args.setNewErrorThreshold(1);
397+
args.setPercentInvalidDatasetsThreshold(25);
398+
399+
List<File> reportDirs =
400+
ImmutableList.of(
401+
constructBeforeAndAfterReports(
402+
"feed-id-a",
403+
ImmutableList.of(new MissingRequiredFileNotice("a.txt")),
404+
ImmutableList.of(new MissingRequiredFileNotice("a.txt"))),
405+
// We have a single dropped info notice in feed b.
406+
constructBeforeAndAfterReports(
407+
"feed-id-b", ImmutableList.of(new UnknownFileNotice("x.txt")), ImmutableList.of()));
408+
409+
ValidationReportComparator comparator = new ValidationReportComparator();
410+
Result result = comparator.compareValidationRuns(args, reportDirs, constructSourceUrls());
411+
412+
// 1 dropped info notice is enough to invalidate one dataset, for an invalid dataset
413+
// threshold of 50%.
414+
assertThat(result.failure()).isTrue();
415+
}
416+
333417
@Test
334418
public void corruptedSource_aboveThresholds_failure() throws Exception {
335419
Arguments args = defaultArgs();

‎output-comparator/src/test/resources/org/mobilitydata/gtfsvalidator/outputcomparator/cli/MainTest-droppedNoticeTypes.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@
2727
]
2828
}
2929
],
30+
"newInfoNotices": [],
31+
"droppedInfoNotices": [],
3032
"corruptedSources": {
3133
"sourceIdCount": 2,
3234
"corruptedSourcesCount": 0,

‎output-comparator/src/test/resources/org/mobilitydata/gtfsvalidator/outputcomparator/cli/MainTest-newNoticeTypes.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,8 @@
7575
}
7676
],
7777
"droppedWarnings": [],
78+
"newInfoNotices": [],
79+
"droppedInfoNotices": [],
7880
"corruptedSources": {
7981
"sourceIdCount": 3,
8082
"corruptedSourcesCount": 0,

‎output-comparator/src/test/resources/org/mobilitydata/gtfsvalidator/outputcomparator/cli/MainTest-noNewNoticeTypes.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
"droppedErrors": [],
44
"newWarnings": [],
55
"droppedWarnings": [],
6+
"newInfoNotices": [],
7+
"droppedInfoNotices": [],
68
"corruptedSources": {
79
"sourceIdCount": 1,
810
"corruptedSourcesCount": 0,

‎output-comparator/src/test/resources/org/mobilitydata/gtfsvalidator/outputcomparator/cli/MainTest-tooManyCorruptedSources.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
"droppedErrors": [],
44
"newWarnings": [],
55
"droppedWarnings": [],
6+
"newInfoNotices": [],
7+
"droppedInfoNotices": [],
68
"corruptedSources": {
79
"sourceIdCount": 5,
810
"corruptedSourcesCount": 2,

0 commit comments

Comments
 (0)