Support Multiple Baselines (#46)

* report decimal information to user

* Support multiple baselines and verification result is able to categorize findings

* add origin information

* CRC guard uses correct file exists reassurance

* support multiple baselines, even for creating baselines

* parallel runs are not supported for baselines

Author: AnakinRaW

README.md

@@ -116,6 +116,22 @@ Uses manual mod setup, including sub-mods and the EaW fallback game, and uses th
   --useDefaultBaseline
 ```
 
+#### Example 3: Layering a mod-specific baseline on top of the default baseline
+A typical mod-dev workflow: filter the base game's known findings with the embedded default baseline, *and* filter your mod's own accepted findings with your own baseline. The two baselines stay independent — you can regenerate your mod baseline without touching the default.
+
+**Windows:**
+```bat
+.\ModVerify.exe verify --path "C:\My Games\FoC\Mods\MyMod" --useDefaultBaseline --baseline ./myModBaseline.json
+```
+
+**Linux:**
+```bash
+./ModVerify verify \
+  --path "/home/user/games/FoC/Mods/MyMod" \
+  --useDefaultBaseline \
+  --baseline ./myModBaseline.json
+```
+
 ---
 
 ## Available Checks
@@ -156,3 +172,22 @@ ModVerify.exe createBaseline --outFile myBaseline.json --path "C:\My Games\FoC\M
   --outFile myBaseline.json \
   --path "C:\My Games\FoC\Mods\MyMod"
 ```
+
+### Creating a mod baseline on top of a base baseline
+
+If you maintain a mod and only want your baseline to contain findings your mod is responsible for, supply the base baselines you want to subtract out. Findings already covered by the base baselines are excluded from the new file. The base baselines themselves are not modified, so they can keep being maintained independently.
+
+**Windows**
+```bat
+.\ModVerify.exe createBaseline --outFile myModBaseline.json --path "C:\My Games\FoC\Mods\MyMod" --useDefaultBaseline
+```
+
+**Linux**
+```bash
+./ModVerify createBaseline \
+  --outFile myModBaseline.json \
+  --path "/home/user/games/FoC/Mods/MyMod" \
+  --useDefaultBaseline
+```
+
+You can also chain a custom base baseline via `--baseline <path>`, and combine it with `--useDefaultBaseline`.

src/ModVerify.CliApp/App/CreateBaselineAction.cs

@@ -1,8 +1,7 @@
-using AET.ModVerify.App.Reporting;
+using AET.ModVerify.App.Reporting;
 using AET.ModVerify.App.Settings;
 using AET.ModVerify.App.Utilities;
 using AET.ModVerify.Reporting;
-using AET.ModVerify.Reporting.Baseline;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using System;
@@ -15,7 +14,7 @@ internal sealed class CreateBaselineAction(AppBaselineSettings settings, IServic
     : ModVerifyApplicationAction<AppBaselineSettings>(settings, serviceProvider)
 {
     private readonly IFileSystem _fileSystem = serviceProvider.GetRequiredService<IFileSystem>();
-    
+
     protected override void PrintAction(VerificationTarget target)
     {
         Console.WriteLine();
@@ -29,11 +28,11 @@ protected override void PrintAction(VerificationTarget target)
     protected override async Task<int> ProcessResult(VerificationResult result)
     {
         var baselineFactory = ServiceProvider.GetRequiredService<IBaselineFactory>();
-        var baseline = baselineFactory.CreateBaseline(result.Target, Settings, result.Errors);
+        var baseline = baselineFactory.CreateBaseline(result.Target, Settings, result.Errors.NewErrors);
 
         var fullPath = _fileSystem.Path.GetFullPath(Settings.NewBaselinePath);
-        Logger?.LogInformation(ModVerifyConstants.ConsoleEventId, 
-            "Writing Baseline to '{FullPath}' with {Number} findings", fullPath, result.Errors.Count);
+        Logger?.LogInformation(ModVerifyConstants.ConsoleEventId,
+            "Writing Baseline to '{FullPath}' with {Number} findings", fullPath, result.Errors.NewErrors.Count);
 
         await baselineFactory.WriteBaselineAsync(baseline, Settings.NewBaselinePath);
 
@@ -46,9 +45,4 @@ protected override async Task<int> ProcessResult(VerificationResult result)
 
         return ModVerifyConstants.Success;
     }
-
-    protected override VerificationBaseline GetBaseline(VerificationTarget verificationTarget)
-    {
-        return VerificationBaseline.Empty;
-    }
 }

src/ModVerify.CliApp/App/ModVerifyApplicationAction.cs

@@ -6,6 +6,7 @@
 using AET.ModVerify.App.Reporting;
 using AET.ModVerify.App.Settings;
 using AET.ModVerify.App.TargetSelectors;
+using AET.ModVerify.App.Utilities;
 using AET.ModVerify.Reporting;
 using AET.ModVerify.Reporting.Baseline;
 using AET.ModVerify.Reporting.Suppressions;
@@ -105,14 +106,30 @@ private int ReportVerificationFailure(Exception verificationException)
     }
 
     protected abstract Task<int> ProcessResult(VerificationResult result);
-    
-    protected abstract VerificationBaseline GetBaseline(VerificationTarget verificationTarget);
+
+    protected virtual BaselineCollection GetBaselines(VerificationTarget verificationTarget)
+    {
+        var baselineSelector = new BaselineSelector(Settings, ServiceProvider);
+        var baselines = baselineSelector.SelectBaselines(verificationTarget);
+        if (!baselines.IsEmpty)
+        {
+            Console.WriteLine();
+            ModVerifyConsoleUtilities.WriteBaselineInfo(baselines);
+            foreach (var entry in baselines)
+            {
+                Logger?.LogDebug("Using baseline {Baseline} from source '{Identifier}'",
+                    entry.Baseline.ToString(), entry.Identifier);
+            }
+            Console.WriteLine();
+        }
+        return baselines;
+    }
 
     private async Task<VerificationResult> VerifyTargetAsync(VerificationTarget verificationTarget)
     {
         var progressReporter = new VerifyConsoleProgressReporter(verificationTarget.Name, Settings.ReportSettings);
 
-        var baseline = GetBaseline(verificationTarget);
+        var baselines = GetBaselines(verificationTarget);
         var suppressions = GetSuppressions();
 
         try
@@ -122,11 +139,11 @@ private async Task<VerificationResult> VerifyTargetAsync(VerificationTarget veri
             Logger?.LogInformation(ModVerifyConstants.ConsoleEventId, "Verifying '{Target}'...", verificationTarget.Name);
 
             var verificationResult = await verifierService.VerifyAsync(
-                verificationTarget, 
+                verificationTarget,
                 Settings.VerifierServiceSettings,
-                baseline, 
+                baselines,
                 suppressions,
-                progressReporter, 
+                progressReporter,
                 new EngineInitializeProgressReporter(verificationTarget.Engine));
 
             progressReporter.Report(string.Empty, 1.0);

src/ModVerify.CliApp/App/VerifyAction.cs

@@ -1,5 +1,4 @@
-using AET.ModVerify.App.Reporting;
-using AET.ModVerify.App.Settings;
+using AET.ModVerify.App.Settings;
 using AET.ModVerify.App.Utilities;
 using AET.ModVerify.Reporting;
 using Microsoft.Extensions.Logging;
@@ -8,7 +7,6 @@
 using System.Linq;
 using System.Threading.Tasks;
 using AET.ModVerify.Reporting.Reporters;
-using AET.ModVerify.Reporting.Baseline;
 
 namespace AET.ModVerify.App;
 
@@ -41,7 +39,7 @@ protected override async Task<int> ProcessResult(VerificationResult result)
         await reportBroker.ReportAsync(result);
 
         if (Settings.AppFailsOnMinimumSeverity.HasValue &&
-            result.Errors.Any(x => x.Severity >= Settings.AppFailsOnMinimumSeverity))
+            result.Errors.NewErrors.Any(x => x.Severity >= Settings.AppFailsOnMinimumSeverity))
         {
             Logger?.LogInformation(ModVerifyConstants.ConsoleEventId,
                 "The verification of {Target} completed with findings of the specified failure severity {Severity}",
@@ -53,32 +51,18 @@ protected override async Task<int> ProcessResult(VerificationResult result)
         return ModVerifyConstants.Success;
     }
 
-    protected override VerificationBaseline GetBaseline(VerificationTarget verificationTarget)
-    {
-        var baselineSelector = new BaselineSelector(Settings, ServiceProvider);
-        var baseline = baselineSelector.SelectBaseline(verificationTarget, out var baselinePath);
-        if (!baseline.IsEmpty)
-        {
-            Console.WriteLine();
-            ModVerifyConsoleUtilities.WriteBaselineInfo(baseline, baselinePath);
-            Logger?.LogDebug("Using baseline {Baseline} from location '{Path}'", 
-                baseline.ToString(), baselinePath ?? "Embedded");
-            Console.WriteLine();
-        }
-        return baseline;
-    }
-
     private IReadOnlyCollection<IVerificationReporter> CreateReporters()
     {
-        var reporters = new List<IVerificationReporter>();
-
-        reporters.Add(IVerificationReporter.CreateConsole(new ConsoleReporterSettings
+        var reporters = new List<IVerificationReporter>
         {
-            Verbose = Settings.ReportSettings.Verbose,
-            MinimumReportSeverity = Settings.VerifierServiceSettings.FailFastSettings.IsFailFast
-                ? VerificationSeverity.Information
-                : VerificationSeverity.Error
-        }, ServiceProvider));
+            IVerificationReporter.CreateConsole(new ConsoleReporterSettings
+            {
+                Verbose = Settings.ReportSettings.Verbose,
+                MinimumReportSeverity = Settings.VerifierServiceSettings.FailFastSettings.IsFailFast
+                    ? VerificationSeverity.Information
+                    : VerificationSeverity.Error
+            }, ServiceProvider)
+        };
 
         var outputDirectory = Settings.ReportDirectory;
         reporters.Add(IVerificationReporter.CreateJson(new JsonReporterSettings

src/ModVerify.CliApp/Properties/launchSettings.json

@@ -6,7 +6,7 @@
     },
     "Verify (Interactive)": {
       "commandName": "Project",
-      "commandLineArgs": "verify  -o verifyResults --offline --minFailSeverity Information --searchBaseline"
+      "commandLineArgs": "verify  -o verifyResults --offline --minFailSeverity Information --useDefaultBaseline"
     },
     "Verify (Automatic Target Selection)": {
       "commandName": "Project",