From 49d23a911d041ff87c33e9b2f168b97b9eaa11cf Mon Sep 17 00:00:00 2001 From: Brennan Beam <64556723+Brennan1994@users.noreply.github.com> Date: Tue, 23 Dec 2025 09:10:35 -0800 Subject: [PATCH 01/13] testing utility init --- .claude/settings.local.json | 15 + Fda.sln | 13 + HEC.FDA.Model/HEC.FDA.Model.csproj | 4 + .../Comparison/ComparisonResult.cs | 36 ++ .../Comparison/StudyBaselineWriter.cs | 51 +++ .../Comparison/XmlResultComparer.cs | 208 +++++++++++ .../Configuration/TestConfiguration.cs | 71 ++++ .../HEC.FDA.TestingUtility.csproj | 22 ++ HEC.FDA.TestingUtility/Program.cs | 84 +++++ .../Services/AlternativeRunner.cs | 82 +++++ .../Services/ScenarioRunner.cs | 68 ++++ .../Services/StageDamageRunner.cs | 37 ++ .../Services/StudyLoader.cs | 160 +++++++++ HEC.FDA.TestingUtility/TestRunner.cs | 328 ++++++++++++++++++ HEC.FDA.ViewModel/HEC.FDA.ViewModel.csproj | 4 + 15 files changed, 1183 insertions(+) create mode 100644 .claude/settings.local.json create mode 100644 HEC.FDA.TestingUtility/Comparison/ComparisonResult.cs create mode 100644 HEC.FDA.TestingUtility/Comparison/StudyBaselineWriter.cs create mode 100644 HEC.FDA.TestingUtility/Comparison/XmlResultComparer.cs create mode 100644 HEC.FDA.TestingUtility/Configuration/TestConfiguration.cs create mode 100644 HEC.FDA.TestingUtility/HEC.FDA.TestingUtility.csproj create mode 100644 HEC.FDA.TestingUtility/Program.cs create mode 100644 HEC.FDA.TestingUtility/Services/AlternativeRunner.cs create mode 100644 HEC.FDA.TestingUtility/Services/ScenarioRunner.cs create mode 100644 HEC.FDA.TestingUtility/Services/StageDamageRunner.cs create mode 100644 HEC.FDA.TestingUtility/Services/StudyLoader.cs create mode 100644 HEC.FDA.TestingUtility/TestRunner.cs diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 000000000..64801ed5d --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,15 @@ +{ + "permissions": { + "allow": [ + "Bash(dotnet test:*)", + "Bash(cat:*)", + "Bash(dotnet sln:*)", + "Bash(dotnet build:*)", + "Bash(dir \"C:\\\\Users\\\\HEC\\\\Data\\\\FDA\\\\Canon\" /b)", + "Bash(nuget sources:*)", + "Bash(dotnet restore:*)", + "Bash(./HEC.FDA.TestingUtility.exe:*)" + ], + "deny": [] + } +} diff --git a/Fda.sln b/Fda.sln index cd22101d3..418a49aa2 100644 --- a/Fda.sln +++ b/Fda.sln @@ -62,6 +62,7 @@ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VisualScratchSpace", "VisualScratchSpace\VisualScratchSpace.csproj", "{90861E11-37B0-49F1-AB98-67D42F39ED1B}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HEC.FDA.Benchmarks", "HEC.FDA.Benchmarks\HEC.FDA.Benchmarks.csproj", "{CFD13894-A4AA-4FBC-8A55-9D62CEAD1893}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HEC.FDA.TestingUtility", "HEC.FDA.TestingUtility\HEC.FDA.TestingUtility.csproj", "{38BE2FDF-9E55-4C96-8759-1A98A67B1789}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -313,6 +314,18 @@ Global {CFD13894-A4AA-4FBC-8A55-9D62CEAD1893}.Release|x64.Build.0 = Release|Any CPU {CFD13894-A4AA-4FBC-8A55-9D62CEAD1893}.Release|x86.ActiveCfg = Release|Any CPU {CFD13894-A4AA-4FBC-8A55-9D62CEAD1893}.Release|x86.Build.0 = Release|Any CPU + {38BE2FDF-9E55-4C96-8759-1A98A67B1789}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {38BE2FDF-9E55-4C96-8759-1A98A67B1789}.Debug|Any CPU.Build.0 = Debug|Any CPU + {38BE2FDF-9E55-4C96-8759-1A98A67B1789}.Debug|x64.ActiveCfg = Debug|Any CPU + {38BE2FDF-9E55-4C96-8759-1A98A67B1789}.Debug|x64.Build.0 = Debug|Any CPU + {38BE2FDF-9E55-4C96-8759-1A98A67B1789}.Debug|x86.ActiveCfg = Debug|Any CPU + {38BE2FDF-9E55-4C96-8759-1A98A67B1789}.Debug|x86.Build.0 = Debug|Any CPU + {38BE2FDF-9E55-4C96-8759-1A98A67B1789}.Release|Any CPU.ActiveCfg = Release|Any CPU + {38BE2FDF-9E55-4C96-8759-1A98A67B1789}.Release|Any CPU.Build.0 = Release|Any CPU + {38BE2FDF-9E55-4C96-8759-1A98A67B1789}.Release|x64.ActiveCfg = Release|Any CPU + {38BE2FDF-9E55-4C96-8759-1A98A67B1789}.Release|x64.Build.0 = Release|Any CPU + {38BE2FDF-9E55-4C96-8759-1A98A67B1789}.Release|x86.ActiveCfg = Release|Any CPU + {38BE2FDF-9E55-4C96-8759-1A98A67B1789}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/HEC.FDA.Model/HEC.FDA.Model.csproj b/HEC.FDA.Model/HEC.FDA.Model.csproj index 6cead47a5..5f8e01863 100644 --- a/HEC.FDA.Model/HEC.FDA.Model.csproj +++ b/HEC.FDA.Model/HEC.FDA.Model.csproj @@ -2,6 +2,10 @@ + + + + diff --git a/HEC.FDA.TestingUtility/Comparison/ComparisonResult.cs b/HEC.FDA.TestingUtility/Comparison/ComparisonResult.cs new file mode 100644 index 000000000..fe242d766 --- /dev/null +++ b/HEC.FDA.TestingUtility/Comparison/ComparisonResult.cs @@ -0,0 +1,36 @@ +namespace HEC.FDA.TestingUtility.Comparison; + +public class ComparisonResult +{ + public string ElementName { get; set; } = string.Empty; + public string ElementType { get; set; } = string.Empty; + public bool Passed { get; set; } = true; + public string? ErrorMessage { get; set; } + public List Differences { get; } = new(); + + public string Summary => Passed + ? "All values match" + : ErrorMessage ?? $"{Differences.Count} difference(s) found"; +} + +public class Difference +{ + public string Metric { get; set; } = string.Empty; + public double? Expected { get; set; } + public double? Actual { get; set; } + public double? AbsoluteDifference => Expected.HasValue && Actual.HasValue + ? Math.Abs(Expected.Value - Actual.Value) + : null; + public double? PercentDifference => Expected.HasValue && Actual.HasValue && Expected.Value != 0 + ? Math.Abs((Expected.Value - Actual.Value) / Expected.Value) * 100 + : null; + + public override string ToString() + { + if (Expected.HasValue && Actual.HasValue) + { + return $"{Metric}: Expected={Expected:F4}, Actual={Actual:F4}, Diff={AbsoluteDifference:F4} ({PercentDifference:F2}%)"; + } + return $"{Metric}: Expected={Expected}, Actual={Actual}"; + } +} diff --git a/HEC.FDA.TestingUtility/Comparison/StudyBaselineWriter.cs b/HEC.FDA.TestingUtility/Comparison/StudyBaselineWriter.cs new file mode 100644 index 000000000..832db0caf --- /dev/null +++ b/HEC.FDA.TestingUtility/Comparison/StudyBaselineWriter.cs @@ -0,0 +1,51 @@ +using System.Xml.Linq; +using HEC.FDA.Model.metrics; +using HEC.FDA.ViewModel.AggregatedStageDamage; + +namespace HEC.FDA.TestingUtility.Comparison; + +public class StudyBaselineWriter +{ + public XElement CreateStudyBaseline(string studyId, string studyName) + { + return new XElement("StudyBaseline", + new XAttribute("studyId", studyId), + new XAttribute("studyName", studyName), + new XAttribute("createdDate", DateTime.Now.ToString("yyyy-MM-dd"))); + } + + public void AddScenarioResults(XElement baseline, string name, ScenarioResults results) + { + var wrapper = new XElement("ScenarioResults", + new XAttribute("name", name), + results.WriteToXML()); + baseline.Add(wrapper); + } + + public void AddAlternativeResults(XElement baseline, string name, AlternativeResults results) + { + var wrapper = new XElement("AlternativeResults", + new XAttribute("name", name), + new XElement("BaseYearResults", results.BaseYearScenarioResults.WriteToXML()), + new XElement("FutureYearResults", results.FutureYearScenarioResults.WriteToXML())); + baseline.Add(wrapper); + } + + public void AddStageDamage(XElement baseline, string name, AggregatedStageDamageElement element) + { + var wrapper = new XElement("StageDamage", + new XAttribute("name", name), + element.ToXML()); + baseline.Add(wrapper); + } + + public void Save(XElement baseline, string path) + { + string? directory = Path.GetDirectoryName(path); + if (!string.IsNullOrEmpty(directory) && !Directory.Exists(directory)) + { + Directory.CreateDirectory(directory); + } + baseline.Save(path); + } +} diff --git a/HEC.FDA.TestingUtility/Comparison/XmlResultComparer.cs b/HEC.FDA.TestingUtility/Comparison/XmlResultComparer.cs new file mode 100644 index 000000000..6e57d6254 --- /dev/null +++ b/HEC.FDA.TestingUtility/Comparison/XmlResultComparer.cs @@ -0,0 +1,208 @@ +using System.Xml.Linq; +using HEC.FDA.Model.metrics; +using HEC.FDA.ViewModel.AggregatedStageDamage; + +namespace HEC.FDA.TestingUtility.Comparison; + +public class XmlResultComparer +{ + private XElement? _baselineDoc; + + public void LoadBaseline(string baselinePath) + { + if (!File.Exists(baselinePath)) + { + throw new FileNotFoundException($"Baseline file not found: {baselinePath}"); + } + _baselineDoc = XElement.Load(baselinePath); + } + + public ComparisonResult CompareScenarioResults(string elementName, ScenarioResults actual) + { + var result = new ComparisonResult { ElementName = elementName, ElementType = "Scenario" }; + + if (_baselineDoc == null) + { + result.Passed = false; + result.ErrorMessage = "Baseline not loaded"; + return result; + } + + // Find the baseline element by name + var baselineElement = _baselineDoc.Elements("ScenarioResults") + .FirstOrDefault(e => e.Attribute("name")?.Value == elementName); + + if (baselineElement == null) + { + result.Passed = false; + result.ErrorMessage = $"Baseline not found for Scenario '{elementName}'"; + return result; + } + + // Get the inner ScenarioResults XML + var innerXml = baselineElement.Element("ScenarioResults"); + if (innerXml == null) + { + result.Passed = false; + result.ErrorMessage = $"Invalid baseline format for Scenario '{elementName}'"; + return result; + } + + ScenarioResults baseline = ScenarioResults.ReadFromXML(innerXml); + + // Use built-in Equals method + result.Passed = actual.Equals(baseline); + + if (!result.Passed) + { + GenerateScenarioDiff(baseline, actual, result); + } + + return result; + } + + public ComparisonResult CompareAlternativeResults(string elementName, AlternativeResults actual) + { + var result = new ComparisonResult { ElementName = elementName, ElementType = "Alternative" }; + + if (_baselineDoc == null) + { + result.Passed = false; + result.ErrorMessage = "Baseline not loaded"; + return result; + } + + var baselineElement = _baselineDoc.Elements("AlternativeResults") + .FirstOrDefault(e => e.Attribute("name")?.Value == elementName); + + if (baselineElement == null) + { + result.Passed = false; + result.ErrorMessage = $"Baseline not found for Alternative '{elementName}'"; + return result; + } + + result.Passed = true; + + // Compare base year scenario results + var baseYearXml = baselineElement.Element("BaseYearResults")?.Element("ScenarioResults"); + if (baseYearXml != null) + { + var baselineBaseYear = ScenarioResults.ReadFromXML(baseYearXml); + bool baseYearMatch = actual.BaseYearScenarioResults.Equals(baselineBaseYear); + result.Passed &= baseYearMatch; + + if (!baseYearMatch) + { + result.Differences.Add(new Difference + { + Metric = "BaseYearResults", + Expected = null, + Actual = null + }); + } + } + + // Compare future year scenario results + var futureYearXml = baselineElement.Element("FutureYearResults")?.Element("ScenarioResults"); + if (futureYearXml != null) + { + var baselineFutureYear = ScenarioResults.ReadFromXML(futureYearXml); + bool futureYearMatch = actual.FutureYearScenarioResults.Equals(baselineFutureYear); + result.Passed &= futureYearMatch; + + if (!futureYearMatch) + { + result.Differences.Add(new Difference + { + Metric = "FutureYearResults", + Expected = null, + Actual = null + }); + } + } + + return result; + } + + public ComparisonResult CompareStageDamage(string elementName, AggregatedStageDamageElement actual) + { + var result = new ComparisonResult { ElementName = elementName, ElementType = "StageDamage" }; + + if (_baselineDoc == null) + { + result.Passed = false; + result.ErrorMessage = "Baseline not loaded"; + return result; + } + + var baselineElement = _baselineDoc.Elements("StageDamage") + .FirstOrDefault(e => e.Attribute("name")?.Value == elementName); + + if (baselineElement == null) + { + result.Passed = false; + result.ErrorMessage = $"Baseline not found for StageDamage '{elementName}'"; + return result; + } + + // Use element's Equals method for comparison + var innerXml = baselineElement.Elements().FirstOrDefault(); + if (innerXml == null) + { + result.Passed = false; + result.ErrorMessage = $"Invalid baseline format for StageDamage '{elementName}'"; + return result; + } + + // Note: AggregatedStageDamageElement.Equals needs to be called on the actual element + // The baseline needs to be reconstructed from XML - this may require additional work + // For now, we'll compare the XML directly + var actualXml = actual.ToXML(); + result.Passed = XmlCompare(innerXml, actualXml); + + if (!result.Passed) + { + result.ErrorMessage = "Stage damage elements do not match"; + } + + return result; + } + + private static bool XmlCompare(XElement expected, XElement actual) + { + // Simple XML comparison - compare serialized strings + // This is a basic comparison; a more sophisticated comparison could be added + return XNode.DeepEquals(expected, actual); + } + + private static void GenerateScenarioDiff(ScenarioResults baseline, ScenarioResults actual, ComparisonResult result) + { + // Compare mean EAD for each impact area / damage category + foreach (int iaId in baseline.GetImpactAreaIDs()) + { + foreach (string damCat in baseline.GetDamageCategories()) + { + foreach (string assetCat in baseline.GetAssetCategories()) + { + double baselineMean = baseline.SampleMeanExpectedAnnualConsequences(iaId, damCat, assetCat); + double actualMean = actual.SampleMeanExpectedAnnualConsequences(iaId, damCat, assetCat); + + double tolerance = 0.01; // 1% relative tolerance for small values + double absoluteDiff = Math.Abs(baselineMean - actualMean); + double relativeDiff = baselineMean != 0 ? absoluteDiff / Math.Abs(baselineMean) : absoluteDiff; + + if (relativeDiff > tolerance && absoluteDiff > 1.0) // At least $1 difference + { + result.Differences.Add(new Difference + { + Metric = $"MeanEAD[IA={iaId},DamCat={damCat},Asset={assetCat}]", + Expected = baselineMean, + Actual = actualMean + }); + } + } + } + } + } +} diff --git a/HEC.FDA.TestingUtility/Configuration/TestConfiguration.cs b/HEC.FDA.TestingUtility/Configuration/TestConfiguration.cs new file mode 100644 index 000000000..47a9bd040 --- /dev/null +++ b/HEC.FDA.TestingUtility/Configuration/TestConfiguration.cs @@ -0,0 +1,71 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace HEC.FDA.TestingUtility.Configuration; + +public class TestConfiguration +{ + [JsonPropertyName("testSuiteId")] + public string TestSuiteId { get; set; } = string.Empty; + + [JsonPropertyName("globalSettings")] + public GlobalSettings GlobalSettings { get; set; } = new(); + + [JsonPropertyName("studies")] + public List Studies { get; set; } = new(); + + public static TestConfiguration LoadFromFile(string path) + { + string json = File.ReadAllText(path); + return JsonSerializer.Deserialize(json) + ?? throw new InvalidOperationException($"Failed to deserialize configuration from {path}"); + } +} + +public class GlobalSettings +{ + [JsonPropertyName("localTempDirectory")] + public string LocalTempDirectory { get; set; } = Path.Combine(Path.GetTempPath(), "FDATests"); + + [JsonPropertyName("timeoutMinutes")] + public int TimeoutMinutes { get; set; } = 30; +} + +public class StudyConfiguration +{ + [JsonPropertyName("studyId")] + public string StudyId { get; set; } = string.Empty; + + [JsonPropertyName("studyName")] + public string StudyName { get; set; } = string.Empty; + + [JsonPropertyName("networkSourcePath")] + public string NetworkSourcePath { get; set; } = string.Empty; + + [JsonPropertyName("baselineDirectory")] + public string BaselineDirectory { get; set; } = string.Empty; + + [JsonPropertyName("runAllScenarios")] + public bool RunAllScenarios { get; set; } = false; + + [JsonPropertyName("runAllAlternatives")] + public bool RunAllAlternatives { get; set; } = false; + + [JsonPropertyName("runAllStageDamage")] + public bool RunAllStageDamage { get; set; } = false; + + [JsonPropertyName("computations")] + public List Computations { get; set; } = new(); +} + +public class ComputeConfiguration +{ + [JsonPropertyName("type")] + public string Type { get; set; } = string.Empty; + + [JsonPropertyName("elementName")] + public string ElementName { get; set; } = string.Empty; + + [JsonPropertyName("elementId")] + public int? ElementId { get; set; } +} diff --git a/HEC.FDA.TestingUtility/HEC.FDA.TestingUtility.csproj b/HEC.FDA.TestingUtility/HEC.FDA.TestingUtility.csproj new file mode 100644 index 000000000..c18ef01cd --- /dev/null +++ b/HEC.FDA.TestingUtility/HEC.FDA.TestingUtility.csproj @@ -0,0 +1,22 @@ + + + Exe + net9.0-windows + true + enable + enable + + + + + + + + + + + + + + + diff --git a/HEC.FDA.TestingUtility/Program.cs b/HEC.FDA.TestingUtility/Program.cs new file mode 100644 index 000000000..0fdffbd71 --- /dev/null +++ b/HEC.FDA.TestingUtility/Program.cs @@ -0,0 +1,84 @@ +using System.CommandLine; +using HEC.FDA.TestingUtility; +using HEC.FDA.TestingUtility.Configuration; + +// Define command line options +var configOption = new Option( + name: "--config", + description: "Path to JSON configuration file") +{ IsRequired = true }; +configOption.AddAlias("-c"); + +var outputOption = new Option( + name: "--output", + description: "Output directory for results", + getDefaultValue: () => new DirectoryInfo(Environment.CurrentDirectory)); +outputOption.AddAlias("-o"); + +var verboseOption = new Option( + name: "--verbose", + description: "Enable verbose output", + getDefaultValue: () => false); +verboseOption.AddAlias("-v"); + +var studyOption = new Option( + name: "--study", + description: "Filter to specific study IDs (can specify multiple)") +{ AllowMultipleArgumentsPerToken = true }; +studyOption.AddAlias("-s"); + +// Create root command +var rootCommand = new RootCommand("FDA Testing Utility - Regression Testing Tool for FDA Studies"); +rootCommand.AddOption(configOption); +rootCommand.AddOption(outputOption); +rootCommand.AddOption(verboseOption); +rootCommand.AddOption(studyOption); + +// Set handler +rootCommand.SetHandler(async (configFile, outputDir, verbose, studyFilter) => +{ + try + { + Console.WriteLine("FDA Testing Utility v1.0"); + Console.WriteLine("========================"); + Console.WriteLine(); + + // Load configuration + if (!configFile.Exists) + { + Console.WriteLine($"Error: Configuration file not found: {configFile.FullName}"); + Environment.Exit(1); + } + + Console.WriteLine($"Loading configuration: {configFile.FullName}"); + var config = TestConfiguration.LoadFromFile(configFile.FullName); + + // Ensure output directory exists + if (!outputDir.Exists) + { + outputDir.Create(); + } + + // Create and run test runner + var runner = new TestRunner( + config, + outputDir.FullName, + verbose, + studyFilter?.Length > 0 ? studyFilter : null); + + int exitCode = await runner.RunAsync(); + Environment.Exit(exitCode); + } + catch (Exception ex) + { + Console.WriteLine($"Fatal error: {ex.Message}"); + if (verbose) + { + Console.WriteLine(ex.StackTrace); + } + Environment.Exit(1); + } +}, configOption, outputOption, verboseOption, studyOption); + +// Run +return await rootCommand.InvokeAsync(args); diff --git a/HEC.FDA.TestingUtility/Services/AlternativeRunner.cs b/HEC.FDA.TestingUtility/Services/AlternativeRunner.cs new file mode 100644 index 000000000..5eb5de7a4 --- /dev/null +++ b/HEC.FDA.TestingUtility/Services/AlternativeRunner.cs @@ -0,0 +1,82 @@ +using HEC.FDA.Model.alternatives; +using HEC.FDA.Model.metrics; +using HEC.FDA.ViewModel; +using HEC.FDA.ViewModel.Alternatives; +using HEC.FDA.ViewModel.ImpactAreaScenario; +using HEC.FDA.ViewModel.Study; +using HEC.FDA.ViewModel.Utilities; + +namespace HEC.FDA.TestingUtility.Services; + +public class AlternativeRunner +{ + public AlternativeResults RunAlternative(string elementName, CancellationToken cancellationToken) + { + // Find element by name + AlternativeElement element = FindElement(elementName); + + Console.WriteLine($" Running alternative '{elementName}'..."); + + // Validate scenarios have results + FdaValidationResult validation = element.RunPreComputeValidation(); + if (!validation.IsValid) + { + throw new InvalidOperationException($"Alternative cannot compute: {validation.ErrorMessage}"); + } + + // Get study properties + StudyPropertiesElement props = BaseViewModel.StudyCache.GetStudyPropertiesElement(); + + // Get scenario results from the referenced scenarios + IASElement baseScenarioElement = element.BaseScenario.GetElement(); + IASElement futureScenarioElement = element.FutureScenario.GetElement(); + + if (baseScenarioElement.Results == null) + { + throw new InvalidOperationException($"Base scenario '{baseScenarioElement.Name}' has no computed results."); + } + + if (futureScenarioElement.Results == null) + { + throw new InvalidOperationException($"Future scenario '{futureScenarioElement.Name}' has no computed results."); + } + + ScenarioResults baseResults = baseScenarioElement.Results; + ScenarioResults futureResults = futureScenarioElement.Results; + + Console.WriteLine($" Using base scenario: {baseScenarioElement.Name} (Year: {element.BaseScenario.Year})"); + Console.WriteLine($" Using future scenario: {futureScenarioElement.Name} (Year: {element.FutureScenario.Year})"); + Console.WriteLine($" Discount rate: {props.DiscountRate}, Period of analysis: {props.PeriodOfAnalysis}"); + + // Compute + AlternativeResults results = Alternative.AnnualizationCompute( + props.DiscountRate, + props.PeriodOfAnalysis, + element.ID, + baseResults, + futureResults, + element.BaseScenario.Year, + element.FutureScenario.Year); + + Console.WriteLine($" Alternative computation complete."); + + return results; + } + + private static T FindElement(string elementName) where T : ChildElement + { + var elements = BaseViewModel.StudyCache.GetChildElementsOfType(); + + var match = elements.FirstOrDefault(e => + e.Name.Equals(elementName, StringComparison.OrdinalIgnoreCase)); + + if (match == null) + { + string availableNames = string.Join(", ", elements.Select(e => e.Name)); + throw new InvalidOperationException( + $"Element '{elementName}' of type {typeof(T).Name} not found. Available: {availableNames}"); + } + + return match; + } +} diff --git a/HEC.FDA.TestingUtility/Services/ScenarioRunner.cs b/HEC.FDA.TestingUtility/Services/ScenarioRunner.cs new file mode 100644 index 000000000..aa6c52f91 --- /dev/null +++ b/HEC.FDA.TestingUtility/Services/ScenarioRunner.cs @@ -0,0 +1,68 @@ +using HEC.FDA.Model.compute; +using HEC.FDA.Model.metrics; +using HEC.FDA.Model.scenarios; +using HEC.FDA.ViewModel; +using HEC.FDA.ViewModel.Compute; +using HEC.FDA.ViewModel.ImpactAreaScenario; +using HEC.FDA.ViewModel.Utilities; +using Statistics; + +namespace HEC.FDA.TestingUtility.Services; + +public class ScenarioRunner +{ + public ScenarioResults RunScenario(string elementName, CancellationToken cancellationToken) + { + // Find element by name + IASElement element = FindElement(elementName); + + Console.WriteLine($" Running scenario '{elementName}'..."); + + // Validate + FdaValidationResult validation = element.CanCompute(); + if (!validation.IsValid) + { + throw new InvalidOperationException($"Scenario cannot compute: {validation.ErrorMessage}"); + } + + // Create simulations (reuse existing static method) + List sims = ComputeScenarioVM.CreateSimulations(element.SpecificIASElements); + + if (sims.Count == 0) + { + throw new InvalidOperationException("No simulations could be created for this scenario."); + } + + // Build scenario + Scenario scenario = new(sims); + + // Get convergence criteria + ConvergenceCriteria cc = BaseViewModel.StudyCache.GetStudyPropertiesElement().GetStudyConvergenceCriteria(); + + Console.WriteLine($" Computing with convergence criteria: min={cc.MinIterations}, max={cc.MaxIterations}"); + + // Compute + ScenarioResults results = scenario.Compute(cc, cancellationToken, computeIsDeterministic: false); + + Console.WriteLine($" Scenario computation complete."); + + return results; + } + + private static T FindElement(string elementName) where T : ChildElement + { + var elements = BaseViewModel.StudyCache.GetChildElementsOfType(); + + var match = elements.FirstOrDefault(e => + e.Name.Equals(elementName, StringComparison.OrdinalIgnoreCase)); + + if (match == null) + { + string availableNames = string.Join(", ", elements.Select(e => e.Name)); + throw new InvalidOperationException( + $"Element '{elementName}' of type {typeof(T).Name} not found. Available: {availableNames}"); + } + + return match; + } +} diff --git a/HEC.FDA.TestingUtility/Services/StageDamageRunner.cs b/HEC.FDA.TestingUtility/Services/StageDamageRunner.cs new file mode 100644 index 000000000..8083b0196 --- /dev/null +++ b/HEC.FDA.TestingUtility/Services/StageDamageRunner.cs @@ -0,0 +1,37 @@ +using HEC.FDA.ViewModel; +using HEC.FDA.ViewModel.AggregatedStageDamage; +using HEC.FDA.ViewModel.Utilities; + +namespace HEC.FDA.TestingUtility.Services; + +public class StageDamageRunner +{ + public AggregatedStageDamageElement GetStageDamageElement(string elementName) + { + Console.WriteLine($" Retrieving stage damage element '{elementName}'..."); + + // Find element by name + AggregatedStageDamageElement element = FindElement(elementName); + + Console.WriteLine($" Found stage damage element with {element.Curves.Count} curves."); + + return element; + } + + private static T FindElement(string elementName) where T : ChildElement + { + var elements = BaseViewModel.StudyCache.GetChildElementsOfType(); + + var match = elements.FirstOrDefault(e => + e.Name.Equals(elementName, StringComparison.OrdinalIgnoreCase)); + + if (match == null) + { + string availableNames = string.Join(", ", elements.Select(e => e.Name)); + throw new InvalidOperationException( + $"Element '{elementName}' of type {typeof(T).Name} not found. Available: {availableNames}"); + } + + return match; + } +} diff --git a/HEC.FDA.TestingUtility/Services/StudyLoader.cs b/HEC.FDA.TestingUtility/Services/StudyLoader.cs new file mode 100644 index 000000000..89b8b1606 --- /dev/null +++ b/HEC.FDA.TestingUtility/Services/StudyLoader.cs @@ -0,0 +1,160 @@ +using HEC.FDA.TestingUtility.Configuration; +using HEC.FDA.ViewModel; +using HEC.FDA.ViewModel.AggregatedStageDamage; +using HEC.FDA.ViewModel.Alternatives; +using HEC.FDA.ViewModel.FlowTransforms; +using HEC.FDA.ViewModel.FrequencyRelationships; +using HEC.FDA.ViewModel.GeoTech; +using HEC.FDA.ViewModel.Hydraulics.GriddedData; +using HEC.FDA.ViewModel.ImpactArea; +using HEC.FDA.ViewModel.ImpactAreaScenario; +using HEC.FDA.ViewModel.Inventory; +using HEC.FDA.ViewModel.Inventory.OccupancyTypes; +using HEC.FDA.ViewModel.Saving; +using HEC.FDA.ViewModel.StageTransforms; +using HEC.FDA.ViewModel.Storage; +using HEC.FDA.ViewModel.Study; +using HEC.FDA.ViewModel.Utilities; +using HEC.FDA.ViewModel.Watershed; + +namespace HEC.FDA.TestingUtility.Services; + +public class StudyLoader : IDisposable +{ + private string? _localStudyPath; + private bool _disposed; + + public void LoadStudy(string networkSourcePath, string localTempDirectory) + { + // 1. Copy study folder from network to local temp directory + _localStudyPath = CopyStudyToLocal(networkSourcePath, localTempDirectory); + + // 2. Find the database file + string dbPath = FindDatabaseFile(_localStudyPath); + + // 3. Set up connection (triggers folder structure creation) + Connection.Instance.ProjectFile = dbPath; + + // 4. Create and assign cache + FDACache cache = new(); + BaseViewModel.StudyCache = cache; + PersistenceFactory.StudyCacheForSaving = cache; + + // 5. Load elements in dependency order + LoadAllElements(); + } + + private static string CopyStudyToLocal(string networkPath, string localTempDir) + { + string studyFolderName = Path.GetFileName(networkPath.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)); + string localPath = Path.Combine(localTempDir, studyFolderName + "_" + Guid.NewGuid().ToString("N")[..8]); + + Console.WriteLine($" Copying study from {networkPath} to {localPath}..."); + + CopyDirectory(networkPath, localPath); + + Console.WriteLine($" Study copied successfully."); + return localPath; + } + + private static void CopyDirectory(string sourceDir, string destDir) + { + Directory.CreateDirectory(destDir); + + foreach (string file in Directory.GetFiles(sourceDir)) + { + string destFile = Path.Combine(destDir, Path.GetFileName(file)); + File.Copy(file, destFile, overwrite: true); + } + + foreach (string subDir in Directory.GetDirectories(sourceDir)) + { + string destSubDir = Path.Combine(destDir, Path.GetFileName(subDir)); + CopyDirectory(subDir, destSubDir); + } + } + + private static string FindDatabaseFile(string studyPath) + { + // Look for .sqlite or .db files + string[] dbExtensions = { "*.sqlite", "*.db" }; + + foreach (string pattern in dbExtensions) + { + string[] files = Directory.GetFiles(studyPath, pattern); + if (files.Length > 0) + { + return files[0]; + } + } + + throw new FileNotFoundException($"No database file (.sqlite or .db) found in study folder: {studyPath}"); + } + + private static void LoadAllElements() + { + // Order matters - dependencies first + Console.WriteLine(" Loading study elements..."); + + LoadElementType("Terrains"); + LoadElementType("Impact Areas"); + LoadElementType("Hydraulics"); + LoadElementType("Frequency Relationships"); + LoadElementType("Inflow-Outflow"); + LoadElementType("Stage-Discharge"); + LoadElementType("Exterior-Interior"); + LoadElementType("Lateral Structures"); + LoadElementType("Occupancy Types"); + LoadElementType("Inventories"); + LoadElementType("Stage Damage"); + LoadElementType("Scenarios"); + LoadElementType("Alternatives"); + LoadElementType("Study Properties"); + + Console.WriteLine(" All elements loaded."); + } + + private static void LoadElementType(string displayName) where T : ChildElement + { + try + { + PersistenceFactory.GetElementManager().Load(); + } + catch (Exception ex) + { + Console.WriteLine($" Warning: Failed to load {displayName}: {ex.Message}"); + } + } + + public void Cleanup() + { + if (_localStudyPath != null && Directory.Exists(_localStudyPath)) + { + try + { + // Close the connection first + if (!Connection.Instance.IsConnectionNull && Connection.Instance.IsOpen) + { + Connection.Instance.Close(); + } + + Directory.Delete(_localStudyPath, recursive: true); + Console.WriteLine($" Cleaned up temp study folder: {_localStudyPath}"); + } + catch (Exception ex) + { + Console.WriteLine($" Warning: Failed to cleanup temp folder: {ex.Message}"); + } + } + } + + public void Dispose() + { + if (!_disposed) + { + Cleanup(); + _disposed = true; + } + GC.SuppressFinalize(this); + } +} diff --git a/HEC.FDA.TestingUtility/TestRunner.cs b/HEC.FDA.TestingUtility/TestRunner.cs new file mode 100644 index 000000000..9e408b7e3 --- /dev/null +++ b/HEC.FDA.TestingUtility/TestRunner.cs @@ -0,0 +1,328 @@ +using System.Diagnostics; +using System.Xml.Linq; +using HEC.FDA.Model.metrics; +using HEC.FDA.TestingUtility.Comparison; +using HEC.FDA.TestingUtility.Configuration; +using HEC.FDA.TestingUtility.Services; +using HEC.FDA.ViewModel; +using HEC.FDA.ViewModel.AggregatedStageDamage; +using HEC.FDA.ViewModel.Alternatives; +using HEC.FDA.ViewModel.ImpactAreaScenario; + +namespace HEC.FDA.TestingUtility; + +public class TestRunner +{ + private readonly TestConfiguration _config; + private readonly string _outputDir; + private readonly bool _verbose; + private readonly string[]? _studyFilter; + private readonly CancellationTokenSource _cts; + + private readonly XmlResultComparer _comparer = new(); + private readonly StudyBaselineWriter _baselineWriter = new(); + private readonly ScenarioRunner _scenarioRunner = new(); + private readonly AlternativeRunner _alternativeRunner = new(); + private readonly StageDamageRunner _stageDamageRunner = new(); + + public TestRunner(TestConfiguration config, string outputDir, bool verbose, string[]? studyFilter) + { + _config = config; + _outputDir = outputDir; + _verbose = verbose; + _studyFilter = studyFilter; + _cts = new CancellationTokenSource(); + + // Set timeout + if (_config.GlobalSettings.TimeoutMinutes > 0) + { + _cts.CancelAfter(TimeSpan.FromMinutes(_config.GlobalSettings.TimeoutMinutes)); + } + } + + public async Task RunAsync() + { + int failures = 0; + int passed = 0; + var totalStopwatch = Stopwatch.StartNew(); + var studyTimings = new List<(string StudyId, TimeSpan Duration, List<(string Type, string Name, TimeSpan Duration)> Computations)>(); + + Console.WriteLine($"Starting test suite: {_config.TestSuiteId}"); + Console.WriteLine($"Output directory: {_outputDir}"); + Console.WriteLine(); + + var studiesToRun = _config.Studies; + if (_studyFilter != null && _studyFilter.Length > 0) + { + studiesToRun = studiesToRun + .Where(s => _studyFilter.Contains(s.StudyId, StringComparer.OrdinalIgnoreCase)) + .ToList(); + + if (studiesToRun.Count == 0) + { + Console.WriteLine($"No studies match the filter: {string.Join(", ", _studyFilter)}"); + return 1; + } + } + + foreach (var study in studiesToRun) + { + Console.WriteLine($"=== Testing study: {study.StudyName} ({study.StudyId}) ==="); + var studyStopwatch = Stopwatch.StartNew(); + var computationTimings = new List<(string Type, string Name, TimeSpan Duration)>(); + + try + { + using var loader = new StudyLoader(); + loader.LoadStudy(study.NetworkSourcePath, _config.GlobalSettings.LocalTempDirectory); + + // Load the single baseline file for this study + string baselinePath = GetBaselinePath(study); + Console.WriteLine($" Loading baseline: {baselinePath}"); + + if (!File.Exists(baselinePath)) + { + Console.WriteLine($" WARNING: Baseline file not found. Tests will fail."); + } + else + { + _comparer.LoadBaseline(baselinePath); + } + + // Create computed results document for debugging + var computedBaseline = _baselineWriter.CreateStudyBaseline(study.StudyId, study.StudyName); + + // Build computation list (from config or auto-discover) + var computations = BuildComputationList(study); + Console.WriteLine($" Found {computations.Count} computations to run."); + + foreach (var compute in computations) + { + _cts.Token.ThrowIfCancellationRequested(); + var computeStopwatch = Stopwatch.StartNew(); + + try + { + ComparisonResult? result = null; + + switch (compute.Type.ToLowerInvariant()) + { + case "scenario": + var scenarioResults = _scenarioRunner.RunScenario(compute.ElementName, _cts.Token); + _baselineWriter.AddScenarioResults(computedBaseline, compute.ElementName, scenarioResults); + result = _comparer.CompareScenarioResults(compute.ElementName, scenarioResults); + break; + + case "alternative": + var altResults = _alternativeRunner.RunAlternative(compute.ElementName, _cts.Token); + _baselineWriter.AddAlternativeResults(computedBaseline, compute.ElementName, altResults); + result = _comparer.CompareAlternativeResults(compute.ElementName, altResults); + break; + + case "stagedamage": + var sdElement = _stageDamageRunner.GetStageDamageElement(compute.ElementName); + _baselineWriter.AddStageDamage(computedBaseline, compute.ElementName, sdElement); + result = _comparer.CompareStageDamage(compute.ElementName, sdElement); + break; + + default: + Console.WriteLine($" SKIP: Unknown compute type '{compute.Type}'"); + continue; + } + + computeStopwatch.Stop(); + computationTimings.Add((compute.Type, compute.ElementName, computeStopwatch.Elapsed)); + + if (result != null) + { + if (!result.Passed) + { + failures++; + Console.WriteLine($" FAIL: {compute.Type} '{compute.ElementName}' [{FormatDuration(computeStopwatch.Elapsed)}]"); + Console.WriteLine($" {result.Summary}"); + PrintDifferences(result); + } + else + { + passed++; + Console.WriteLine($" PASS: {compute.Type} '{compute.ElementName}' [{FormatDuration(computeStopwatch.Elapsed)}]"); + } + } + } + catch (Exception ex) + { + computeStopwatch.Stop(); + computationTimings.Add((compute.Type, compute.ElementName, computeStopwatch.Elapsed)); + failures++; + Console.WriteLine($" ERROR: {compute.Type} '{compute.ElementName}' [{FormatDuration(computeStopwatch.Elapsed)}] - {ex.Message}"); + if (_verbose) + { + Console.WriteLine($" {ex.StackTrace}"); + } + } + } + + // Save computed results for debugging + SaveComputedResults(computedBaseline, study); + } + catch (OperationCanceledException) + { + Console.WriteLine(" TIMEOUT: Test run exceeded time limit."); + failures++; + break; + } + catch (Exception ex) + { + Console.WriteLine($" ERROR loading study: {ex.Message}"); + if (_verbose) + { + Console.WriteLine($" {ex.StackTrace}"); + } + failures++; + } + + studyStopwatch.Stop(); + studyTimings.Add((study.StudyId, studyStopwatch.Elapsed, computationTimings)); + Console.WriteLine($" Study completed in {FormatDuration(studyStopwatch.Elapsed)}"); + Console.WriteLine(); + } + + totalStopwatch.Stop(); + + // Summary + Console.WriteLine("=== Summary ==="); + Console.WriteLine($"Passed: {passed}"); + Console.WriteLine($"Failed: {failures}"); + Console.WriteLine($"Total: {passed + failures}"); + Console.WriteLine(); + + // Timing Summary + Console.WriteLine("=== Timing Summary ==="); + Console.WriteLine($"Total Duration: {FormatDuration(totalStopwatch.Elapsed)}"); + Console.WriteLine(); + + foreach (var (studyId, duration, computations) in studyTimings) + { + Console.WriteLine($" {studyId}: {FormatDuration(duration)}"); + foreach (var (type, name, compDuration) in computations) + { + Console.WriteLine($" - {type} '{name}': {FormatDuration(compDuration)}"); + } + } + + return failures > 0 ? 1 : 0; + } + + private static string FormatDuration(TimeSpan duration) + { + if (duration.TotalHours >= 1) + { + return $"{duration.Hours}h {duration.Minutes}m {duration.Seconds}s"; + } + else if (duration.TotalMinutes >= 1) + { + return $"{duration.Minutes}m {duration.Seconds}.{duration.Milliseconds / 100}s"; + } + else + { + return $"{duration.Seconds}.{duration.Milliseconds:D3}s"; + } + } + + private List BuildComputationList(StudyConfiguration study) + { + var computations = new List(study.Computations); + + // Auto-discover scenarios + if (study.RunAllScenarios) + { + var scenarios = BaseViewModel.StudyCache.GetChildElementsOfType(); + foreach (var scenario in scenarios) + { + if (!computations.Any(c => c.Type.Equals("scenario", StringComparison.OrdinalIgnoreCase) + && c.ElementName.Equals(scenario.Name, StringComparison.OrdinalIgnoreCase))) + { + computations.Add(new ComputeConfiguration + { + Type = "scenario", + ElementName = scenario.Name + }); + Console.WriteLine($" Auto-discovered scenario: {scenario.Name}"); + } + } + } + + // Auto-discover stage damage elements + if (study.RunAllStageDamage) + { + var stageDamages = BaseViewModel.StudyCache.GetChildElementsOfType(); + foreach (var sd in stageDamages) + { + if (!computations.Any(c => c.Type.Equals("stagedamage", StringComparison.OrdinalIgnoreCase) + && c.ElementName.Equals(sd.Name, StringComparison.OrdinalIgnoreCase))) + { + computations.Add(new ComputeConfiguration + { + Type = "stagedamage", + ElementName = sd.Name + }); + Console.WriteLine($" Auto-discovered stage damage: {sd.Name}"); + } + } + } + + // Auto-discover alternatives (these depend on scenario results, so run last) + if (study.RunAllAlternatives) + { + var alternatives = BaseViewModel.StudyCache.GetChildElementsOfType(); + foreach (var alt in alternatives) + { + if (!computations.Any(c => c.Type.Equals("alternative", StringComparison.OrdinalIgnoreCase) + && c.ElementName.Equals(alt.Name, StringComparison.OrdinalIgnoreCase))) + { + computations.Add(new ComputeConfiguration + { + Type = "alternative", + ElementName = alt.Name + }); + Console.WriteLine($" Auto-discovered alternative: {alt.Name}"); + } + } + } + + return computations; + } + + private string GetBaselinePath(StudyConfiguration study) + { + // Single baseline file per study + return Path.Combine(study.BaselineDirectory, $"{study.StudyId}_baseline.xml"); + } + + private void SaveComputedResults(XElement computedBaseline, StudyConfiguration study) + { + // Save computed results in same format as baseline for easy comparison + string outputPath = Path.Combine(_outputDir, $"{study.StudyId}_computed.xml"); + _baselineWriter.Save(computedBaseline, outputPath); + Console.WriteLine($" Computed results saved to: {outputPath}"); + } + + private void PrintDifferences(ComparisonResult result) + { + if (!_verbose || result.Differences.Count == 0) + { + return; + } + + Console.WriteLine(" Differences:"); + foreach (var diff in result.Differences.Take(10)) // Limit to first 10 + { + Console.WriteLine($" - {diff}"); + } + + if (result.Differences.Count > 10) + { + Console.WriteLine($" ... and {result.Differences.Count - 10} more"); + } + } +} diff --git a/HEC.FDA.ViewModel/HEC.FDA.ViewModel.csproj b/HEC.FDA.ViewModel/HEC.FDA.ViewModel.csproj index e3c9b6b5c..ab8d8b21f 100644 --- a/HEC.FDA.ViewModel/HEC.FDA.ViewModel.csproj +++ b/HEC.FDA.ViewModel/HEC.FDA.ViewModel.csproj @@ -5,6 +5,10 @@ true HEC.FDA.ViewModel + + + + From aae48b2d51a255bd5500a33f34b1b29879f2084c Mon Sep 17 00:00:00 2001 From: Brennan Beam <64556723+Brennan1994@users.noreply.github.com> Date: Tue, 23 Dec 2025 10:44:19 -0800 Subject: [PATCH 02/13] better reporting --- .claude/settings.local.json | 3 +- .../Reporting/CsvReportFactory.cs | 277 ++++++++++++++++++ HEC.FDA.TestingUtility/TestRunner.cs | 10 + 3 files changed, 289 insertions(+), 1 deletion(-) create mode 100644 HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 64801ed5d..a9fb42add 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -8,7 +8,8 @@ "Bash(dir \"C:\\\\Users\\\\HEC\\\\Data\\\\FDA\\\\Canon\" /b)", "Bash(nuget sources:*)", "Bash(dotnet restore:*)", - "Bash(./HEC.FDA.TestingUtility.exe:*)" + "Bash(./HEC.FDA.TestingUtility.exe:*)", + "Bash(./HEC.FDA.TestingUtility/bin/Debug/net9.0-windows/HEC.FDA.TestingUtility.exe:*)" ], "deny": [] } diff --git a/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs b/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs new file mode 100644 index 000000000..5b35e9743 --- /dev/null +++ b/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs @@ -0,0 +1,277 @@ +using System.Text; +using HEC.FDA.Model.metrics; +using HEC.FDA.ViewModel.AggregatedStageDamage; + +namespace HEC.FDA.TestingUtility.Reporting; + +/// +/// Factory class that produces a comprehensive CSV report containing all tabularly +/// visualized results from FDA computations across all studies. +/// +public class CsvReportFactory +{ + private readonly StringBuilder _scenarioResults = new(); + private readonly StringBuilder _scenarioDamageByCategory = new(); + private readonly StringBuilder _scenarioPerformance = new(); + private readonly StringBuilder _alternativeResults = new(); + private readonly StringBuilder _alternativeDamageByCategory = new(); + private readonly StringBuilder _stageDamageSummary = new(); + + public CsvReportFactory() + { + WriteHeaders(); + } + + private void WriteHeaders() + { + // Scenario Results header + _scenarioResults.AppendLine("Study ID,Scenario Name,Impact Area ID,Mean EAD,EAD 25th Pct,EAD 50th Pct,EAD 75th Pct"); + + // Scenario Damage by Category header + _scenarioDamageByCategory.AppendLine("Study ID,Scenario Name,Impact Area ID,Damage Category,Asset Category,Mean EAD"); + + // Scenario Performance header + _scenarioPerformance.AppendLine("Study ID,Scenario Name,Impact Area ID,Threshold ID,Mean AEP,Median AEP,Assurance 0.10,Assurance 0.04,Assurance 0.02,Assurance 0.01,LT Risk 10yr,LT Risk 30yr,LT Risk 50yr"); + + // Alternative Results header + _alternativeResults.AppendLine("Study ID,Alternative Name,Impact Area ID,Base Year,Future Year,Period of Analysis,Mean Base EAD,Mean Future EAD,Mean EqAD,EqAD 25th Pct,EqAD 50th Pct,EqAD 75th Pct"); + + // Alternative Damage by Category header + _alternativeDamageByCategory.AppendLine("Study ID,Alternative Name,Impact Area ID,Damage Category,Asset Category,Mean EqAD"); + + // Stage Damage Summary header + _stageDamageSummary.AppendLine("Study ID,Element Name,Impact Area ID,Impact Area Name,Damage Category,Asset Category,Point Count,Min Stage,Max Stage"); + } + + /// + /// Adds scenario results from a computation to the report. + /// + public void AddScenarioResults(string studyId, string scenarioName, ScenarioResults results) + { + if (results == null) return; + + try + { + var impactAreaIds = results.GetImpactAreaIDs(); + var damageCategories = results.GetDamageCategories(); + var assetCategories = results.GetAssetCategories(); + + // Write aggregate results per impact area + foreach (int impactAreaId in impactAreaIds) + { + double meanEAD = results.SampleMeanExpectedAnnualConsequences(impactAreaId); + double ead25 = results.ConsequencesExceededWithProbabilityQ(0.75, impactAreaId); // 25th percentile = exceeded by 75% + double ead50 = results.ConsequencesExceededWithProbabilityQ(0.50, impactAreaId); + double ead75 = results.ConsequencesExceededWithProbabilityQ(0.25, impactAreaId); // 75th percentile = exceeded by 25% + + _scenarioResults.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(scenarioName)},{impactAreaId},{meanEAD:F2},{ead25:F2},{ead50:F2},{ead75:F2}"); + } + + // Write damage by category + foreach (int impactAreaId in impactAreaIds) + { + foreach (string damCat in damageCategories) + { + foreach (string assetCat in assetCategories) + { + double meanEAD = results.SampleMeanExpectedAnnualConsequences(impactAreaId, damCat, assetCat); + if (meanEAD != 0) // Only write non-zero values + { + _scenarioDamageByCategory.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(scenarioName)},{impactAreaId},{EscapeCsv(damCat)},{EscapeCsv(assetCat)},{meanEAD:F2}"); + } + } + } + } + + // Write performance metrics (if thresholds exist) + foreach (int impactAreaId in impactAreaIds) + { + WritePerformanceMetrics(studyId, scenarioName, impactAreaId, results); + } + } + catch (Exception ex) + { + Console.WriteLine($" Warning: Error extracting scenario results for CSV: {ex.Message}"); + } + } + + private void WritePerformanceMetrics(string studyId, string scenarioName, int impactAreaId, ScenarioResults results) + { + results.ResultsList + try + { + // Try threshold ID 0 (default) - this is typically the only threshold in most scenarios + int thresholdId = 0; + + double meanAEP = results.MeanAEP(impactAreaId, thresholdId); + double medianAEP = results.MedianAEP(impactAreaId, thresholdId); + + // Assurance values (probability of not exceeding standard event) + double assurance10 = results.AssuranceOfEvent(impactAreaId, 0.10, thresholdId); + double assurance04 = results.AssuranceOfEvent(impactAreaId, 0.04, thresholdId); + double assurance02 = results.AssuranceOfEvent(impactAreaId, 0.02, thresholdId); + double assurance01 = results.AssuranceOfEvent(impactAreaId, 0.01, thresholdId); + + // Long-term risk + double ltRisk10 = results.LongTermExceedanceProbability(impactAreaId, 10, thresholdId); + double ltRisk30 = results.LongTermExceedanceProbability(impactAreaId, 30, thresholdId); + double ltRisk50 = results.LongTermExceedanceProbability(impactAreaId, 50, thresholdId); + + _scenarioPerformance.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(scenarioName)},{impactAreaId},{thresholdId},{meanAEP:F6},{medianAEP:F6},{assurance10:F4},{assurance04:F4},{assurance02:F4},{assurance01:F4},{ltRisk10:F4},{ltRisk30:F4},{ltRisk50:F4}"); + } + catch + { + // Performance metrics may not be available for all scenarios + } + } + + /// + /// Adds alternative results from a computation to the report. + /// + public void AddAlternativeResults(string studyId, string alternativeName, AlternativeResults results) + { + if (results == null || results.IsNull) return; + + try + { + var impactAreaIds = results.GetImpactAreaIDs(); + var damageCategories = results.GetDamageCategories(); + var assetCategories = results.GetAssetCategories(); + + int baseYear = results.AnalysisYears.Count > 0 ? results.AnalysisYears[0] : 0; + int futureYear = results.AnalysisYears.Count > 1 ? results.AnalysisYears[1] : 0; + int periodOfAnalysis = results.PeriodOfAnalysis; + + // Write aggregate results per impact area + foreach (int impactAreaId in impactAreaIds) + { + double meanBaseEAD = results.SampleMeanBaseYearEAD(impactAreaId); + double meanFutureEAD = results.SampleMeanFutureYearEAD(impactAreaId); + double meanEqAD = results.SampleMeanEqad(impactAreaId); + double eqad25 = results.EqadExceededWithProbabilityQ(0.75, impactAreaId); + double eqad50 = results.EqadExceededWithProbabilityQ(0.50, impactAreaId); + double eqad75 = results.EqadExceededWithProbabilityQ(0.25, impactAreaId); + + _alternativeResults.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(alternativeName)},{impactAreaId},{baseYear},{futureYear},{periodOfAnalysis},{meanBaseEAD:F2},{meanFutureEAD:F2},{meanEqAD:F2},{eqad25:F2},{eqad50:F2},{eqad75:F2}"); + } + + // Write damage by category + foreach (int impactAreaId in impactAreaIds) + { + foreach (string damCat in damageCategories) + { + foreach (string assetCat in assetCategories) + { + double meanEqAD = results.SampleMeanEqad(impactAreaId, damCat, assetCat); + if (meanEqAD != 0) // Only write non-zero values + { + _alternativeDamageByCategory.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(alternativeName)},{impactAreaId},{EscapeCsv(damCat)},{EscapeCsv(assetCat)},{meanEqAD:F2}"); + } + } + } + } + } + catch (Exception ex) + { + Console.WriteLine($" Warning: Error extracting alternative results for CSV: {ex.Message}"); + } + } + + /// + /// Adds stage damage summary from an element to the report. + /// + public void AddStageDamageSummary(string studyId, AggregatedStageDamageElement element) + { + if (element == null) return; + + try + { + foreach (var curve in element.Curves) + { + int impactAreaId = curve.ImpArea?.ID ?? -1; + string impactAreaName = curve.ImpArea?.Name ?? ""; + string damCat = curve.DamCat ?? ""; + string assetCat = curve.AssetCategory ?? ""; + + // Get curve data points + int pointCount = 0; + double minStage = 0; + double maxStage = 0; + + try + { + var pairedData = curve.ComputeComponent?.SelectedItemToPairedData(); + if (pairedData != null && pairedData.Xvals != null && pairedData.Xvals.Length > 0) + { + pointCount = pairedData.Xvals.Length; + minStage = pairedData.Xvals.Min(); + maxStage = pairedData.Xvals.Max(); + } + } + catch + { + // Curve data may not be available + } + + _stageDamageSummary.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(element.Name)},{impactAreaId},{EscapeCsv(impactAreaName)},{EscapeCsv(damCat)},{EscapeCsv(assetCat)},{pointCount},{minStage:F2},{maxStage:F2}"); + } + } + catch (Exception ex) + { + Console.WriteLine($" Warning: Error extracting stage damage for CSV: {ex.Message}"); + } + } + + /// + /// Saves the comprehensive report to a CSV file. + /// + public void SaveReport(string outputPath) + { + var report = new StringBuilder(); + + report.AppendLine("=== FDA COMPUTATION RESULTS REPORT ==="); + report.AppendLine($"Generated: {DateTime.Now:yyyy-MM-dd HH:mm:ss}"); + report.AppendLine(); + + report.AppendLine("=== SCENARIO RESULTS ==="); + report.Append(_scenarioResults); + report.AppendLine(); + + report.AppendLine("=== SCENARIO DAMAGE BY CATEGORY ==="); + report.Append(_scenarioDamageByCategory); + report.AppendLine(); + + report.AppendLine("=== SCENARIO PERFORMANCE ==="); + report.Append(_scenarioPerformance); + report.AppendLine(); + + report.AppendLine("=== ALTERNATIVE RESULTS ==="); + report.Append(_alternativeResults); + report.AppendLine(); + + report.AppendLine("=== ALTERNATIVE DAMAGE BY CATEGORY ==="); + report.Append(_alternativeDamageByCategory); + report.AppendLine(); + + report.AppendLine("=== STAGE DAMAGE SUMMARY ==="); + report.Append(_stageDamageSummary); + + File.WriteAllText(outputPath, report.ToString()); + Console.WriteLine($"CSV report saved to: {outputPath}"); + } + + /// + /// Escapes a value for CSV output (handles commas and quotes). + /// + private static string EscapeCsv(string value) + { + if (string.IsNullOrEmpty(value)) return ""; + + if (value.Contains(',') || value.Contains('"') || value.Contains('\n')) + { + return $"\"{value.Replace("\"", "\"\"")}\""; + } + + return value; + } +} diff --git a/HEC.FDA.TestingUtility/TestRunner.cs b/HEC.FDA.TestingUtility/TestRunner.cs index 9e408b7e3..ad8d026b8 100644 --- a/HEC.FDA.TestingUtility/TestRunner.cs +++ b/HEC.FDA.TestingUtility/TestRunner.cs @@ -3,6 +3,7 @@ using HEC.FDA.Model.metrics; using HEC.FDA.TestingUtility.Comparison; using HEC.FDA.TestingUtility.Configuration; +using HEC.FDA.TestingUtility.Reporting; using HEC.FDA.TestingUtility.Services; using HEC.FDA.ViewModel; using HEC.FDA.ViewModel.AggregatedStageDamage; @@ -24,6 +25,7 @@ public class TestRunner private readonly ScenarioRunner _scenarioRunner = new(); private readonly AlternativeRunner _alternativeRunner = new(); private readonly StageDamageRunner _stageDamageRunner = new(); + private readonly CsvReportFactory _csvReportFactory = new(); public TestRunner(TestConfiguration config, string outputDir, bool verbose, string[]? studyFilter) { @@ -110,18 +112,21 @@ public async Task RunAsync() case "scenario": var scenarioResults = _scenarioRunner.RunScenario(compute.ElementName, _cts.Token); _baselineWriter.AddScenarioResults(computedBaseline, compute.ElementName, scenarioResults); + _csvReportFactory.AddScenarioResults(study.StudyId, compute.ElementName, scenarioResults); result = _comparer.CompareScenarioResults(compute.ElementName, scenarioResults); break; case "alternative": var altResults = _alternativeRunner.RunAlternative(compute.ElementName, _cts.Token); _baselineWriter.AddAlternativeResults(computedBaseline, compute.ElementName, altResults); + _csvReportFactory.AddAlternativeResults(study.StudyId, compute.ElementName, altResults); result = _comparer.CompareAlternativeResults(compute.ElementName, altResults); break; case "stagedamage": var sdElement = _stageDamageRunner.GetStageDamageElement(compute.ElementName); _baselineWriter.AddStageDamage(computedBaseline, compute.ElementName, sdElement); + _csvReportFactory.AddStageDamageSummary(study.StudyId, sdElement); result = _comparer.CompareStageDamage(compute.ElementName, sdElement); break; @@ -210,6 +215,11 @@ public async Task RunAsync() } } + // Save CSV report + Console.WriteLine(); + string csvPath = Path.Combine(_outputDir, "results_report.csv"); + _csvReportFactory.SaveReport(csvPath); + return failures > 0 ? 1 : 0; } From eefe0f4cf74e41d948250e26bd0da5c1a787cc3b Mon Sep 17 00:00:00 2001 From: Brennan Beam <64556723+Brennan1994@users.noreply.github.com> Date: Tue, 23 Dec 2025 10:57:09 -0800 Subject: [PATCH 03/13] report all thresholds --- .../Reporting/CsvReportFactory.cs | 93 +++++++++---------- 1 file changed, 44 insertions(+), 49 deletions(-) diff --git a/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs b/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs index 5b35e9743..3bd4178ae 100644 --- a/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs +++ b/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs @@ -45,6 +45,7 @@ private void WriteHeaders() /// /// Adds scenario results from a computation to the report. + /// Iterates over ResultsList for direct access to each ImpactAreaScenarioResults. /// public void AddScenarioResults(string studyId, string scenarioName, ScenarioResults results) { @@ -52,41 +53,31 @@ public void AddScenarioResults(string studyId, string scenarioName, ScenarioResu try { - var impactAreaIds = results.GetImpactAreaIDs(); - var damageCategories = results.GetDamageCategories(); - var assetCategories = results.GetAssetCategories(); - - // Write aggregate results per impact area - foreach (int impactAreaId in impactAreaIds) + // Iterate over the ResultsList directly for better access to impact area data + foreach (var iaResult in results.ResultsList) { - double meanEAD = results.SampleMeanExpectedAnnualConsequences(impactAreaId); - double ead25 = results.ConsequencesExceededWithProbabilityQ(0.75, impactAreaId); // 25th percentile = exceeded by 75% - double ead50 = results.ConsequencesExceededWithProbabilityQ(0.50, impactAreaId); - double ead75 = results.ConsequencesExceededWithProbabilityQ(0.25, impactAreaId); // 75th percentile = exceeded by 25% + int impactAreaId = iaResult.ImpactAreaID; + + // Write aggregate EAD results for this impact area + double meanEAD = iaResult.MeanExpectedAnnualConsequences(); + double ead25 = iaResult.ConsequencesExceededWithProbabilityQ(0.75); // 25th percentile = exceeded by 75% + double ead50 = iaResult.ConsequencesExceededWithProbabilityQ(0.50); + double ead75 = iaResult.ConsequencesExceededWithProbabilityQ(0.25); // 75th percentile = exceeded by 25% _scenarioResults.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(scenarioName)},{impactAreaId},{meanEAD:F2},{ead25:F2},{ead50:F2},{ead75:F2}"); - } - // Write damage by category - foreach (int impactAreaId in impactAreaIds) - { - foreach (string damCat in damageCategories) + // Write damage by category from ConsequenceResults + foreach (var consequence in iaResult.ConsequenceResults.ConsequenceResultList) { - foreach (string assetCat in assetCategories) - { - double meanEAD = results.SampleMeanExpectedAnnualConsequences(impactAreaId, damCat, assetCat); - if (meanEAD != 0) // Only write non-zero values - { - _scenarioDamageByCategory.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(scenarioName)},{impactAreaId},{EscapeCsv(damCat)},{EscapeCsv(assetCat)},{meanEAD:F2}"); - } - } + string damCat = consequence.DamageCategory; + string assetCat = consequence.AssetCategory; + double categoryMeanEAD = iaResult.MeanExpectedAnnualConsequences(impactAreaId, damCat, assetCat); + + _scenarioDamageByCategory.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(scenarioName)},{impactAreaId},{EscapeCsv(damCat)},{EscapeCsv(assetCat)},{categoryMeanEAD:F2}"); } - } - // Write performance metrics (if thresholds exist) - foreach (int impactAreaId in impactAreaIds) - { - WritePerformanceMetrics(studyId, scenarioName, impactAreaId, results); + // Write performance metrics for each threshold in this impact area + WritePerformanceMetrics(studyId, scenarioName, iaResult); } } catch (Exception ex) @@ -95,33 +86,37 @@ public void AddScenarioResults(string studyId, string scenarioName, ScenarioResu } } - private void WritePerformanceMetrics(string studyId, string scenarioName, int impactAreaId, ScenarioResults results) + private void WritePerformanceMetrics(string studyId, string scenarioName, ImpactAreaScenarioResults iaResult) { - results.ResultsList - try + int impactAreaId = iaResult.ImpactAreaID; + + // Iterate over all available thresholds for this impact area + foreach (var threshold in iaResult.PerformanceByThresholds.ListOfThresholds) { - // Try threshold ID 0 (default) - this is typically the only threshold in most scenarios - int thresholdId = 0; + try + { + int thresholdId = threshold.ThresholdID; - double meanAEP = results.MeanAEP(impactAreaId, thresholdId); - double medianAEP = results.MedianAEP(impactAreaId, thresholdId); + double meanAEP = iaResult.MeanAEP(thresholdId); + double medianAEP = iaResult.MedianAEP(thresholdId); - // Assurance values (probability of not exceeding standard event) - double assurance10 = results.AssuranceOfEvent(impactAreaId, 0.10, thresholdId); - double assurance04 = results.AssuranceOfEvent(impactAreaId, 0.04, thresholdId); - double assurance02 = results.AssuranceOfEvent(impactAreaId, 0.02, thresholdId); - double assurance01 = results.AssuranceOfEvent(impactAreaId, 0.01, thresholdId); + // Assurance values (probability of not exceeding standard event) + double assurance10 = iaResult.AssuranceOfEvent(thresholdId, 0.10); + double assurance04 = iaResult.AssuranceOfEvent(thresholdId, 0.04); + double assurance02 = iaResult.AssuranceOfEvent(thresholdId, 0.02); + double assurance01 = iaResult.AssuranceOfEvent(thresholdId, 0.01); - // Long-term risk - double ltRisk10 = results.LongTermExceedanceProbability(impactAreaId, 10, thresholdId); - double ltRisk30 = results.LongTermExceedanceProbability(impactAreaId, 30, thresholdId); - double ltRisk50 = results.LongTermExceedanceProbability(impactAreaId, 50, thresholdId); + // Long-term risk + double ltRisk10 = iaResult.LongTermExceedanceProbability(thresholdId, 10); + double ltRisk30 = iaResult.LongTermExceedanceProbability(thresholdId, 30); + double ltRisk50 = iaResult.LongTermExceedanceProbability(thresholdId, 50); - _scenarioPerformance.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(scenarioName)},{impactAreaId},{thresholdId},{meanAEP:F6},{medianAEP:F6},{assurance10:F4},{assurance04:F4},{assurance02:F4},{assurance01:F4},{ltRisk10:F4},{ltRisk30:F4},{ltRisk50:F4}"); - } - catch - { - // Performance metrics may not be available for all scenarios + _scenarioPerformance.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(scenarioName)},{impactAreaId},{thresholdId},{meanAEP:F6},{medianAEP:F6},{assurance10:F4},{assurance04:F4},{assurance02:F4},{assurance01:F4},{ltRisk10:F4},{ltRisk30:F4},{ltRisk50:F4}"); + } + catch + { + // Performance metrics may not be available for this threshold + } } } From f32979ef2b7fca7c596a895bca1dd56f556a1fe0 Mon Sep 17 00:00:00 2001 From: Brennan Beam <64556723+Brennan1994@users.noreply.github.com> Date: Tue, 23 Dec 2025 11:18:56 -0800 Subject: [PATCH 04/13] Stage Damage runner implemented better. actually runs compute now --- .../Comparison/ComparisonResult.cs | 6 ++ .../Comparison/StudyBaselineWriter.cs | 13 ++- .../Comparison/XmlResultComparer.cs | 89 ++++++++++++++--- .../Reporting/CsvReportFactory.cs | 38 +++----- .../Services/StageDamageRunner.cs | 97 ++++++++++++++++++- HEC.FDA.TestingUtility/TestRunner.cs | 9 +- 6 files changed, 201 insertions(+), 51 deletions(-) diff --git a/HEC.FDA.TestingUtility/Comparison/ComparisonResult.cs b/HEC.FDA.TestingUtility/Comparison/ComparisonResult.cs index fe242d766..35db2910d 100644 --- a/HEC.FDA.TestingUtility/Comparison/ComparisonResult.cs +++ b/HEC.FDA.TestingUtility/Comparison/ComparisonResult.cs @@ -18,6 +18,8 @@ public class Difference public string Metric { get; set; } = string.Empty; public double? Expected { get; set; } public double? Actual { get; set; } + public string? ExpectedDescription { get; set; } + public string? ActualDescription { get; set; } public double? AbsoluteDifference => Expected.HasValue && Actual.HasValue ? Math.Abs(Expected.Value - Actual.Value) : null; @@ -31,6 +33,10 @@ public override string ToString() { return $"{Metric}: Expected={Expected:F4}, Actual={Actual:F4}, Diff={AbsoluteDifference:F4} ({PercentDifference:F2}%)"; } + if (ExpectedDescription != null || ActualDescription != null) + { + return $"{Metric}: Expected={ExpectedDescription ?? "null"}, Actual={ActualDescription ?? "null"}"; + } return $"{Metric}: Expected={Expected}, Actual={Actual}"; } } diff --git a/HEC.FDA.TestingUtility/Comparison/StudyBaselineWriter.cs b/HEC.FDA.TestingUtility/Comparison/StudyBaselineWriter.cs index 832db0caf..4556947b3 100644 --- a/HEC.FDA.TestingUtility/Comparison/StudyBaselineWriter.cs +++ b/HEC.FDA.TestingUtility/Comparison/StudyBaselineWriter.cs @@ -1,6 +1,6 @@ using System.Xml.Linq; using HEC.FDA.Model.metrics; -using HEC.FDA.ViewModel.AggregatedStageDamage; +using HEC.FDA.Model.paireddata; namespace HEC.FDA.TestingUtility.Comparison; @@ -31,11 +31,18 @@ public void AddAlternativeResults(XElement baseline, string name, AlternativeRes baseline.Add(wrapper); } - public void AddStageDamage(XElement baseline, string name, AggregatedStageDamageElement element) + public void AddStageDamage(XElement baseline, string name, List curves) { + var curvesElement = new XElement("Curves"); + foreach (var curve in curves) + { + curvesElement.Add(curve.WriteToXML()); + } + var wrapper = new XElement("StageDamage", new XAttribute("name", name), - element.ToXML()); + new XAttribute("curveCount", curves.Count), + curvesElement); baseline.Add(wrapper); } diff --git a/HEC.FDA.TestingUtility/Comparison/XmlResultComparer.cs b/HEC.FDA.TestingUtility/Comparison/XmlResultComparer.cs index 6e57d6254..449a810ca 100644 --- a/HEC.FDA.TestingUtility/Comparison/XmlResultComparer.cs +++ b/HEC.FDA.TestingUtility/Comparison/XmlResultComparer.cs @@ -1,6 +1,6 @@ using System.Xml.Linq; using HEC.FDA.Model.metrics; -using HEC.FDA.ViewModel.AggregatedStageDamage; +using HEC.FDA.Model.paireddata; namespace HEC.FDA.TestingUtility.Comparison; @@ -125,7 +125,7 @@ public ComparisonResult CompareAlternativeResults(string elementName, Alternativ return result; } - public ComparisonResult CompareStageDamage(string elementName, AggregatedStageDamageElement actual) + public ComparisonResult CompareStageDamage(string elementName, List actualCurves) { var result = new ComparisonResult { ElementName = elementName, ElementType = "StageDamage" }; @@ -146,24 +146,87 @@ public ComparisonResult CompareStageDamage(string elementName, AggregatedStageDa return result; } - // Use element's Equals method for comparison - var innerXml = baselineElement.Elements().FirstOrDefault(); - if (innerXml == null) + // Get baseline curves + var curvesElement = baselineElement.Element("Curves"); + if (curvesElement == null) { result.Passed = false; - result.ErrorMessage = $"Invalid baseline format for StageDamage '{elementName}'"; + result.ErrorMessage = $"Invalid baseline format for StageDamage '{elementName}' - no Curves element"; return result; } - // Note: AggregatedStageDamageElement.Equals needs to be called on the actual element - // The baseline needs to be reconstructed from XML - this may require additional work - // For now, we'll compare the XML directly - var actualXml = actual.ToXML(); - result.Passed = XmlCompare(innerXml, actualXml); + var baselineCurveElements = curvesElement.Elements("UncertainPairedData").ToList(); - if (!result.Passed) + // Compare curve counts + if (baselineCurveElements.Count != actualCurves.Count) { - result.ErrorMessage = "Stage damage elements do not match"; + result.Passed = false; + result.Differences.Add(new Difference + { + Metric = "CurveCount", + Expected = baselineCurveElements.Count, + Actual = actualCurves.Count + }); + return result; + } + + result.Passed = true; + + // Compare each curve + for (int i = 0; i < actualCurves.Count; i++) + { + var baselineCurve = UncertainPairedData.ReadFromXML(baselineCurveElements[i]); + var actualCurve = actualCurves[i]; + + // Compare metadata + if (baselineCurve.ImpactAreaID != actualCurve.ImpactAreaID || + baselineCurve.DamageCategory != actualCurve.DamageCategory || + baselineCurve.AssetCategory != actualCurve.AssetCategory) + { + result.Passed = false; + result.Differences.Add(new Difference + { + Metric = $"Curve[{i}].Metadata", + ExpectedDescription = $"IA={baselineCurve.ImpactAreaID}, DamCat={baselineCurve.DamageCategory}, Asset={baselineCurve.AssetCategory}", + ActualDescription = $"IA={actualCurve.ImpactAreaID}, DamCat={actualCurve.DamageCategory}, Asset={actualCurve.AssetCategory}" + }); + continue; + } + + // Compare X values (stages) + if (baselineCurve.Xvals.Length != actualCurve.Xvals.Length) + { + result.Passed = false; + result.Differences.Add(new Difference + { + Metric = $"Curve[{i}].XValueCount", + Expected = baselineCurve.Xvals.Length, + Actual = actualCurve.Xvals.Length + }); + continue; + } + + // Compare mean damage values at each stage + for (int j = 0; j < baselineCurve.Xvals.Length; j++) + { + double baselineMean = baselineCurve.Yvals[j].InverseCDF(0.5); + double actualMean = actualCurve.Yvals[j].InverseCDF(0.5); + + double tolerance = 0.01; + double absoluteDiff = Math.Abs(baselineMean - actualMean); + double relativeDiff = baselineMean != 0 ? absoluteDiff / Math.Abs(baselineMean) : absoluteDiff; + + if (relativeDiff > tolerance && absoluteDiff > 1.0) + { + result.Passed = false; + result.Differences.Add(new Difference + { + Metric = $"Curve[{i}].Stage[{baselineCurve.Xvals[j]:F2}].MedianDamage", + Expected = baselineMean, + Actual = actualMean + }); + } + } } return result; diff --git a/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs b/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs index 3bd4178ae..58dc3258d 100644 --- a/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs +++ b/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs @@ -1,6 +1,6 @@ using System.Text; using HEC.FDA.Model.metrics; -using HEC.FDA.ViewModel.AggregatedStageDamage; +using HEC.FDA.Model.paireddata; namespace HEC.FDA.TestingUtility.Reporting; @@ -173,42 +173,26 @@ public void AddAlternativeResults(string studyId, string alternativeName, Altern } /// - /// Adds stage damage summary from an element to the report. + /// Adds stage damage summary from computed curves to the report. /// - public void AddStageDamageSummary(string studyId, AggregatedStageDamageElement element) + public void AddStageDamageSummary(string studyId, string elementName, List curves) { - if (element == null) return; + if (curves == null) return; try { - foreach (var curve in element.Curves) + foreach (var curve in curves) { - int impactAreaId = curve.ImpArea?.ID ?? -1; - string impactAreaName = curve.ImpArea?.Name ?? ""; - string damCat = curve.DamCat ?? ""; + int impactAreaId = curve.ImpactAreaID; + string damCat = curve.DamageCategory ?? ""; string assetCat = curve.AssetCategory ?? ""; // Get curve data points - int pointCount = 0; - double minStage = 0; - double maxStage = 0; + int pointCount = curve.Xvals?.Length ?? 0; + double minStage = pointCount > 0 ? curve.Xvals!.Min() : 0; + double maxStage = pointCount > 0 ? curve.Xvals!.Max() : 0; - try - { - var pairedData = curve.ComputeComponent?.SelectedItemToPairedData(); - if (pairedData != null && pairedData.Xvals != null && pairedData.Xvals.Length > 0) - { - pointCount = pairedData.Xvals.Length; - minStage = pairedData.Xvals.Min(); - maxStage = pairedData.Xvals.Max(); - } - } - catch - { - // Curve data may not be available - } - - _stageDamageSummary.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(element.Name)},{impactAreaId},{EscapeCsv(impactAreaName)},{EscapeCsv(damCat)},{EscapeCsv(assetCat)},{pointCount},{minStage:F2},{maxStage:F2}"); + _stageDamageSummary.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(elementName)},{impactAreaId},,{EscapeCsv(damCat)},{EscapeCsv(assetCat)},{pointCount},{minStage:F2},{maxStage:F2}"); } } catch (Exception ex) diff --git a/HEC.FDA.TestingUtility/Services/StageDamageRunner.cs b/HEC.FDA.TestingUtility/Services/StageDamageRunner.cs index 8083b0196..7526c8455 100644 --- a/HEC.FDA.TestingUtility/Services/StageDamageRunner.cs +++ b/HEC.FDA.TestingUtility/Services/StageDamageRunner.cs @@ -1,21 +1,94 @@ +using HEC.FDA.Model.paireddata; +using HEC.FDA.Model.stageDamage; using HEC.FDA.ViewModel; using HEC.FDA.ViewModel.AggregatedStageDamage; +using HEC.FDA.ViewModel.Hydraulics.GriddedData; +using HEC.FDA.ViewModel.ImpactArea; +using HEC.FDA.ViewModel.Inventory; using HEC.FDA.ViewModel.Utilities; namespace HEC.FDA.TestingUtility.Services; public class StageDamageRunner { - public AggregatedStageDamageElement GetStageDamageElement(string elementName) + public List RunStageDamage(string elementName) { - Console.WriteLine($" Retrieving stage damage element '{elementName}'..."); + Console.WriteLine($" Running stage damage compute for '{elementName}'..."); // Find element by name AggregatedStageDamageElement element = FindElement(elementName); - Console.WriteLine($" Found stage damage element with {element.Curves.Count} curves."); + // Check if this is a manual stage damage (no compute needed) + if (element.IsManual) + { + Console.WriteLine($" Stage damage '{elementName}' is manual - returning existing curves."); + return ConvertCurvesToUPD(element.Curves); + } + + // Get the referenced elements needed for compute + ImpactAreaElement impactAreaElement = GetImpactAreaElement(); + HydraulicElement hydraulicElement = FindElementById(element.SelectedWSE); + InventoryElement inventoryElement = FindElementById(element.SelectedStructures); + + Console.WriteLine($" Using hydraulics: {hydraulicElement.Name}"); + Console.WriteLine($" Using inventory: {inventoryElement.Name}"); + Console.WriteLine($" Analysis year: {element.AnalysisYear}"); + + // Create the configuration + StageDamageConfiguration config = new( + impactAreaElement, + hydraulicElement, + inventoryElement, + element.ImpactAreaFrequencyRows, + element.AnalysisYear); - return element; + // Validate the configuration + FdaValidationResult validation = config.ValidateConfiguration(); + if (!validation.IsValid) + { + throw new InvalidOperationException($"Stage damage configuration is invalid: {validation.ErrorMessage}"); + } + + // Create the stage damages and compute + List impactAreaStageDamages = config.CreateStageDamages(); + ScenarioStageDamage scenarioStageDamage = new(impactAreaStageDamages); + + // Validate structure count + int totalStructureCount = impactAreaStageDamages.Sum(sd => sd.Inventory.Structures.Count); + if (totalStructureCount == 0) + { + throw new InvalidOperationException("No structures found in any impact area for stage damage compute."); + } + + Console.WriteLine($" Computing stage damage with {totalStructureCount} structures..."); + + // Run the compute + (List stageDamageFunctions, List _) = scenarioStageDamage.Compute(); + + Console.WriteLine($" Stage damage computation complete. Generated {stageDamageFunctions.Count} curves."); + + return stageDamageFunctions; + } + + private static List ConvertCurvesToUPD(List curves) + { + List result = new(); + foreach (var curve in curves) + { + UncertainPairedData upd = curve.ComputeComponent.SelectedItemToPairedData(); + result.Add(upd); + } + return result; + } + + private static ImpactAreaElement GetImpactAreaElement() + { + var impactAreaElements = BaseViewModel.StudyCache.GetChildElementsOfType(); + if (impactAreaElements.Count == 0) + { + throw new InvalidOperationException("No impact area element found in study."); + } + return impactAreaElements[0]; } private static T FindElement(string elementName) where T : ChildElement @@ -34,4 +107,20 @@ private static T FindElement(string elementName) where T : ChildElement return match; } + + private static T FindElementById(int id) where T : ChildElement + { + var elements = BaseViewModel.StudyCache.GetChildElementsOfType(); + + var match = elements.FirstOrDefault(e => e.ID == id); + + if (match == null) + { + string availableIds = string.Join(", ", elements.Select(e => $"{e.Name}({e.ID})")); + throw new InvalidOperationException( + $"Element of type {typeof(T).Name} with ID {id} not found. Available: {availableIds}"); + } + + return match; + } } diff --git a/HEC.FDA.TestingUtility/TestRunner.cs b/HEC.FDA.TestingUtility/TestRunner.cs index ad8d026b8..b7299d7d9 100644 --- a/HEC.FDA.TestingUtility/TestRunner.cs +++ b/HEC.FDA.TestingUtility/TestRunner.cs @@ -1,6 +1,7 @@ using System.Diagnostics; using System.Xml.Linq; using HEC.FDA.Model.metrics; +using HEC.FDA.Model.paireddata; using HEC.FDA.TestingUtility.Comparison; using HEC.FDA.TestingUtility.Configuration; using HEC.FDA.TestingUtility.Reporting; @@ -124,10 +125,10 @@ public async Task RunAsync() break; case "stagedamage": - var sdElement = _stageDamageRunner.GetStageDamageElement(compute.ElementName); - _baselineWriter.AddStageDamage(computedBaseline, compute.ElementName, sdElement); - _csvReportFactory.AddStageDamageSummary(study.StudyId, sdElement); - result = _comparer.CompareStageDamage(compute.ElementName, sdElement); + List sdCurves = _stageDamageRunner.RunStageDamage(compute.ElementName); + _baselineWriter.AddStageDamage(computedBaseline, compute.ElementName, sdCurves); + _csvReportFactory.AddStageDamageSummary(study.StudyId, compute.ElementName, sdCurves); + result = _comparer.CompareStageDamage(compute.ElementName, sdCurves); break; default: From e8cadfe05aa1ffaa82c9ab27f25c14f8fcc93c51 Mon Sep 17 00:00:00 2001 From: Brennan Beam <64556723+Brennan1994@users.noreply.github.com> Date: Tue, 23 Dec 2025 15:30:30 -0800 Subject: [PATCH 05/13] cleanup --- .claude/settings.local.json | 4 +- .../Comparison/XmlResultComparer.cs | 26 +++--- HEC.FDA.TestingUtility/Program.cs | 4 + .../Reporting/CsvReportFactory.cs | 6 +- .../Services/AlternativeRunner.cs | 68 +++++++-------- .../Services/ScenarioRunner.cs | 48 ++++++++--- .../Services/StageDamageRunner.cs | 86 ++++++------------- HEC.FDA.TestingUtility/TestRunner.cs | 9 +- 8 files changed, 118 insertions(+), 133 deletions(-) diff --git a/.claude/settings.local.json b/.claude/settings.local.json index a9fb42add..efb824584 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -9,7 +9,9 @@ "Bash(nuget sources:*)", "Bash(dotnet restore:*)", "Bash(./HEC.FDA.TestingUtility.exe:*)", - "Bash(./HEC.FDA.TestingUtility/bin/Debug/net9.0-windows/HEC.FDA.TestingUtility.exe:*)" + "Bash(./HEC.FDA.TestingUtility/bin/Debug/net9.0-windows/HEC.FDA.TestingUtility.exe:*)", + "Bash(dir /s \"C:\\\\Programs\\\\Source\\\\HEC-FDA\\\\HEC.FDA.TestingUtility\\\\*.cs\")", + "Bash(findstr:*)" ], "deny": [] } diff --git a/HEC.FDA.TestingUtility/Comparison/XmlResultComparer.cs b/HEC.FDA.TestingUtility/Comparison/XmlResultComparer.cs index 449a810ca..89a49557e 100644 --- a/HEC.FDA.TestingUtility/Comparison/XmlResultComparer.cs +++ b/HEC.FDA.TestingUtility/Comparison/XmlResultComparer.cs @@ -6,6 +6,9 @@ namespace HEC.FDA.TestingUtility.Comparison; public class XmlResultComparer { + private const double RelativeTolerance = 0.01; // 1% relative tolerance + private const double MinimumAbsoluteDifference = 1.0; // $1 minimum to consider a difference + private XElement? _baselineDoc; public void LoadBaseline(string baselinePath) @@ -212,11 +215,7 @@ public ComparisonResult CompareStageDamage(string elementName, List tolerance && absoluteDiff > 1.0) + if (!ValuesAreEqual(baselineMean, actualMean)) { result.Passed = false; result.Differences.Add(new Difference @@ -232,16 +231,17 @@ public ComparisonResult CompareStageDamage(string elementName, List tolerance && absoluteDiff > 1.0) // At least $1 difference + if (!ValuesAreEqual(baselineMean, actualMean)) { result.Differences.Add(new Difference { diff --git a/HEC.FDA.TestingUtility/Program.cs b/HEC.FDA.TestingUtility/Program.cs index 0fdffbd71..41f39b84e 100644 --- a/HEC.FDA.TestingUtility/Program.cs +++ b/HEC.FDA.TestingUtility/Program.cs @@ -1,4 +1,5 @@ using System.CommandLine; +using Geospatial.GDALAssist; using HEC.FDA.TestingUtility; using HEC.FDA.TestingUtility.Configuration; @@ -43,6 +44,9 @@ Console.WriteLine("========================"); Console.WriteLine(); + // Initialize GDAL for spatial operations + GDALSetup.InitializeMultiplatform(); + // Load configuration if (!configFile.Exists) { diff --git a/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs b/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs index 58dc3258d..4f0ac0960 100644 --- a/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs +++ b/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs @@ -100,22 +100,20 @@ private void WritePerformanceMetrics(string studyId, string scenarioName, Impact double meanAEP = iaResult.MeanAEP(thresholdId); double medianAEP = iaResult.MedianAEP(thresholdId); - // Assurance values (probability of not exceeding standard event) double assurance10 = iaResult.AssuranceOfEvent(thresholdId, 0.10); double assurance04 = iaResult.AssuranceOfEvent(thresholdId, 0.04); double assurance02 = iaResult.AssuranceOfEvent(thresholdId, 0.02); double assurance01 = iaResult.AssuranceOfEvent(thresholdId, 0.01); - // Long-term risk double ltRisk10 = iaResult.LongTermExceedanceProbability(thresholdId, 10); double ltRisk30 = iaResult.LongTermExceedanceProbability(thresholdId, 30); double ltRisk50 = iaResult.LongTermExceedanceProbability(thresholdId, 50); _scenarioPerformance.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(scenarioName)},{impactAreaId},{thresholdId},{meanAEP:F6},{medianAEP:F6},{assurance10:F4},{assurance04:F4},{assurance02:F4},{assurance01:F4},{ltRisk10:F4},{ltRisk30:F4},{ltRisk50:F4}"); } - catch + catch (Exception ex) { - // Performance metrics may not be available for this threshold + Console.WriteLine($" Warning: Could not extract performance metrics for threshold {threshold.ThresholdID}: {ex.Message}"); } } } diff --git a/HEC.FDA.TestingUtility/Services/AlternativeRunner.cs b/HEC.FDA.TestingUtility/Services/AlternativeRunner.cs index 5eb5de7a4..d236bf380 100644 --- a/HEC.FDA.TestingUtility/Services/AlternativeRunner.cs +++ b/HEC.FDA.TestingUtility/Services/AlternativeRunner.cs @@ -8,75 +8,75 @@ namespace HEC.FDA.TestingUtility.Services; -public class AlternativeRunner +public static class AlternativeRunner { - public AlternativeResults RunAlternative(string elementName, CancellationToken cancellationToken) + public static AlternativeResults RunAlternative(string elementName, CancellationToken cancellationToken) { - // Find element by name - AlternativeElement element = FindElement(elementName); + if (string.IsNullOrWhiteSpace(elementName)) + { + throw new ArgumentException("Alternative element name cannot be empty.", nameof(elementName)); + } + + AlternativeElement element = ScenarioRunner.FindElement(elementName); Console.WriteLine($" Running alternative '{elementName}'..."); - // Validate scenarios have results FdaValidationResult validation = element.RunPreComputeValidation(); if (!validation.IsValid) { throw new InvalidOperationException($"Alternative cannot compute: {validation.ErrorMessage}"); } - // Get study properties - StudyPropertiesElement props = BaseViewModel.StudyCache.GetStudyPropertiesElement(); + StudyPropertiesElement? props = BaseViewModel.StudyCache.GetStudyPropertiesElement(); + if (props == null) + { + throw new InvalidOperationException("Study properties not found."); + } + + IASElement? baseScenarioElement = element.BaseScenario?.GetElement(); + IASElement? futureScenarioElement = element.FutureScenario?.GetElement(); - // Get scenario results from the referenced scenarios - IASElement baseScenarioElement = element.BaseScenario.GetElement(); - IASElement futureScenarioElement = element.FutureScenario.GetElement(); + if (baseScenarioElement == null) + { + throw new InvalidOperationException("Base scenario element not found."); + } + + if (futureScenarioElement == null) + { + throw new InvalidOperationException("Future scenario element not found."); + } if (baseScenarioElement.Results == null) { - throw new InvalidOperationException($"Base scenario '{baseScenarioElement.Name}' has no computed results."); + throw new InvalidOperationException($"Base scenario '{baseScenarioElement.Name}' has no computed results. Run the scenario first."); } if (futureScenarioElement.Results == null) { - throw new InvalidOperationException($"Future scenario '{futureScenarioElement.Name}' has no computed results."); + throw new InvalidOperationException($"Future scenario '{futureScenarioElement.Name}' has no computed results. Run the scenario first."); } ScenarioResults baseResults = baseScenarioElement.Results; ScenarioResults futureResults = futureScenarioElement.Results; - Console.WriteLine($" Using base scenario: {baseScenarioElement.Name} (Year: {element.BaseScenario.Year})"); - Console.WriteLine($" Using future scenario: {futureScenarioElement.Name} (Year: {element.FutureScenario.Year})"); + int baseYear = element.BaseScenario?.Year ?? 0; + int futureYear = element.FutureScenario?.Year ?? 0; + + Console.WriteLine($" Using base scenario: {baseScenarioElement.Name} (Year: {baseYear})"); + Console.WriteLine($" Using future scenario: {futureScenarioElement.Name} (Year: {futureYear})"); Console.WriteLine($" Discount rate: {props.DiscountRate}, Period of analysis: {props.PeriodOfAnalysis}"); - // Compute AlternativeResults results = Alternative.AnnualizationCompute( props.DiscountRate, props.PeriodOfAnalysis, element.ID, baseResults, futureResults, - element.BaseScenario.Year, - element.FutureScenario.Year); + baseYear, + futureYear); Console.WriteLine($" Alternative computation complete."); return results; } - - private static T FindElement(string elementName) where T : ChildElement - { - var elements = BaseViewModel.StudyCache.GetChildElementsOfType(); - - var match = elements.FirstOrDefault(e => - e.Name.Equals(elementName, StringComparison.OrdinalIgnoreCase)); - - if (match == null) - { - string availableNames = string.Join(", ", elements.Select(e => e.Name)); - throw new InvalidOperationException( - $"Element '{elementName}' of type {typeof(T).Name} not found. Available: {availableNames}"); - } - - return match; - } } diff --git a/HEC.FDA.TestingUtility/Services/ScenarioRunner.cs b/HEC.FDA.TestingUtility/Services/ScenarioRunner.cs index aa6c52f91..5db4a3e0f 100644 --- a/HEC.FDA.TestingUtility/Services/ScenarioRunner.cs +++ b/HEC.FDA.TestingUtility/Services/ScenarioRunner.cs @@ -9,60 +9,80 @@ namespace HEC.FDA.TestingUtility.Services; -public class ScenarioRunner +public static class ScenarioRunner { - public ScenarioResults RunScenario(string elementName, CancellationToken cancellationToken) + public static ScenarioResults RunScenario(string elementName, CancellationToken cancellationToken) { - // Find element by name + if (string.IsNullOrWhiteSpace(elementName)) + { + throw new ArgumentException("Scenario element name cannot be empty.", nameof(elementName)); + } + IASElement element = FindElement(elementName); Console.WriteLine($" Running scenario '{elementName}'..."); - // Validate FdaValidationResult validation = element.CanCompute(); if (!validation.IsValid) { throw new InvalidOperationException($"Scenario cannot compute: {validation.ErrorMessage}"); } - // Create simulations (reuse existing static method) List sims = ComputeScenarioVM.CreateSimulations(element.SpecificIASElements); - if (sims.Count == 0) { throw new InvalidOperationException("No simulations could be created for this scenario."); } - // Build scenario Scenario scenario = new(sims); - // Get convergence criteria - ConvergenceCriteria cc = BaseViewModel.StudyCache.GetStudyPropertiesElement().GetStudyConvergenceCriteria(); + var studyProps = BaseViewModel.StudyCache.GetStudyPropertiesElement(); + if (studyProps == null) + { + throw new InvalidOperationException("Study properties not found. Cannot retrieve convergence criteria."); + } + ConvergenceCriteria cc = studyProps.GetStudyConvergenceCriteria(); Console.WriteLine($" Computing with convergence criteria: min={cc.MinIterations}, max={cc.MaxIterations}"); - // Compute ScenarioResults results = scenario.Compute(cc, cancellationToken, computeIsDeterministic: false); - Console.WriteLine($" Scenario computation complete."); return results; } - private static T FindElement(string elementName) where T : ChildElement + internal static T FindElement(string elementName) where T : ChildElement { var elements = BaseViewModel.StudyCache.GetChildElementsOfType(); - var match = elements.FirstOrDefault(e => e.Name.Equals(elementName, StringComparison.OrdinalIgnoreCase)); if (match == null) { - string availableNames = string.Join(", ", elements.Select(e => e.Name)); + string availableNames = elements.Count > 0 + ? string.Join(", ", elements.Select(e => e.Name)) + : "(none)"; throw new InvalidOperationException( $"Element '{elementName}' of type {typeof(T).Name} not found. Available: {availableNames}"); } return match; } + + internal static T FindElementById(int id) where T : ChildElement + { + var elements = BaseViewModel.StudyCache.GetChildElementsOfType(); + var match = elements.FirstOrDefault(e => e.ID == id); + + if (match == null) + { + string availableIds = elements.Count > 0 + ? string.Join(", ", elements.Select(e => $"{e.Name}(ID={e.ID})")) + : "(none)"; + throw new InvalidOperationException( + $"Element of type {typeof(T).Name} with ID {id} not found. Available: {availableIds}"); + } + + return match; + } } diff --git a/HEC.FDA.TestingUtility/Services/StageDamageRunner.cs b/HEC.FDA.TestingUtility/Services/StageDamageRunner.cs index 7526c8455..69cad01a8 100644 --- a/HEC.FDA.TestingUtility/Services/StageDamageRunner.cs +++ b/HEC.FDA.TestingUtility/Services/StageDamageRunner.cs @@ -9,32 +9,39 @@ namespace HEC.FDA.TestingUtility.Services; -public class StageDamageRunner +public static class StageDamageRunner { - public List RunStageDamage(string elementName) + public static List RunStageDamage(string elementName) { + if (string.IsNullOrWhiteSpace(elementName)) + { + throw new ArgumentException("Stage damage element name cannot be empty.", nameof(elementName)); + } + Console.WriteLine($" Running stage damage compute for '{elementName}'..."); - // Find element by name - AggregatedStageDamageElement element = FindElement(elementName); + AggregatedStageDamageElement element = ScenarioRunner.FindElement(elementName); - // Check if this is a manual stage damage (no compute needed) if (element.IsManual) { Console.WriteLine($" Stage damage '{elementName}' is manual - returning existing curves."); return ConvertCurvesToUPD(element.Curves); } - // Get the referenced elements needed for compute - ImpactAreaElement impactAreaElement = GetImpactAreaElement(); - HydraulicElement hydraulicElement = FindElementById(element.SelectedWSE); - InventoryElement inventoryElement = FindElementById(element.SelectedStructures); + var impactAreaElements = BaseViewModel.StudyCache.GetChildElementsOfType(); + if (impactAreaElements.Count == 0) + { + throw new InvalidOperationException("No impact area element found in study."); + } + ImpactAreaElement impactAreaElement = impactAreaElements[0]; + + HydraulicElement hydraulicElement = ScenarioRunner.FindElementById(element.SelectedWSE); + InventoryElement inventoryElement = ScenarioRunner.FindElementById(element.SelectedStructures); Console.WriteLine($" Using hydraulics: {hydraulicElement.Name}"); Console.WriteLine($" Using inventory: {inventoryElement.Name}"); Console.WriteLine($" Analysis year: {element.AnalysisYear}"); - // Create the configuration StageDamageConfiguration config = new( impactAreaElement, hydraulicElement, @@ -42,19 +49,21 @@ public List RunStageDamage(string elementName) element.ImpactAreaFrequencyRows, element.AnalysisYear); - // Validate the configuration FdaValidationResult validation = config.ValidateConfiguration(); if (!validation.IsValid) { throw new InvalidOperationException($"Stage damage configuration is invalid: {validation.ErrorMessage}"); } - // Create the stage damages and compute List impactAreaStageDamages = config.CreateStageDamages(); + if (impactAreaStageDamages.Count == 0) + { + throw new InvalidOperationException("No impact area stage damages could be created."); + } + ScenarioStageDamage scenarioStageDamage = new(impactAreaStageDamages); - // Validate structure count - int totalStructureCount = impactAreaStageDamages.Sum(sd => sd.Inventory.Structures.Count); + int totalStructureCount = impactAreaStageDamages.Sum(sd => sd.Inventory?.Structures?.Count ?? 0); if (totalStructureCount == 0) { throw new InvalidOperationException("No structures found in any impact area for stage damage compute."); @@ -62,8 +71,7 @@ public List RunStageDamage(string elementName) Console.WriteLine($" Computing stage damage with {totalStructureCount} structures..."); - // Run the compute - (List stageDamageFunctions, List _) = scenarioStageDamage.Compute(); + (List stageDamageFunctions, _) = scenarioStageDamage.Compute(); Console.WriteLine($" Stage damage computation complete. Generated {stageDamageFunctions.Count} curves."); @@ -73,54 +81,14 @@ public List RunStageDamage(string elementName) private static List ConvertCurvesToUPD(List curves) { List result = new(); + if (curves == null) return result; + foreach (var curve in curves) { + if (curve?.ComputeComponent == null) continue; UncertainPairedData upd = curve.ComputeComponent.SelectedItemToPairedData(); result.Add(upd); } return result; } - - private static ImpactAreaElement GetImpactAreaElement() - { - var impactAreaElements = BaseViewModel.StudyCache.GetChildElementsOfType(); - if (impactAreaElements.Count == 0) - { - throw new InvalidOperationException("No impact area element found in study."); - } - return impactAreaElements[0]; - } - - private static T FindElement(string elementName) where T : ChildElement - { - var elements = BaseViewModel.StudyCache.GetChildElementsOfType(); - - var match = elements.FirstOrDefault(e => - e.Name.Equals(elementName, StringComparison.OrdinalIgnoreCase)); - - if (match == null) - { - string availableNames = string.Join(", ", elements.Select(e => e.Name)); - throw new InvalidOperationException( - $"Element '{elementName}' of type {typeof(T).Name} not found. Available: {availableNames}"); - } - - return match; - } - - private static T FindElementById(int id) where T : ChildElement - { - var elements = BaseViewModel.StudyCache.GetChildElementsOfType(); - - var match = elements.FirstOrDefault(e => e.ID == id); - - if (match == null) - { - string availableIds = string.Join(", ", elements.Select(e => $"{e.Name}({e.ID})")); - throw new InvalidOperationException( - $"Element of type {typeof(T).Name} with ID {id} not found. Available: {availableIds}"); - } - - return match; - } } diff --git a/HEC.FDA.TestingUtility/TestRunner.cs b/HEC.FDA.TestingUtility/TestRunner.cs index b7299d7d9..47ddf0022 100644 --- a/HEC.FDA.TestingUtility/TestRunner.cs +++ b/HEC.FDA.TestingUtility/TestRunner.cs @@ -23,9 +23,6 @@ public class TestRunner private readonly XmlResultComparer _comparer = new(); private readonly StudyBaselineWriter _baselineWriter = new(); - private readonly ScenarioRunner _scenarioRunner = new(); - private readonly AlternativeRunner _alternativeRunner = new(); - private readonly StageDamageRunner _stageDamageRunner = new(); private readonly CsvReportFactory _csvReportFactory = new(); public TestRunner(TestConfiguration config, string outputDir, bool verbose, string[]? studyFilter) @@ -111,21 +108,21 @@ public async Task RunAsync() switch (compute.Type.ToLowerInvariant()) { case "scenario": - var scenarioResults = _scenarioRunner.RunScenario(compute.ElementName, _cts.Token); + var scenarioResults = ScenarioRunner.RunScenario(compute.ElementName, _cts.Token); _baselineWriter.AddScenarioResults(computedBaseline, compute.ElementName, scenarioResults); _csvReportFactory.AddScenarioResults(study.StudyId, compute.ElementName, scenarioResults); result = _comparer.CompareScenarioResults(compute.ElementName, scenarioResults); break; case "alternative": - var altResults = _alternativeRunner.RunAlternative(compute.ElementName, _cts.Token); + var altResults = AlternativeRunner.RunAlternative(compute.ElementName, _cts.Token); _baselineWriter.AddAlternativeResults(computedBaseline, compute.ElementName, altResults); _csvReportFactory.AddAlternativeResults(study.StudyId, compute.ElementName, altResults); result = _comparer.CompareAlternativeResults(compute.ElementName, altResults); break; case "stagedamage": - List sdCurves = _stageDamageRunner.RunStageDamage(compute.ElementName); + List sdCurves = StageDamageRunner.RunStageDamage(compute.ElementName); _baselineWriter.AddStageDamage(computedBaseline, compute.ElementName, sdCurves); _csvReportFactory.AddStageDamageSummary(study.StudyId, compute.ElementName, sdCurves); result = _comparer.CompareStageDamage(compute.ElementName, sdCurves); From c1074d9078102e5fd103a6dba5269512000393df Mon Sep 17 00:00:00 2001 From: Brennan Beam <64556723+Brennan1994@users.noreply.github.com> Date: Tue, 23 Dec 2025 15:42:41 -0800 Subject: [PATCH 06/13] add alternative comparison reports --- .../Comparison/StudyBaselineWriter.cs | 64 ++++++++- .../Comparison/XmlResultComparer.cs | 94 +++++++++++++ .../Configuration/TestConfiguration.cs | 3 + .../Reporting/CsvReportFactory.cs | 82 +++++++++++ .../Services/AlternativeComparisonRunner.cs | 127 ++++++++++++++++++ HEC.FDA.TestingUtility/TestRunner.cs | 64 +++++++-- 6 files changed, 420 insertions(+), 14 deletions(-) create mode 100644 HEC.FDA.TestingUtility/Services/AlternativeComparisonRunner.cs diff --git a/HEC.FDA.TestingUtility/Comparison/StudyBaselineWriter.cs b/HEC.FDA.TestingUtility/Comparison/StudyBaselineWriter.cs index 4556947b3..5ec9fcf41 100644 --- a/HEC.FDA.TestingUtility/Comparison/StudyBaselineWriter.cs +++ b/HEC.FDA.TestingUtility/Comparison/StudyBaselineWriter.cs @@ -4,9 +4,9 @@ namespace HEC.FDA.TestingUtility.Comparison; -public class StudyBaselineWriter +public static class StudyBaselineWriter { - public XElement CreateStudyBaseline(string studyId, string studyName) + public static XElement CreateStudyBaseline(string studyId, string studyName) { return new XElement("StudyBaseline", new XAttribute("studyId", studyId), @@ -14,7 +14,7 @@ public XElement CreateStudyBaseline(string studyId, string studyName) new XAttribute("createdDate", DateTime.Now.ToString("yyyy-MM-dd"))); } - public void AddScenarioResults(XElement baseline, string name, ScenarioResults results) + public static void AddScenarioResults(XElement baseline, string name, ScenarioResults results) { var wrapper = new XElement("ScenarioResults", new XAttribute("name", name), @@ -22,7 +22,7 @@ public void AddScenarioResults(XElement baseline, string name, ScenarioResults r baseline.Add(wrapper); } - public void AddAlternativeResults(XElement baseline, string name, AlternativeResults results) + public static void AddAlternativeResults(XElement baseline, string name, AlternativeResults results) { var wrapper = new XElement("AlternativeResults", new XAttribute("name", name), @@ -31,7 +31,7 @@ public void AddAlternativeResults(XElement baseline, string name, AlternativeRes baseline.Add(wrapper); } - public void AddStageDamage(XElement baseline, string name, List curves) + public static void AddStageDamage(XElement baseline, string name, List curves) { var curvesElement = new XElement("Curves"); foreach (var curve in curves) @@ -46,7 +46,59 @@ public void AddStageDamage(XElement baseline, string name, List withProjectAlternatives) + { + if (results == null) return; + + var wrapper = new XElement("AlternativeComparisonReport", + new XAttribute("name", name)); + + var impactAreaIds = results.GetImpactAreaIDs(); + var damageCategories = results.GetDamageCategories(); + var assetCategories = results.GetAssetCategories(); + + foreach (var (altId, altName) in withProjectAlternatives) + { + var altElement = new XElement("WithProjectAlternative", + new XAttribute("id", altId), + new XAttribute("name", altName)); + + foreach (int impactAreaId in impactAreaIds) + { + var iaElement = new XElement("ImpactArea", + new XAttribute("id", impactAreaId), + new XAttribute("eqadReduced", results.SampleMeanEqadReduced(altId, impactAreaId)), + new XAttribute("baseEadReduced", results.SampleMeanBaseYearEADReduced(altId, impactAreaId)), + new XAttribute("futureEadReduced", results.SampleMeanFutureYearEADReduced(altId, impactAreaId))); + + // Add category breakdowns + foreach (string damCat in damageCategories) + { + foreach (string assetCat in assetCategories) + { + double eqadReduced = results.SampleMeanEqadReduced(altId, impactAreaId, damCat, assetCat); + if (eqadReduced != 0) + { + iaElement.Add(new XElement("Category", + new XAttribute("damageCategory", damCat), + new XAttribute("assetCategory", assetCat), + new XAttribute("eqadReduced", eqadReduced), + new XAttribute("baseEadReduced", results.SampleMeanBaseYearEADReduced(altId, impactAreaId, damCat, assetCat)), + new XAttribute("futureEadReduced", results.SampleMeanFutureYearEADReduced(altId, impactAreaId, damCat, assetCat)))); + } + } + } + + altElement.Add(iaElement); + } + + wrapper.Add(altElement); + } + + baseline.Add(wrapper); + } + + public static void Save(XElement baseline, string path) { string? directory = Path.GetDirectoryName(path); if (!string.IsNullOrEmpty(directory) && !Directory.Exists(directory)) diff --git a/HEC.FDA.TestingUtility/Comparison/XmlResultComparer.cs b/HEC.FDA.TestingUtility/Comparison/XmlResultComparer.cs index 89a49557e..d6654e62e 100644 --- a/HEC.FDA.TestingUtility/Comparison/XmlResultComparer.cs +++ b/HEC.FDA.TestingUtility/Comparison/XmlResultComparer.cs @@ -264,4 +264,98 @@ private static void GenerateScenarioDiff(ScenarioResults baseline, ScenarioResul } } } + + public ComparisonResult CompareAlternativeComparisonResults(string elementName, AlternativeComparisonReportResults actual, List<(int altId, string altName)> withProjectAlternatives) + { + var result = new ComparisonResult { ElementName = elementName, ElementType = "AlternativeComparisonReport" }; + + if (_baselineDoc == null) + { + result.Passed = false; + result.ErrorMessage = "Baseline not loaded"; + return result; + } + + var baselineElement = _baselineDoc.Elements("AlternativeComparisonReport") + .FirstOrDefault(e => e.Attribute("name")?.Value == elementName); + + if (baselineElement == null) + { + result.Passed = false; + result.ErrorMessage = $"Baseline not found for AlternativeComparisonReport '{elementName}'"; + return result; + } + + result.Passed = true; + + foreach (var (altId, altName) in withProjectAlternatives) + { + var baselineAltElement = baselineElement.Elements("WithProjectAlternative") + .FirstOrDefault(e => e.Attribute("id")?.Value == altId.ToString()); + + if (baselineAltElement == null) + { + result.Passed = false; + result.Differences.Add(new Difference + { + Metric = $"WithProjectAlternative[{altName}]", + ExpectedDescription = "present in baseline", + ActualDescription = "missing" + }); + continue; + } + + foreach (var baselineIaElement in baselineAltElement.Elements("ImpactArea")) + { + int impactAreaId = int.Parse(baselineIaElement.Attribute("id")?.Value ?? "0"); + + // Compare EqAD Reduced + double baselineEqadReduced = double.Parse(baselineIaElement.Attribute("eqadReduced")?.Value ?? "0"); + double actualEqadReduced = actual.SampleMeanEqadReduced(altId, impactAreaId); + + if (!ValuesAreEqual(baselineEqadReduced, actualEqadReduced)) + { + result.Passed = false; + result.Differences.Add(new Difference + { + Metric = $"EqadReduced[Alt={altName},IA={impactAreaId}]", + Expected = baselineEqadReduced, + Actual = actualEqadReduced + }); + } + + // Compare Base EAD Reduced + double baselineBaseEadReduced = double.Parse(baselineIaElement.Attribute("baseEadReduced")?.Value ?? "0"); + double actualBaseEadReduced = actual.SampleMeanBaseYearEADReduced(altId, impactAreaId); + + if (!ValuesAreEqual(baselineBaseEadReduced, actualBaseEadReduced)) + { + result.Passed = false; + result.Differences.Add(new Difference + { + Metric = $"BaseEadReduced[Alt={altName},IA={impactAreaId}]", + Expected = baselineBaseEadReduced, + Actual = actualBaseEadReduced + }); + } + + // Compare Future EAD Reduced + double baselineFutureEadReduced = double.Parse(baselineIaElement.Attribute("futureEadReduced")?.Value ?? "0"); + double actualFutureEadReduced = actual.SampleMeanFutureYearEADReduced(altId, impactAreaId); + + if (!ValuesAreEqual(baselineFutureEadReduced, actualFutureEadReduced)) + { + result.Passed = false; + result.Differences.Add(new Difference + { + Metric = $"FutureEadReduced[Alt={altName},IA={impactAreaId}]", + Expected = baselineFutureEadReduced, + Actual = actualFutureEadReduced + }); + } + } + } + + return result; + } } diff --git a/HEC.FDA.TestingUtility/Configuration/TestConfiguration.cs b/HEC.FDA.TestingUtility/Configuration/TestConfiguration.cs index 47a9bd040..1e4865c55 100644 --- a/HEC.FDA.TestingUtility/Configuration/TestConfiguration.cs +++ b/HEC.FDA.TestingUtility/Configuration/TestConfiguration.cs @@ -54,6 +54,9 @@ public class StudyConfiguration [JsonPropertyName("runAllStageDamage")] public bool RunAllStageDamage { get; set; } = false; + [JsonPropertyName("runAllAlternativeComparisons")] + public bool RunAllAlternativeComparisons { get; set; } = false; + [JsonPropertyName("computations")] public List Computations { get; set; } = new(); } diff --git a/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs b/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs index 4f0ac0960..be7ca7629 100644 --- a/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs +++ b/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs @@ -16,6 +16,8 @@ public class CsvReportFactory private readonly StringBuilder _alternativeResults = new(); private readonly StringBuilder _alternativeDamageByCategory = new(); private readonly StringBuilder _stageDamageSummary = new(); + private readonly StringBuilder _altComparisonSummary = new(); + private readonly StringBuilder _altComparisonByCategory = new(); public CsvReportFactory() { @@ -41,6 +43,12 @@ private void WriteHeaders() // Stage Damage Summary header _stageDamageSummary.AppendLine("Study ID,Element Name,Impact Area ID,Impact Area Name,Damage Category,Asset Category,Point Count,Min Stage,Max Stage"); + + // Alternative Comparison Summary header + _altComparisonSummary.AppendLine("Study ID,Report Name,With Project Alt,Impact Area ID,Without Proj EqAD,With Proj EqAD,EqAD Reduced,EqAD Reduced 25th Pct,EqAD Reduced 50th Pct,EqAD Reduced 75th Pct,Without Proj Base EAD,With Proj Base EAD,Base EAD Reduced,Without Proj Future EAD,With Proj Future EAD,Future EAD Reduced"); + + // Alternative Comparison by Category header + _altComparisonByCategory.AppendLine("Study ID,Report Name,With Project Alt,Impact Area ID,Damage Category,Asset Category,EqAD Reduced,Base Year EAD Reduced,Future Year EAD Reduced"); } /// @@ -199,6 +207,72 @@ public void AddStageDamageSummary(string studyId, string elementName, List + /// Adds alternative comparison report results to the report. + /// + public void AddAlternativeComparisonResults(string studyId, string reportName, AlternativeComparisonReportResults results, List<(int altId, string altName)> withProjectAlternatives) + { + if (results == null) return; + + try + { + var impactAreaIds = results.GetImpactAreaIDs(); + var damageCategories = results.GetDamageCategories(); + var assetCategories = results.GetAssetCategories(); + + foreach (var (altId, altName) in withProjectAlternatives) + { + // Write aggregated summary per impact area + foreach (int impactAreaId in impactAreaIds) + { + // EqAD values + double withoutProjEqad = results.SampleMeanWithoutProjectEqad(impactAreaId); + double withProjEqad = results.SampleMeanWithProjectEqad(altId, impactAreaId); + double eqadReduced = results.SampleMeanEqadReduced(altId, impactAreaId); + double eqadReduced25 = results.EqadReducedExceededWithProbabilityQ(0.75, altId, impactAreaId); + double eqadReduced50 = results.EqadReducedExceededWithProbabilityQ(0.50, altId, impactAreaId); + double eqadReduced75 = results.EqadReducedExceededWithProbabilityQ(0.25, altId, impactAreaId); + + // Base year EAD values + double withoutProjBaseEad = results.SampleMeanWithoutProjectBaseYearEAD(impactAreaId); + double withProjBaseEad = results.SampleMeanWithProjectBaseYearEAD(altId, impactAreaId); + double baseEadReduced = results.SampleMeanBaseYearEADReduced(altId, impactAreaId); + + // Future year EAD values + double withoutProjFutureEad = results.SampleMeanWithoutProjectFutureYearEAD(impactAreaId); + double withProjFutureEad = results.SampleMeanWithProjectFutureYearEAD(altId, impactAreaId); + double futureEadReduced = results.SampleMeanFutureYearEADReduced(altId, impactAreaId); + + _altComparisonSummary.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(reportName)},{EscapeCsv(altName)},{impactAreaId},{withoutProjEqad:F2},{withProjEqad:F2},{eqadReduced:F2},{eqadReduced25:F2},{eqadReduced50:F2},{eqadReduced75:F2},{withoutProjBaseEad:F2},{withProjBaseEad:F2},{baseEadReduced:F2},{withoutProjFutureEad:F2},{withProjFutureEad:F2},{futureEadReduced:F2}"); + } + + // Write by category breakdown + foreach (int impactAreaId in impactAreaIds) + { + foreach (string damCat in damageCategories) + { + foreach (string assetCat in assetCategories) + { + double eqadReduced = results.SampleMeanEqadReduced(altId, impactAreaId, damCat, assetCat); + double baseEadReduced = results.SampleMeanBaseYearEADReduced(altId, impactAreaId, damCat, assetCat); + double futureEadReduced = results.SampleMeanFutureYearEADReduced(altId, impactAreaId, damCat, assetCat); + + // Only write non-zero values + if (eqadReduced != 0 || baseEadReduced != 0 || futureEadReduced != 0) + { + _altComparisonByCategory.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(reportName)},{EscapeCsv(altName)},{impactAreaId},{EscapeCsv(damCat)},{EscapeCsv(assetCat)},{eqadReduced:F2},{baseEadReduced:F2},{futureEadReduced:F2}"); + } + } + } + } + } + } + catch (Exception ex) + { + Console.WriteLine($" Warning: Error extracting alternative comparison results for CSV: {ex.Message}"); + } + } + /// /// Saves the comprehensive report to a CSV file. /// @@ -232,6 +306,14 @@ public void SaveReport(string outputPath) report.AppendLine("=== STAGE DAMAGE SUMMARY ==="); report.Append(_stageDamageSummary); + report.AppendLine(); + + report.AppendLine("=== ALTERNATIVE COMPARISON SUMMARY ==="); + report.Append(_altComparisonSummary); + report.AppendLine(); + + report.AppendLine("=== ALTERNATIVE COMPARISON BY CATEGORY ==="); + report.Append(_altComparisonByCategory); File.WriteAllText(outputPath, report.ToString()); Console.WriteLine($"CSV report saved to: {outputPath}"); diff --git a/HEC.FDA.TestingUtility/Services/AlternativeComparisonRunner.cs b/HEC.FDA.TestingUtility/Services/AlternativeComparisonRunner.cs new file mode 100644 index 000000000..7cae232a4 --- /dev/null +++ b/HEC.FDA.TestingUtility/Services/AlternativeComparisonRunner.cs @@ -0,0 +1,127 @@ +using HEC.FDA.Model.alternatives; +using HEC.FDA.Model.alternativeComparisonReport; +using HEC.FDA.Model.metrics; +using HEC.FDA.ViewModel; +using HEC.FDA.ViewModel.Alternatives; +using HEC.FDA.ViewModel.AlternativeComparisonReport; +using HEC.FDA.ViewModel.Study; +using HEC.FDA.ViewModel.Utilities; + +namespace HEC.FDA.TestingUtility.Services; + +public static class AlternativeComparisonRunner +{ + public static AlternativeComparisonReportResults RunAlternativeComparison(string elementName, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(elementName)) + { + throw new ArgumentException("Alternative comparison report element name cannot be empty.", nameof(elementName)); + } + + Console.WriteLine($" Running alternative comparison report '{elementName}'..."); + + var element = ScenarioRunner.FindElement(elementName); + + // Get study properties + StudyPropertiesElement? props = BaseViewModel.StudyCache.GetStudyPropertiesElement(); + if (props == null) + { + throw new InvalidOperationException("Study properties not found."); + } + + // Get the without-project alternative + AlternativeElement? withoutProjAlt = GetAlternativeById(element.WithoutProjAltID); + if (withoutProjAlt == null) + { + throw new InvalidOperationException($"Without-project alternative (ID={element.WithoutProjAltID}) not found."); + } + + // Get all with-project alternatives + List withProjAlts = new(); + foreach (int altId in element.WithProjAltIDs) + { + var alt = GetAlternativeById(altId); + if (alt == null) + { + throw new InvalidOperationException($"With-project alternative (ID={altId}) not found."); + } + withProjAlts.Add(alt); + } + + if (withProjAlts.Count == 0) + { + throw new InvalidOperationException("No with-project alternatives found for comparison."); + } + + Console.WriteLine($" Without-project alternative: {withoutProjAlt.Name}"); + Console.WriteLine($" With-project alternatives: {string.Join(", ", withProjAlts.Select(a => a.Name))}"); + + // Compute without-project alternative results + Console.WriteLine($" Computing without-project alternative '{withoutProjAlt.Name}'..."); + AlternativeResults withoutProjResults = ComputeAlternativeResults(withoutProjAlt, props, cancellationToken); + + // Compute each with-project alternative results + List withProjResults = new(); + foreach (var withProjAlt in withProjAlts) + { + Console.WriteLine($" Computing with-project alternative '{withProjAlt.Name}'..."); + var results = ComputeAlternativeResults(withProjAlt, props, cancellationToken); + withProjResults.Add(results); + } + + // Compute the comparison report + Console.WriteLine($" Computing alternative comparison report..."); + var comparisonResults = AlternativeComparisonReport.ComputeAlternativeComparisonReport( + withoutProjResults, + withProjResults); + + if (comparisonResults == null) + { + throw new InvalidOperationException("Alternative comparison report computation failed."); + } + + Console.WriteLine($" Alternative comparison report complete."); + + return comparisonResults; + } + + private static AlternativeElement? GetAlternativeById(int id) + { + var alternatives = BaseViewModel.StudyCache.GetChildElementsOfType(); + return alternatives.FirstOrDefault(a => a.ID == id); + } + + private static AlternativeResults ComputeAlternativeResults(AlternativeElement element, StudyPropertiesElement props, CancellationToken cancellationToken) + { + FdaValidationResult validation = element.RunPreComputeValidation(); + if (!validation.IsValid) + { + throw new InvalidOperationException($"Alternative '{element.Name}' cannot compute: {validation.ErrorMessage}"); + } + + var baseScenarioElement = element.BaseScenario?.GetElement(); + var futureScenarioElement = element.FutureScenario?.GetElement(); + + if (baseScenarioElement?.Results == null) + { + throw new InvalidOperationException($"Base scenario for alternative '{element.Name}' has no computed results."); + } + + if (futureScenarioElement?.Results == null) + { + throw new InvalidOperationException($"Future scenario for alternative '{element.Name}' has no computed results."); + } + + int baseYear = element.BaseScenario?.Year ?? 0; + int futureYear = element.FutureScenario?.Year ?? 0; + + return Alternative.AnnualizationCompute( + props.DiscountRate, + props.PeriodOfAnalysis, + element.ID, + baseScenarioElement.Results, + futureScenarioElement.Results, + baseYear, + futureYear); + } +} diff --git a/HEC.FDA.TestingUtility/TestRunner.cs b/HEC.FDA.TestingUtility/TestRunner.cs index 47ddf0022..8498b27a4 100644 --- a/HEC.FDA.TestingUtility/TestRunner.cs +++ b/HEC.FDA.TestingUtility/TestRunner.cs @@ -9,6 +9,7 @@ using HEC.FDA.ViewModel; using HEC.FDA.ViewModel.AggregatedStageDamage; using HEC.FDA.ViewModel.Alternatives; +using HEC.FDA.ViewModel.AlternativeComparisonReport; using HEC.FDA.ViewModel.ImpactAreaScenario; namespace HEC.FDA.TestingUtility; @@ -22,7 +23,6 @@ public class TestRunner private readonly CancellationTokenSource _cts; private readonly XmlResultComparer _comparer = new(); - private readonly StudyBaselineWriter _baselineWriter = new(); private readonly CsvReportFactory _csvReportFactory = new(); public TestRunner(TestConfiguration config, string outputDir, bool verbose, string[]? studyFilter) @@ -90,7 +90,7 @@ public async Task RunAsync() } // Create computed results document for debugging - var computedBaseline = _baselineWriter.CreateStudyBaseline(study.StudyId, study.StudyName); + var computedBaseline = StudyBaselineWriter.CreateStudyBaseline(study.StudyId, study.StudyName); // Build computation list (from config or auto-discover) var computations = BuildComputationList(study); @@ -109,25 +109,32 @@ public async Task RunAsync() { case "scenario": var scenarioResults = ScenarioRunner.RunScenario(compute.ElementName, _cts.Token); - _baselineWriter.AddScenarioResults(computedBaseline, compute.ElementName, scenarioResults); + StudyBaselineWriter.AddScenarioResults(computedBaseline, compute.ElementName, scenarioResults); _csvReportFactory.AddScenarioResults(study.StudyId, compute.ElementName, scenarioResults); result = _comparer.CompareScenarioResults(compute.ElementName, scenarioResults); break; case "alternative": var altResults = AlternativeRunner.RunAlternative(compute.ElementName, _cts.Token); - _baselineWriter.AddAlternativeResults(computedBaseline, compute.ElementName, altResults); + StudyBaselineWriter.AddAlternativeResults(computedBaseline, compute.ElementName, altResults); _csvReportFactory.AddAlternativeResults(study.StudyId, compute.ElementName, altResults); result = _comparer.CompareAlternativeResults(compute.ElementName, altResults); break; case "stagedamage": List sdCurves = StageDamageRunner.RunStageDamage(compute.ElementName); - _baselineWriter.AddStageDamage(computedBaseline, compute.ElementName, sdCurves); + StudyBaselineWriter.AddStageDamage(computedBaseline, compute.ElementName, sdCurves); _csvReportFactory.AddStageDamageSummary(study.StudyId, compute.ElementName, sdCurves); result = _comparer.CompareStageDamage(compute.ElementName, sdCurves); break; + case "alternativecomparison": + var (compResults, withProjAlts) = RunAlternativeComparisonWithMetadata(compute.ElementName, _cts.Token); + StudyBaselineWriter.AddAlternativeComparisonResults(computedBaseline, compute.ElementName, compResults, withProjAlts); + _csvReportFactory.AddAlternativeComparisonResults(study.StudyId, compute.ElementName, compResults, withProjAlts); + result = _comparer.CompareAlternativeComparisonResults(compute.ElementName, compResults, withProjAlts); + break; + default: Console.WriteLine($" SKIP: Unknown compute type '{compute.Type}'"); continue; @@ -237,7 +244,7 @@ private static string FormatDuration(TimeSpan duration) } } - private List BuildComputationList(StudyConfiguration study) + private static List BuildComputationList(StudyConfiguration study) { var computations = new List(study.Computations); @@ -298,10 +305,51 @@ private List BuildComputationList(StudyConfiguration study } } + // Auto-discover alternative comparison reports (depend on alternatives, so run last) + if (study.RunAllAlternativeComparisons) + { + var altCompReports = BaseViewModel.StudyCache.GetChildElementsOfType(); + foreach (var report in altCompReports) + { + if (!computations.Any(c => c.Type.Equals("alternativecomparison", StringComparison.OrdinalIgnoreCase) + && c.ElementName.Equals(report.Name, StringComparison.OrdinalIgnoreCase))) + { + computations.Add(new ComputeConfiguration + { + Type = "alternativecomparison", + ElementName = report.Name + }); + Console.WriteLine($" Auto-discovered alternative comparison: {report.Name}"); + } + } + } + return computations; } - private string GetBaselinePath(StudyConfiguration study) + private static (AlternativeComparisonReportResults results, List<(int altId, string altName)> withProjectAlternatives) RunAlternativeComparisonWithMetadata(string elementName, CancellationToken cancellationToken) + { + // Get the element to extract the with-project alternative IDs + var element = ScenarioRunner.FindElement(elementName); + + // Build the list of with-project alternatives with names + var withProjectAlternatives = new List<(int altId, string altName)>(); + var allAlternatives = BaseViewModel.StudyCache.GetChildElementsOfType(); + + foreach (int altId in element.WithProjAltIDs) + { + var alt = allAlternatives.FirstOrDefault(a => a.ID == altId); + string altName = alt?.Name ?? $"Alternative_{altId}"; + withProjectAlternatives.Add((altId, altName)); + } + + // Run the computation + var results = AlternativeComparisonRunner.RunAlternativeComparison(elementName, cancellationToken); + + return (results, withProjectAlternatives); + } + + private static string GetBaselinePath(StudyConfiguration study) { // Single baseline file per study return Path.Combine(study.BaselineDirectory, $"{study.StudyId}_baseline.xml"); @@ -311,7 +359,7 @@ private void SaveComputedResults(XElement computedBaseline, StudyConfiguration s { // Save computed results in same format as baseline for easy comparison string outputPath = Path.Combine(_outputDir, $"{study.StudyId}_computed.xml"); - _baselineWriter.Save(computedBaseline, outputPath); + StudyBaselineWriter.Save(computedBaseline, outputPath); Console.WriteLine($" Computed results saved to: {outputPath}"); } From 4c0be23ca0c6dcd52042555ee2ce7cea9d895dde Mon Sep 17 00:00:00 2001 From: Brennan Beam <64556723+Brennan1994@users.noreply.github.com> Date: Tue, 23 Dec 2025 15:52:38 -0800 Subject: [PATCH 07/13] formatting --- .../Comparison/StudyBaselineWriter.cs | 24 +++++++++---------- .../Comparison/XmlResultComparer.cs | 8 +++---- HEC.FDA.TestingUtility/Program.cs | 12 +++++----- .../Reporting/CsvReportFactory.cs | 2 +- .../Services/AlternativeComparisonRunner.cs | 8 +++---- HEC.FDA.TestingUtility/TestRunner.cs | 10 ++++---- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/HEC.FDA.TestingUtility/Comparison/StudyBaselineWriter.cs b/HEC.FDA.TestingUtility/Comparison/StudyBaselineWriter.cs index 5ec9fcf41..681353896 100644 --- a/HEC.FDA.TestingUtility/Comparison/StudyBaselineWriter.cs +++ b/HEC.FDA.TestingUtility/Comparison/StudyBaselineWriter.cs @@ -16,7 +16,7 @@ public static XElement CreateStudyBaseline(string studyId, string studyName) public static void AddScenarioResults(XElement baseline, string name, ScenarioResults results) { - var wrapper = new XElement("ScenarioResults", + XElement wrapper = new("ScenarioResults", new XAttribute("name", name), results.WriteToXML()); baseline.Add(wrapper); @@ -24,7 +24,7 @@ public static void AddScenarioResults(XElement baseline, string name, ScenarioRe public static void AddAlternativeResults(XElement baseline, string name, AlternativeResults results) { - var wrapper = new XElement("AlternativeResults", + XElement wrapper = new("AlternativeResults", new XAttribute("name", name), new XElement("BaseYearResults", results.BaseYearScenarioResults.WriteToXML()), new XElement("FutureYearResults", results.FutureYearScenarioResults.WriteToXML())); @@ -33,13 +33,13 @@ public static void AddAlternativeResults(XElement baseline, string name, Alterna public static void AddStageDamage(XElement baseline, string name, List curves) { - var curvesElement = new XElement("Curves"); - foreach (var curve in curves) + XElement curvesElement = new("Curves"); + foreach (UncertainPairedData curve in curves) { curvesElement.Add(curve.WriteToXML()); } - var wrapper = new XElement("StageDamage", + XElement wrapper = new("StageDamage", new XAttribute("name", name), new XAttribute("curveCount", curves.Count), curvesElement); @@ -50,22 +50,22 @@ public static void AddAlternativeComparisonResults(XElement baseline, string nam { if (results == null) return; - var wrapper = new XElement("AlternativeComparisonReport", + XElement wrapper = new("AlternativeComparisonReport", new XAttribute("name", name)); - var impactAreaIds = results.GetImpactAreaIDs(); - var damageCategories = results.GetDamageCategories(); - var assetCategories = results.GetAssetCategories(); + List impactAreaIds = results.GetImpactAreaIDs(); + List damageCategories = results.GetDamageCategories(); + List assetCategories = results.GetAssetCategories(); - foreach (var (altId, altName) in withProjectAlternatives) + foreach ((int altId, string altName) in withProjectAlternatives) { - var altElement = new XElement("WithProjectAlternative", + XElement altElement = new("WithProjectAlternative", new XAttribute("id", altId), new XAttribute("name", altName)); foreach (int impactAreaId in impactAreaIds) { - var iaElement = new XElement("ImpactArea", + XElement iaElement = new("ImpactArea", new XAttribute("id", impactAreaId), new XAttribute("eqadReduced", results.SampleMeanEqadReduced(altId, impactAreaId)), new XAttribute("baseEadReduced", results.SampleMeanBaseYearEADReduced(altId, impactAreaId)), diff --git a/HEC.FDA.TestingUtility/Comparison/XmlResultComparer.cs b/HEC.FDA.TestingUtility/Comparison/XmlResultComparer.cs index d6654e62e..0350c4532 100644 --- a/HEC.FDA.TestingUtility/Comparison/XmlResultComparer.cs +++ b/HEC.FDA.TestingUtility/Comparison/XmlResultComparer.cs @@ -22,7 +22,7 @@ public void LoadBaseline(string baselinePath) public ComparisonResult CompareScenarioResults(string elementName, ScenarioResults actual) { - var result = new ComparisonResult { ElementName = elementName, ElementType = "Scenario" }; + ComparisonResult result = new() { ElementName = elementName, ElementType = "Scenario" }; if (_baselineDoc == null) { @@ -66,7 +66,7 @@ public ComparisonResult CompareScenarioResults(string elementName, ScenarioResul public ComparisonResult CompareAlternativeResults(string elementName, AlternativeResults actual) { - var result = new ComparisonResult { ElementName = elementName, ElementType = "Alternative" }; + ComparisonResult result = new() { ElementName = elementName, ElementType = "Alternative" }; if (_baselineDoc == null) { @@ -130,7 +130,7 @@ public ComparisonResult CompareAlternativeResults(string elementName, Alternativ public ComparisonResult CompareStageDamage(string elementName, List actualCurves) { - var result = new ComparisonResult { ElementName = elementName, ElementType = "StageDamage" }; + ComparisonResult result = new() { ElementName = elementName, ElementType = "StageDamage" }; if (_baselineDoc == null) { @@ -267,7 +267,7 @@ private static void GenerateScenarioDiff(ScenarioResults baseline, ScenarioResul public ComparisonResult CompareAlternativeComparisonResults(string elementName, AlternativeComparisonReportResults actual, List<(int altId, string altName)> withProjectAlternatives) { - var result = new ComparisonResult { ElementName = elementName, ElementType = "AlternativeComparisonReport" }; + ComparisonResult result = new() { ElementName = elementName, ElementType = "AlternativeComparisonReport" }; if (_baselineDoc == null) { diff --git a/HEC.FDA.TestingUtility/Program.cs b/HEC.FDA.TestingUtility/Program.cs index 41f39b84e..39b543c41 100644 --- a/HEC.FDA.TestingUtility/Program.cs +++ b/HEC.FDA.TestingUtility/Program.cs @@ -4,32 +4,32 @@ using HEC.FDA.TestingUtility.Configuration; // Define command line options -var configOption = new Option( +Option configOption = new( name: "--config", description: "Path to JSON configuration file") { IsRequired = true }; configOption.AddAlias("-c"); -var outputOption = new Option( +Option outputOption = new( name: "--output", description: "Output directory for results", getDefaultValue: () => new DirectoryInfo(Environment.CurrentDirectory)); outputOption.AddAlias("-o"); -var verboseOption = new Option( +Option verboseOption = new( name: "--verbose", description: "Enable verbose output", getDefaultValue: () => false); verboseOption.AddAlias("-v"); -var studyOption = new Option( +Option studyOption = new( name: "--study", description: "Filter to specific study IDs (can specify multiple)") { AllowMultipleArgumentsPerToken = true }; studyOption.AddAlias("-s"); // Create root command -var rootCommand = new RootCommand("FDA Testing Utility - Regression Testing Tool for FDA Studies"); +RootCommand rootCommand = new("FDA Testing Utility - Regression Testing Tool for FDA Studies"); rootCommand.AddOption(configOption); rootCommand.AddOption(outputOption); rootCommand.AddOption(verboseOption); @@ -64,7 +64,7 @@ } // Create and run test runner - var runner = new TestRunner( + TestRunner runner = new( config, outputDir.FullName, verbose, diff --git a/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs b/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs index be7ca7629..56faa2b86 100644 --- a/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs +++ b/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs @@ -278,7 +278,7 @@ public void AddAlternativeComparisonResults(string studyId, string reportName, A /// public void SaveReport(string outputPath) { - var report = new StringBuilder(); + StringBuilder report = new(); report.AppendLine("=== FDA COMPUTATION RESULTS REPORT ==="); report.AppendLine($"Generated: {DateTime.Now:yyyy-MM-dd HH:mm:ss}"); diff --git a/HEC.FDA.TestingUtility/Services/AlternativeComparisonRunner.cs b/HEC.FDA.TestingUtility/Services/AlternativeComparisonRunner.cs index 7cae232a4..3d1cb6be8 100644 --- a/HEC.FDA.TestingUtility/Services/AlternativeComparisonRunner.cs +++ b/HEC.FDA.TestingUtility/Services/AlternativeComparisonRunner.cs @@ -40,7 +40,7 @@ public static AlternativeComparisonReportResults RunAlternativeComparison(string List withProjAlts = new(); foreach (int altId in element.WithProjAltIDs) { - var alt = GetAlternativeById(altId); + AlternativeElement? alt = GetAlternativeById(altId); if (alt == null) { throw new InvalidOperationException($"With-project alternative (ID={altId}) not found."); @@ -62,16 +62,16 @@ public static AlternativeComparisonReportResults RunAlternativeComparison(string // Compute each with-project alternative results List withProjResults = new(); - foreach (var withProjAlt in withProjAlts) + foreach (AlternativeElement withProjAlt in withProjAlts) { Console.WriteLine($" Computing with-project alternative '{withProjAlt.Name}'..."); - var results = ComputeAlternativeResults(withProjAlt, props, cancellationToken); + AlternativeResults results = ComputeAlternativeResults(withProjAlt, props, cancellationToken); withProjResults.Add(results); } // Compute the comparison report Console.WriteLine($" Computing alternative comparison report..."); - var comparisonResults = AlternativeComparisonReport.ComputeAlternativeComparisonReport( + AlternativeComparisonReportResults? comparisonResults = AlternativeComparisonReport.ComputeAlternativeComparisonReport( withoutProjResults, withProjResults); diff --git a/HEC.FDA.TestingUtility/TestRunner.cs b/HEC.FDA.TestingUtility/TestRunner.cs index 8498b27a4..1d8826765 100644 --- a/HEC.FDA.TestingUtility/TestRunner.cs +++ b/HEC.FDA.TestingUtility/TestRunner.cs @@ -45,7 +45,7 @@ public async Task RunAsync() int failures = 0; int passed = 0; var totalStopwatch = Stopwatch.StartNew(); - var studyTimings = new List<(string StudyId, TimeSpan Duration, List<(string Type, string Name, TimeSpan Duration)> Computations)>(); + List<(string StudyId, TimeSpan Duration, List<(string Type, string Name, TimeSpan Duration)> Computations)> studyTimings = new(); Console.WriteLine($"Starting test suite: {_config.TestSuiteId}"); Console.WriteLine($"Output directory: {_outputDir}"); @@ -69,11 +69,11 @@ public async Task RunAsync() { Console.WriteLine($"=== Testing study: {study.StudyName} ({study.StudyId}) ==="); var studyStopwatch = Stopwatch.StartNew(); - var computationTimings = new List<(string Type, string Name, TimeSpan Duration)>(); + List<(string Type, string Name, TimeSpan Duration)> computationTimings = new(); try { - using var loader = new StudyLoader(); + using StudyLoader loader = new(); loader.LoadStudy(study.NetworkSourcePath, _config.GlobalSettings.LocalTempDirectory); // Load the single baseline file for this study @@ -246,7 +246,7 @@ private static string FormatDuration(TimeSpan duration) private static List BuildComputationList(StudyConfiguration study) { - var computations = new List(study.Computations); + List computations = new(study.Computations); // Auto-discover scenarios if (study.RunAllScenarios) @@ -333,7 +333,7 @@ private static (AlternativeComparisonReportResults results, List<(int altId, str var element = ScenarioRunner.FindElement(elementName); // Build the list of with-project alternatives with names - var withProjectAlternatives = new List<(int altId, string altName)>(); + List<(int altId, string altName)> withProjectAlternatives = new(); var allAlternatives = BaseViewModel.StudyCache.GetChildElementsOfType(); foreach (int altId in element.WithProjAltIDs) From 06e0d671c9ae7aef01a11f91a649acd2eded055c Mon Sep 17 00:00:00 2001 From: Brennan Beam <64556723+Brennan1994@users.noreply.github.com> Date: Tue, 23 Dec 2025 16:09:33 -0800 Subject: [PATCH 08/13] fix order, ensure results are being saved. --- HEC.FDA.TestingUtility/TestRunner.cs | 107 +++++++++++++++++++++++---- 1 file changed, 93 insertions(+), 14 deletions(-) diff --git a/HEC.FDA.TestingUtility/TestRunner.cs b/HEC.FDA.TestingUtility/TestRunner.cs index 1d8826765..e82285dfb 100644 --- a/HEC.FDA.TestingUtility/TestRunner.cs +++ b/HEC.FDA.TestingUtility/TestRunner.cs @@ -11,6 +11,10 @@ using HEC.FDA.ViewModel.Alternatives; using HEC.FDA.ViewModel.AlternativeComparisonReport; using HEC.FDA.ViewModel.ImpactAreaScenario; +using HEC.FDA.ViewModel.ImpactArea; +using HEC.FDA.ViewModel.Saving; +using HEC.FDA.ViewModel.TableWithPlot; +using HEC.FDA.ViewModel.Utilities; namespace HEC.FDA.TestingUtility; @@ -107,29 +111,32 @@ public async Task RunAsync() switch (compute.Type.ToLowerInvariant()) { + case "stagedamage": + List sdCurves = StageDamageRunner.RunStageDamage(compute.ElementName); + SaveStageDamageResults(compute.ElementName, sdCurves); + StudyBaselineWriter.AddStageDamage(computedBaseline, compute.ElementName, sdCurves); + _csvReportFactory.AddStageDamageSummary(study.StudyId, compute.ElementName, sdCurves); + result = _comparer.CompareStageDamage(compute.ElementName, sdCurves); + break; + case "scenario": - var scenarioResults = ScenarioRunner.RunScenario(compute.ElementName, _cts.Token); + ScenarioResults scenarioResults = ScenarioRunner.RunScenario(compute.ElementName, _cts.Token); + SaveScenarioResults(compute.ElementName, scenarioResults); StudyBaselineWriter.AddScenarioResults(computedBaseline, compute.ElementName, scenarioResults); _csvReportFactory.AddScenarioResults(study.StudyId, compute.ElementName, scenarioResults); result = _comparer.CompareScenarioResults(compute.ElementName, scenarioResults); break; case "alternative": - var altResults = AlternativeRunner.RunAlternative(compute.ElementName, _cts.Token); + AlternativeResults altResults = AlternativeRunner.RunAlternative(compute.ElementName, _cts.Token); + SaveAlternativeResults(compute.ElementName, altResults); StudyBaselineWriter.AddAlternativeResults(computedBaseline, compute.ElementName, altResults); _csvReportFactory.AddAlternativeResults(study.StudyId, compute.ElementName, altResults); result = _comparer.CompareAlternativeResults(compute.ElementName, altResults); break; - case "stagedamage": - List sdCurves = StageDamageRunner.RunStageDamage(compute.ElementName); - StudyBaselineWriter.AddStageDamage(computedBaseline, compute.ElementName, sdCurves); - _csvReportFactory.AddStageDamageSummary(study.StudyId, compute.ElementName, sdCurves); - result = _comparer.CompareStageDamage(compute.ElementName, sdCurves); - break; - case "alternativecomparison": - var (compResults, withProjAlts) = RunAlternativeComparisonWithMetadata(compute.ElementName, _cts.Token); + (AlternativeComparisonReportResults compResults, List<(int altId, string altName)> withProjAlts) = RunAlternativeComparisonWithMetadata(compute.ElementName, _cts.Token); StudyBaselineWriter.AddAlternativeComparisonResults(computedBaseline, compute.ElementName, compResults, withProjAlts); _csvReportFactory.AddAlternativeComparisonResults(study.StudyId, compute.ElementName, compResults, withProjAlts); result = _comparer.CompareAlternativeComparisonResults(compute.ElementName, compResults, withProjAlts); @@ -308,8 +315,8 @@ private static List BuildComputationList(StudyConfiguratio // Auto-discover alternative comparison reports (depend on alternatives, so run last) if (study.RunAllAlternativeComparisons) { - var altCompReports = BaseViewModel.StudyCache.GetChildElementsOfType(); - foreach (var report in altCompReports) + List altCompReports = BaseViewModel.StudyCache.GetChildElementsOfType(); + foreach (AlternativeComparisonReportElement report in altCompReports) { if (!computations.Any(c => c.Type.Equals("alternativecomparison", StringComparison.OrdinalIgnoreCase) && c.ElementName.Equals(report.Name, StringComparison.OrdinalIgnoreCase))) @@ -324,7 +331,22 @@ private static List BuildComputationList(StudyConfiguratio } } - return computations; + // Sort by dependency order: stagedamage → scenario → alternative → alternativecomparison + return SortByDependencyOrder(computations); + } + + private static List SortByDependencyOrder(List computations) + { + int GetOrder(string type) => type.ToLowerInvariant() switch + { + "stagedamage" => 0, + "scenario" => 1, + "alternative" => 2, + "alternativecomparison" => 3, + _ => 99 + }; + + return computations.OrderBy(c => GetOrder(c.Type)).ToList(); } private static (AlternativeComparisonReportResults results, List<(int altId, string altName)> withProjectAlternatives) RunAlternativeComparisonWithMetadata(string elementName, CancellationToken cancellationToken) @@ -371,7 +393,7 @@ private void PrintDifferences(ComparisonResult result) } Console.WriteLine(" Differences:"); - foreach (var diff in result.Differences.Take(10)) // Limit to first 10 + foreach (Difference diff in result.Differences.Take(10)) { Console.WriteLine($" - {diff}"); } @@ -381,4 +403,61 @@ private void PrintDifferences(ComparisonResult result) Console.WriteLine($" ... and {result.Differences.Count - 10} more"); } } + + /// + /// Saves scenario results to the temp database so downstream computations can use them. + /// + private static void SaveScenarioResults(string elementName, ScenarioResults results) + { + IASElement element = ScenarioRunner.FindElement(elementName); + element.Results = results; + PersistenceFactory.GetIASManager().SaveExisting(element); + Console.WriteLine($" Saved scenario results to temp database."); + } + + /// + /// Saves alternative results to the temp database so downstream computations can use them. + /// + private static void SaveAlternativeResults(string elementName, AlternativeResults results) + { + AlternativeElement element = ScenarioRunner.FindElement(elementName); + element.Results = results; + PersistenceFactory.GetElementManager().SaveExisting(element); + Console.WriteLine($" Saved alternative results to temp database."); + } + + /// + /// Saves stage damage curves to the temp database so downstream computations can use them. + /// + private static void SaveStageDamageResults(string elementName, List curves) + { + AggregatedStageDamageElement element = ScenarioRunner.FindElement(elementName); + + // Get impact area element to look up names + List impactAreaElements = BaseViewModel.StudyCache.GetChildElementsOfType(); + ImpactAreaElement? impactAreaElement = impactAreaElements.Count > 0 ? impactAreaElements[0] : null; + + // Convert UncertainPairedData curves to StageDamageCurves and update the element + List stageDamageCurves = new(); + foreach (UncertainPairedData upd in curves) + { + // Create CurveComponentVM and set the paired data + CurveComponentVM curveComponent = new(StringConstants.STAGE_DAMAGE, StringConstants.STAGE, StringConstants.DAMAGE, DistributionOptions.HISTOGRAM_ONLY); + curveComponent.SetPairedData(upd); + + // Get the impact area row item + ImpactAreaRowItem impactAreaRowItem = impactAreaElement?.GetImpactAreaRow(upd.ImpactAreaID) + ?? new ImpactAreaRowItem(upd.ImpactAreaID, ""); + + StageDamageCurve sdCurve = new(impactAreaRowItem, upd.DamageCategory, curveComponent, upd.AssetCategory, StageDamageConstructionType.COMPUTED); + stageDamageCurves.Add(sdCurve); + } + + // Update the element's curves - this modifies the in-memory element + element.Curves.Clear(); + element.Curves.AddRange(stageDamageCurves); + + PersistenceFactory.GetElementManager().SaveExisting(element); + Console.WriteLine($" Saved {curves.Count} stage damage curves to temp database."); + } } From b8a89ce4c4f2bae6accbaf63f33ff80d600382dd Mon Sep 17 00:00:00 2001 From: Brennan Beam <64556723+Brennan1994@users.noreply.github.com> Date: Tue, 23 Dec 2025 16:25:11 -0800 Subject: [PATCH 09/13] Separate compute and compare --- HEC.FDA.TestingUtility/CompareRunner.cs | 275 +++++++++++++++++ .../{TestRunner.cs => ComputeRunner.cs} | 276 ++++++------------ HEC.FDA.TestingUtility/Program.cs | 133 ++++++--- 3 files changed, 454 insertions(+), 230 deletions(-) create mode 100644 HEC.FDA.TestingUtility/CompareRunner.cs rename HEC.FDA.TestingUtility/{TestRunner.cs => ComputeRunner.cs} (54%) diff --git a/HEC.FDA.TestingUtility/CompareRunner.cs b/HEC.FDA.TestingUtility/CompareRunner.cs new file mode 100644 index 000000000..6a74f8cef --- /dev/null +++ b/HEC.FDA.TestingUtility/CompareRunner.cs @@ -0,0 +1,275 @@ +using System.Text; +using System.Xml.Linq; + +namespace HEC.FDA.TestingUtility; + +/// +/// Compares two sets of FDA computation results and generates a comparison report. +/// +public class CompareRunner +{ + private readonly string _baselineDir; + private readonly string _newDir; + private readonly string _outputPath; + private readonly double _tolerance; + + public CompareRunner(string baselineDir, string newDir, string outputPath, double tolerance) + { + _baselineDir = baselineDir; + _newDir = newDir; + _outputPath = outputPath; + _tolerance = tolerance; + } + + public int Run() + { + Console.WriteLine($"Baseline directory: {_baselineDir}"); + Console.WriteLine($"New results directory: {_newDir}"); + Console.WriteLine($"Tolerance: {_tolerance:P1}"); + Console.WriteLine(); + + // Find all XML result files in both directories + string[] baselineFiles = Directory.GetFiles(_baselineDir, "*_results.xml"); + string[] newFiles = Directory.GetFiles(_newDir, "*_results.xml"); + + if (baselineFiles.Length == 0) + { + Console.WriteLine("No baseline result files (*_results.xml) found."); + return 1; + } + + if (newFiles.Length == 0) + { + Console.WriteLine("No new result files (*_results.xml) found."); + return 1; + } + + StringBuilder report = new(); + report.AppendLine("FDA Results Comparison Report"); + report.AppendLine("============================="); + report.AppendLine($"Generated: {DateTime.Now:yyyy-MM-dd HH:mm:ss}"); + report.AppendLine($"Baseline: {_baselineDir}"); + report.AppendLine($"New: {_newDir}"); + report.AppendLine($"Tolerance: {_tolerance:P1}"); + report.AppendLine(); + + int totalDifferences = 0; + int filesCompared = 0; + + // Match files by study ID + foreach (string baselineFile in baselineFiles) + { + string fileName = Path.GetFileName(baselineFile); + string newFile = Path.Combine(_newDir, fileName); + + if (!File.Exists(newFile)) + { + report.AppendLine($"MISSING: {fileName} not found in new results"); + Console.WriteLine($"MISSING: {fileName}"); + totalDifferences++; + continue; + } + + Console.WriteLine($"Comparing: {fileName}"); + int differences = CompareFiles(baselineFile, newFile, report); + totalDifferences += differences; + filesCompared++; + + if (differences == 0) + { + Console.WriteLine($" MATCH: No differences found"); + } + else + { + Console.WriteLine($" DIFF: {differences} difference(s) found"); + } + } + + // Check for files in new that aren't in baseline + foreach (string newFile in newFiles) + { + string fileName = Path.GetFileName(newFile); + string baselineFile = Path.Combine(_baselineDir, fileName); + + if (!File.Exists(baselineFile)) + { + report.AppendLine($"NEW: {fileName} exists only in new results"); + Console.WriteLine($"NEW: {fileName}"); + } + } + + // Summary + report.AppendLine(); + report.AppendLine("=== Summary ==="); + report.AppendLine($"Files compared: {filesCompared}"); + report.AppendLine($"Total differences: {totalDifferences}"); + report.AppendLine($"Result: {(totalDifferences == 0 ? "PASS" : "FAIL")}"); + + // Save report + File.WriteAllText(_outputPath, report.ToString()); + Console.WriteLine(); + Console.WriteLine($"Report saved to: {_outputPath}"); + Console.WriteLine(); + Console.WriteLine($"=== Result: {(totalDifferences == 0 ? "PASS" : "FAIL")} ==="); + Console.WriteLine($"Files compared: {filesCompared}"); + Console.WriteLine($"Total differences: {totalDifferences}"); + + return totalDifferences > 0 ? 1 : 0; + } + + private int CompareFiles(string baselinePath, string newPath, StringBuilder report) + { + int differences = 0; + + try + { + XElement baseline = XElement.Load(baselinePath); + XElement newResults = XElement.Load(newPath); + + string studyId = baseline.Attribute("studyId")?.Value ?? Path.GetFileNameWithoutExtension(baselinePath); + report.AppendLine($"=== {studyId} ==="); + + // Compare scenarios + differences += CompareElements(baseline, newResults, "ScenarioResults", "name", report); + + // Compare alternatives + differences += CompareElements(baseline, newResults, "AlternativeResults", "name", report); + + // Compare stage damage + differences += CompareElements(baseline, newResults, "StageDamage", "name", report); + + // Compare alternative comparison reports + differences += CompareElements(baseline, newResults, "AlternativeComparisonReport", "name", report); + + if (differences == 0) + { + report.AppendLine(" All values match within tolerance."); + } + report.AppendLine(); + } + catch (Exception ex) + { + report.AppendLine($" ERROR: {ex.Message}"); + differences++; + } + + return differences; + } + + private int CompareElements(XElement baseline, XElement newResults, string elementType, string nameAttribute, StringBuilder report) + { + int differences = 0; + + IEnumerable baselineElements = baseline.Elements(elementType); + IEnumerable newElements = newResults.Elements(elementType); + + foreach (XElement baselineElem in baselineElements) + { + string name = baselineElem.Attribute(nameAttribute)?.Value ?? "unknown"; + XElement? newElem = newElements.FirstOrDefault(e => e.Attribute(nameAttribute)?.Value == name); + + if (newElem == null) + { + report.AppendLine($" MISSING: {elementType} '{name}' not in new results"); + differences++; + continue; + } + + // Compare numeric attributes + List<(string path, double baseline, double newVal)> numericDiffs = new(); + CompareNumericValues(baselineElem, newElem, "", numericDiffs); + + foreach ((string path, double baselineVal, double newVal) in numericDiffs) + { + double absDiff = Math.Abs(baselineVal - newVal); + double relDiff = baselineVal != 0 ? absDiff / Math.Abs(baselineVal) : absDiff; + + if (relDiff > _tolerance && absDiff > 1.0) // At least $1 difference + { + differences++; + report.AppendLine($" DIFF: {elementType} '{name}' {path}"); + report.AppendLine($" Baseline: {baselineVal:F4}"); + report.AppendLine($" New: {newVal:F4}"); + report.AppendLine($" Diff: {absDiff:F4} ({relDiff:P2})"); + } + } + } + + // Check for elements in new that aren't in baseline + foreach (XElement newElem in newElements) + { + string name = newElem.Attribute(nameAttribute)?.Value ?? "unknown"; + XElement? baselineElem = baselineElements.FirstOrDefault(e => e.Attribute(nameAttribute)?.Value == name); + + if (baselineElem == null) + { + report.AppendLine($" NEW: {elementType} '{name}' only in new results"); + } + } + + return differences; + } + + private void CompareNumericValues(XElement baseline, XElement newElem, string path, List<(string, double, double)> diffs) + { + // Compare attributes + foreach (XAttribute attr in baseline.Attributes()) + { + if (double.TryParse(attr.Value, out double baselineVal)) + { + XAttribute? newAttr = newElem.Attribute(attr.Name); + if (newAttr != null && double.TryParse(newAttr.Value, out double newVal)) + { + string attrPath = string.IsNullOrEmpty(path) ? $"@{attr.Name}" : $"{path}/@{attr.Name}"; + diffs.Add((attrPath, baselineVal, newVal)); + } + } + } + + // Compare child elements recursively + foreach (XElement baselineChild in baseline.Elements()) + { + string childName = baselineChild.Name.LocalName; + + // Try to find matching child by element name and any identifying attributes + XElement? matchingChild = FindMatchingChild(newElem, baselineChild); + + if (matchingChild != null) + { + string childPath = string.IsNullOrEmpty(path) ? childName : $"{path}/{childName}"; + + // Add identifying attributes to path if present + string? id = baselineChild.Attribute("id")?.Value ?? baselineChild.Attribute("name")?.Value; + if (id != null) + { + childPath += $"[{id}]"; + } + + CompareNumericValues(baselineChild, matchingChild, childPath, diffs); + } + } + } + + private static XElement? FindMatchingChild(XElement parent, XElement target) + { + string targetName = target.Name.LocalName; + IEnumerable candidates = parent.Elements(targetName); + + // Try to match by id or name attribute + string? targetId = target.Attribute("id")?.Value; + string? targetNameAttr = target.Attribute("name")?.Value; + + if (targetId != null) + { + return candidates.FirstOrDefault(c => c.Attribute("id")?.Value == targetId); + } + + if (targetNameAttr != null) + { + return candidates.FirstOrDefault(c => c.Attribute("name")?.Value == targetNameAttr); + } + + // If no identifying attributes, just take the first one with the same name + return candidates.FirstOrDefault(); + } +} diff --git a/HEC.FDA.TestingUtility/TestRunner.cs b/HEC.FDA.TestingUtility/ComputeRunner.cs similarity index 54% rename from HEC.FDA.TestingUtility/TestRunner.cs rename to HEC.FDA.TestingUtility/ComputeRunner.cs index e82285dfb..2a0ecdf60 100644 --- a/HEC.FDA.TestingUtility/TestRunner.cs +++ b/HEC.FDA.TestingUtility/ComputeRunner.cs @@ -18,26 +18,25 @@ namespace HEC.FDA.TestingUtility; -public class TestRunner +/// +/// Runs FDA computations and generates result files (XML and CSV). +/// Does not perform comparisons - use CompareRunner for that. +/// +public class ComputeRunner { private readonly TestConfiguration _config; private readonly string _outputDir; - private readonly bool _verbose; private readonly string[]? _studyFilter; private readonly CancellationTokenSource _cts; - - private readonly XmlResultComparer _comparer = new(); private readonly CsvReportFactory _csvReportFactory = new(); - public TestRunner(TestConfiguration config, string outputDir, bool verbose, string[]? studyFilter) + public ComputeRunner(TestConfiguration config, string outputDir, string[]? studyFilter) { _config = config; _outputDir = outputDir; - _verbose = verbose; _studyFilter = studyFilter; _cts = new CancellationTokenSource(); - // Set timeout if (_config.GlobalSettings.TimeoutMinutes > 0) { _cts.CancelAfter(TimeSpan.FromMinutes(_config.GlobalSettings.TimeoutMinutes)); @@ -46,16 +45,16 @@ public TestRunner(TestConfiguration config, string outputDir, bool verbose, stri public async Task RunAsync() { - int failures = 0; - int passed = 0; - var totalStopwatch = Stopwatch.StartNew(); - List<(string StudyId, TimeSpan Duration, List<(string Type, string Name, TimeSpan Duration)> Computations)> studyTimings = new(); + int errors = 0; + int completed = 0; + Stopwatch totalStopwatch = Stopwatch.StartNew(); + List<(string StudyId, TimeSpan Duration, int ComputeCount, int ErrorCount)> studyTimings = new(); - Console.WriteLine($"Starting test suite: {_config.TestSuiteId}"); + Console.WriteLine($"Configuration: {_config.TestSuiteId}"); Console.WriteLine($"Output directory: {_outputDir}"); Console.WriteLine(); - var studiesToRun = _config.Studies; + List studiesToRun = _config.Studies; if (_studyFilter != null && _studyFilter.Length > 0) { studiesToRun = studiesToRun @@ -69,77 +68,57 @@ public async Task RunAsync() } } - foreach (var study in studiesToRun) + foreach (StudyConfiguration study in studiesToRun) { - Console.WriteLine($"=== Testing study: {study.StudyName} ({study.StudyId}) ==="); - var studyStopwatch = Stopwatch.StartNew(); - List<(string Type, string Name, TimeSpan Duration)> computationTimings = new(); + Console.WriteLine($"=== Computing: {study.StudyName} ({study.StudyId}) ==="); + Stopwatch studyStopwatch = Stopwatch.StartNew(); + int studyErrors = 0; + int studyCompleted = 0; try { using StudyLoader loader = new(); loader.LoadStudy(study.NetworkSourcePath, _config.GlobalSettings.LocalTempDirectory); - // Load the single baseline file for this study - string baselinePath = GetBaselinePath(study); - Console.WriteLine($" Loading baseline: {baselinePath}"); - - if (!File.Exists(baselinePath)) - { - Console.WriteLine($" WARNING: Baseline file not found. Tests will fail."); - } - else - { - _comparer.LoadBaseline(baselinePath); - } - - // Create computed results document for debugging - var computedBaseline = StudyBaselineWriter.CreateStudyBaseline(study.StudyId, study.StudyName); + XElement computedResults = StudyBaselineWriter.CreateStudyBaseline(study.StudyId, study.StudyName); - // Build computation list (from config or auto-discover) - var computations = BuildComputationList(study); + List computations = BuildComputationList(study); Console.WriteLine($" Found {computations.Count} computations to run."); - foreach (var compute in computations) + foreach (ComputeConfiguration compute in computations) { _cts.Token.ThrowIfCancellationRequested(); - var computeStopwatch = Stopwatch.StartNew(); + Stopwatch computeStopwatch = Stopwatch.StartNew(); try { - ComparisonResult? result = null; - switch (compute.Type.ToLowerInvariant()) { case "stagedamage": List sdCurves = StageDamageRunner.RunStageDamage(compute.ElementName); SaveStageDamageResults(compute.ElementName, sdCurves); - StudyBaselineWriter.AddStageDamage(computedBaseline, compute.ElementName, sdCurves); + StudyBaselineWriter.AddStageDamage(computedResults, compute.ElementName, sdCurves); _csvReportFactory.AddStageDamageSummary(study.StudyId, compute.ElementName, sdCurves); - result = _comparer.CompareStageDamage(compute.ElementName, sdCurves); break; case "scenario": ScenarioResults scenarioResults = ScenarioRunner.RunScenario(compute.ElementName, _cts.Token); SaveScenarioResults(compute.ElementName, scenarioResults); - StudyBaselineWriter.AddScenarioResults(computedBaseline, compute.ElementName, scenarioResults); + StudyBaselineWriter.AddScenarioResults(computedResults, compute.ElementName, scenarioResults); _csvReportFactory.AddScenarioResults(study.StudyId, compute.ElementName, scenarioResults); - result = _comparer.CompareScenarioResults(compute.ElementName, scenarioResults); break; case "alternative": AlternativeResults altResults = AlternativeRunner.RunAlternative(compute.ElementName, _cts.Token); SaveAlternativeResults(compute.ElementName, altResults); - StudyBaselineWriter.AddAlternativeResults(computedBaseline, compute.ElementName, altResults); + StudyBaselineWriter.AddAlternativeResults(computedResults, compute.ElementName, altResults); _csvReportFactory.AddAlternativeResults(study.StudyId, compute.ElementName, altResults); - result = _comparer.CompareAlternativeResults(compute.ElementName, altResults); break; case "alternativecomparison": (AlternativeComparisonReportResults compResults, List<(int altId, string altName)> withProjAlts) = RunAlternativeComparisonWithMetadata(compute.ElementName, _cts.Token); - StudyBaselineWriter.AddAlternativeComparisonResults(computedBaseline, compute.ElementName, compResults, withProjAlts); + StudyBaselineWriter.AddAlternativeComparisonResults(computedResults, compute.ElementName, compResults, withProjAlts); _csvReportFactory.AddAlternativeComparisonResults(study.StudyId, compute.ElementName, compResults, withProjAlts); - result = _comparer.CompareAlternativeComparisonResults(compute.ElementName, compResults, withProjAlts); break; default: @@ -148,59 +127,40 @@ public async Task RunAsync() } computeStopwatch.Stop(); - computationTimings.Add((compute.Type, compute.ElementName, computeStopwatch.Elapsed)); - - if (result != null) - { - if (!result.Passed) - { - failures++; - Console.WriteLine($" FAIL: {compute.Type} '{compute.ElementName}' [{FormatDuration(computeStopwatch.Elapsed)}]"); - Console.WriteLine($" {result.Summary}"); - PrintDifferences(result); - } - else - { - passed++; - Console.WriteLine($" PASS: {compute.Type} '{compute.ElementName}' [{FormatDuration(computeStopwatch.Elapsed)}]"); - } - } + studyCompleted++; + Console.WriteLine($" OK: {compute.Type} '{compute.ElementName}' [{FormatDuration(computeStopwatch.Elapsed)}]"); } catch (Exception ex) { computeStopwatch.Stop(); - computationTimings.Add((compute.Type, compute.ElementName, computeStopwatch.Elapsed)); - failures++; - Console.WriteLine($" ERROR: {compute.Type} '{compute.ElementName}' [{FormatDuration(computeStopwatch.Elapsed)}] - {ex.Message}"); - if (_verbose) - { - Console.WriteLine($" {ex.StackTrace}"); - } + studyErrors++; + Console.WriteLine($" ERROR: {compute.Type} '{compute.ElementName}' [{FormatDuration(computeStopwatch.Elapsed)}]"); + Console.WriteLine($" {ex.Message}"); + Console.WriteLine($" {ex.StackTrace}"); } } - // Save computed results for debugging - SaveComputedResults(computedBaseline, study); + // Save results + SaveResults(computedResults, study); } catch (OperationCanceledException) { - Console.WriteLine(" TIMEOUT: Test run exceeded time limit."); - failures++; + Console.WriteLine(" TIMEOUT: Computation exceeded time limit."); + studyErrors++; break; } catch (Exception ex) { Console.WriteLine($" ERROR loading study: {ex.Message}"); - if (_verbose) - { - Console.WriteLine($" {ex.StackTrace}"); - } - failures++; + Console.WriteLine($" {ex.StackTrace}"); + studyErrors++; } studyStopwatch.Stop(); - studyTimings.Add((study.StudyId, studyStopwatch.Elapsed, computationTimings)); - Console.WriteLine($" Study completed in {FormatDuration(studyStopwatch.Elapsed)}"); + studyTimings.Add((study.StudyId, studyStopwatch.Elapsed, studyCompleted, studyErrors)); + completed += studyCompleted; + errors += studyErrors; + Console.WriteLine($" Completed in {FormatDuration(studyStopwatch.Elapsed)} ({studyCompleted} succeeded, {studyErrors} failed)"); Console.WriteLine(); } @@ -208,31 +168,24 @@ public async Task RunAsync() // Summary Console.WriteLine("=== Summary ==="); - Console.WriteLine($"Passed: {passed}"); - Console.WriteLine($"Failed: {failures}"); - Console.WriteLine($"Total: {passed + failures}"); + Console.WriteLine($"Completed: {completed}"); + Console.WriteLine($"Errors: {errors}"); + Console.WriteLine($"Duration: {FormatDuration(totalStopwatch.Elapsed)}"); Console.WriteLine(); - // Timing Summary - Console.WriteLine("=== Timing Summary ==="); - Console.WriteLine($"Total Duration: {FormatDuration(totalStopwatch.Elapsed)}"); - Console.WriteLine(); - - foreach (var (studyId, duration, computations) in studyTimings) - { - Console.WriteLine($" {studyId}: {FormatDuration(duration)}"); - foreach (var (type, name, compDuration) in computations) - { - Console.WriteLine($" - {type} '{name}': {FormatDuration(compDuration)}"); - } - } - // Save CSV report - Console.WriteLine(); string csvPath = Path.Combine(_outputDir, "results_report.csv"); _csvReportFactory.SaveReport(csvPath); - return failures > 0 ? 1 : 0; + Console.WriteLine(); + Console.WriteLine("Generated files:"); + foreach ((string studyId, _, _, _) in studyTimings) + { + Console.WriteLine($" {studyId}_results.xml"); + } + Console.WriteLine($" results_report.csv"); + + return errors > 0 ? 1 : 0; } private static string FormatDuration(TimeSpan duration) @@ -255,64 +208,48 @@ private static List BuildComputationList(StudyConfiguratio { List computations = new(study.Computations); - // Auto-discover scenarios - if (study.RunAllScenarios) + if (study.RunAllStageDamage) { - var scenarios = BaseViewModel.StudyCache.GetChildElementsOfType(); - foreach (var scenario in scenarios) + List stageDamages = BaseViewModel.StudyCache.GetChildElementsOfType(); + foreach (AggregatedStageDamageElement sd in stageDamages) { - if (!computations.Any(c => c.Type.Equals("scenario", StringComparison.OrdinalIgnoreCase) - && c.ElementName.Equals(scenario.Name, StringComparison.OrdinalIgnoreCase))) + if (!computations.Any(c => c.Type.Equals("stagedamage", StringComparison.OrdinalIgnoreCase) + && c.ElementName.Equals(sd.Name, StringComparison.OrdinalIgnoreCase))) { - computations.Add(new ComputeConfiguration - { - Type = "scenario", - ElementName = scenario.Name - }); - Console.WriteLine($" Auto-discovered scenario: {scenario.Name}"); + computations.Add(new ComputeConfiguration { Type = "stagedamage", ElementName = sd.Name }); + Console.WriteLine($" Auto-discovered stage damage: {sd.Name}"); } } } - // Auto-discover stage damage elements - if (study.RunAllStageDamage) + if (study.RunAllScenarios) { - var stageDamages = BaseViewModel.StudyCache.GetChildElementsOfType(); - foreach (var sd in stageDamages) + List scenarios = BaseViewModel.StudyCache.GetChildElementsOfType(); + foreach (IASElement scenario in scenarios) { - if (!computations.Any(c => c.Type.Equals("stagedamage", StringComparison.OrdinalIgnoreCase) - && c.ElementName.Equals(sd.Name, StringComparison.OrdinalIgnoreCase))) + if (!computations.Any(c => c.Type.Equals("scenario", StringComparison.OrdinalIgnoreCase) + && c.ElementName.Equals(scenario.Name, StringComparison.OrdinalIgnoreCase))) { - computations.Add(new ComputeConfiguration - { - Type = "stagedamage", - ElementName = sd.Name - }); - Console.WriteLine($" Auto-discovered stage damage: {sd.Name}"); + computations.Add(new ComputeConfiguration { Type = "scenario", ElementName = scenario.Name }); + Console.WriteLine($" Auto-discovered scenario: {scenario.Name}"); } } } - // Auto-discover alternatives (these depend on scenario results, so run last) if (study.RunAllAlternatives) { - var alternatives = BaseViewModel.StudyCache.GetChildElementsOfType(); - foreach (var alt in alternatives) + List alternatives = BaseViewModel.StudyCache.GetChildElementsOfType(); + foreach (AlternativeElement alt in alternatives) { if (!computations.Any(c => c.Type.Equals("alternative", StringComparison.OrdinalIgnoreCase) && c.ElementName.Equals(alt.Name, StringComparison.OrdinalIgnoreCase))) { - computations.Add(new ComputeConfiguration - { - Type = "alternative", - ElementName = alt.Name - }); + computations.Add(new ComputeConfiguration { Type = "alternative", ElementName = alt.Name }); Console.WriteLine($" Auto-discovered alternative: {alt.Name}"); } } } - // Auto-discover alternative comparison reports (depend on alternatives, so run last) if (study.RunAllAlternativeComparisons) { List altCompReports = BaseViewModel.StudyCache.GetChildElementsOfType(); @@ -321,17 +258,12 @@ private static List BuildComputationList(StudyConfiguratio if (!computations.Any(c => c.Type.Equals("alternativecomparison", StringComparison.OrdinalIgnoreCase) && c.ElementName.Equals(report.Name, StringComparison.OrdinalIgnoreCase))) { - computations.Add(new ComputeConfiguration - { - Type = "alternativecomparison", - ElementName = report.Name - }); + computations.Add(new ComputeConfiguration { Type = "alternativecomparison", ElementName = report.Name }); Console.WriteLine($" Auto-discovered alternative comparison: {report.Name}"); } } } - // Sort by dependency order: stagedamage → scenario → alternative → alternativecomparison return SortByDependencyOrder(computations); } @@ -351,101 +283,58 @@ private static List SortByDependencyOrder(List withProjectAlternatives) RunAlternativeComparisonWithMetadata(string elementName, CancellationToken cancellationToken) { - // Get the element to extract the with-project alternative IDs - var element = ScenarioRunner.FindElement(elementName); + AlternativeComparisonReportElement element = ScenarioRunner.FindElement(elementName); - // Build the list of with-project alternatives with names List<(int altId, string altName)> withProjectAlternatives = new(); - var allAlternatives = BaseViewModel.StudyCache.GetChildElementsOfType(); + List allAlternatives = BaseViewModel.StudyCache.GetChildElementsOfType(); foreach (int altId in element.WithProjAltIDs) { - var alt = allAlternatives.FirstOrDefault(a => a.ID == altId); + AlternativeElement? alt = allAlternatives.FirstOrDefault(a => a.ID == altId); string altName = alt?.Name ?? $"Alternative_{altId}"; withProjectAlternatives.Add((altId, altName)); } - // Run the computation - var results = AlternativeComparisonRunner.RunAlternativeComparison(elementName, cancellationToken); - + AlternativeComparisonReportResults results = AlternativeComparisonRunner.RunAlternativeComparison(elementName, cancellationToken); return (results, withProjectAlternatives); } - private static string GetBaselinePath(StudyConfiguration study) - { - // Single baseline file per study - return Path.Combine(study.BaselineDirectory, $"{study.StudyId}_baseline.xml"); - } - - private void SaveComputedResults(XElement computedBaseline, StudyConfiguration study) + private void SaveResults(XElement computedResults, StudyConfiguration study) { - // Save computed results in same format as baseline for easy comparison - string outputPath = Path.Combine(_outputDir, $"{study.StudyId}_computed.xml"); - StudyBaselineWriter.Save(computedBaseline, outputPath); - Console.WriteLine($" Computed results saved to: {outputPath}"); - } - - private void PrintDifferences(ComparisonResult result) - { - if (!_verbose || result.Differences.Count == 0) - { - return; - } - - Console.WriteLine(" Differences:"); - foreach (Difference diff in result.Differences.Take(10)) - { - Console.WriteLine($" - {diff}"); - } - - if (result.Differences.Count > 10) - { - Console.WriteLine($" ... and {result.Differences.Count - 10} more"); - } + string outputPath = Path.Combine(_outputDir, $"{study.StudyId}_results.xml"); + StudyBaselineWriter.Save(computedResults, outputPath); + Console.WriteLine($" Results saved to: {outputPath}"); } - /// - /// Saves scenario results to the temp database so downstream computations can use them. - /// private static void SaveScenarioResults(string elementName, ScenarioResults results) { IASElement element = ScenarioRunner.FindElement(elementName); element.Results = results; PersistenceFactory.GetIASManager().SaveExisting(element); - Console.WriteLine($" Saved scenario results to temp database."); + Console.WriteLine($" Saved to temp database."); } - /// - /// Saves alternative results to the temp database so downstream computations can use them. - /// private static void SaveAlternativeResults(string elementName, AlternativeResults results) { AlternativeElement element = ScenarioRunner.FindElement(elementName); element.Results = results; PersistenceFactory.GetElementManager().SaveExisting(element); - Console.WriteLine($" Saved alternative results to temp database."); + Console.WriteLine($" Saved to temp database."); } - /// - /// Saves stage damage curves to the temp database so downstream computations can use them. - /// private static void SaveStageDamageResults(string elementName, List curves) { AggregatedStageDamageElement element = ScenarioRunner.FindElement(elementName); - // Get impact area element to look up names List impactAreaElements = BaseViewModel.StudyCache.GetChildElementsOfType(); ImpactAreaElement? impactAreaElement = impactAreaElements.Count > 0 ? impactAreaElements[0] : null; - // Convert UncertainPairedData curves to StageDamageCurves and update the element List stageDamageCurves = new(); foreach (UncertainPairedData upd in curves) { - // Create CurveComponentVM and set the paired data CurveComponentVM curveComponent = new(StringConstants.STAGE_DAMAGE, StringConstants.STAGE, StringConstants.DAMAGE, DistributionOptions.HISTOGRAM_ONLY); curveComponent.SetPairedData(upd); - // Get the impact area row item ImpactAreaRowItem impactAreaRowItem = impactAreaElement?.GetImpactAreaRow(upd.ImpactAreaID) ?? new ImpactAreaRowItem(upd.ImpactAreaID, ""); @@ -453,11 +342,10 @@ private static void SaveStageDamageResults(string elementName, List().SaveExisting(element); - Console.WriteLine($" Saved {curves.Count} stage damage curves to temp database."); + Console.WriteLine($" Saved {curves.Count} curves to temp database."); } } diff --git a/HEC.FDA.TestingUtility/Program.cs b/HEC.FDA.TestingUtility/Program.cs index 39b543c41..c887b7d95 100644 --- a/HEC.FDA.TestingUtility/Program.cs +++ b/HEC.FDA.TestingUtility/Program.cs @@ -3,51 +3,45 @@ using HEC.FDA.TestingUtility; using HEC.FDA.TestingUtility.Configuration; -// Define command line options -Option configOption = new( +// Initialize GDAL early +GDALSetup.InitializeMultiplatform(); + +// Create root command +RootCommand rootCommand = new("FDA Testing Utility - Regression Testing Tool for FDA Studies"); + +// ============ COMPUTE COMMAND ============ +Command computeCommand = new("compute", "Run computations on FDA studies and generate result files"); + +Option computeConfigOption = new( name: "--config", description: "Path to JSON configuration file") { IsRequired = true }; -configOption.AddAlias("-c"); +computeConfigOption.AddAlias("-c"); -Option outputOption = new( +Option computeOutputOption = new( name: "--output", - description: "Output directory for results", + description: "Output directory for generated files", getDefaultValue: () => new DirectoryInfo(Environment.CurrentDirectory)); -outputOption.AddAlias("-o"); +computeOutputOption.AddAlias("-o"); -Option verboseOption = new( - name: "--verbose", - description: "Enable verbose output", - getDefaultValue: () => false); -verboseOption.AddAlias("-v"); - -Option studyOption = new( +Option computeStudyOption = new( name: "--study", description: "Filter to specific study IDs (can specify multiple)") { AllowMultipleArgumentsPerToken = true }; -studyOption.AddAlias("-s"); +computeStudyOption.AddAlias("-s"); -// Create root command -RootCommand rootCommand = new("FDA Testing Utility - Regression Testing Tool for FDA Studies"); -rootCommand.AddOption(configOption); -rootCommand.AddOption(outputOption); -rootCommand.AddOption(verboseOption); -rootCommand.AddOption(studyOption); +computeCommand.AddOption(computeConfigOption); +computeCommand.AddOption(computeOutputOption); +computeCommand.AddOption(computeStudyOption); -// Set handler -rootCommand.SetHandler(async (configFile, outputDir, verbose, studyFilter) => +computeCommand.SetHandler(async (configFile, outputDir, studyFilter) => { try { - Console.WriteLine("FDA Testing Utility v1.0"); - Console.WriteLine("========================"); + Console.WriteLine("FDA Testing Utility - Compute"); + Console.WriteLine("============================="); Console.WriteLine(); - // Initialize GDAL for spatial operations - GDALSetup.InitializeMultiplatform(); - - // Load configuration if (!configFile.Exists) { Console.WriteLine($"Error: Configuration file not found: {configFile.FullName}"); @@ -55,19 +49,16 @@ } Console.WriteLine($"Loading configuration: {configFile.FullName}"); - var config = TestConfiguration.LoadFromFile(configFile.FullName); + TestConfiguration config = TestConfiguration.LoadFromFile(configFile.FullName); - // Ensure output directory exists if (!outputDir.Exists) { outputDir.Create(); } - // Create and run test runner - TestRunner runner = new( + ComputeRunner runner = new( config, outputDir.FullName, - verbose, studyFilter?.Length > 0 ? studyFilter : null); int exitCode = await runner.RunAsync(); @@ -76,13 +67,83 @@ catch (Exception ex) { Console.WriteLine($"Fatal error: {ex.Message}"); - if (verbose) + Console.WriteLine(ex.StackTrace); + Environment.Exit(1); + } +}, computeConfigOption, computeOutputOption, computeStudyOption); + +// ============ COMPARE COMMAND ============ +Command compareCommand = new("compare", "Compare two sets of computed results and generate a comparison report"); + +Option baselineOption = new( + name: "--baseline", + description: "Directory containing baseline result files") +{ IsRequired = true }; +baselineOption.AddAlias("-b"); + +Option newResultsOption = new( + name: "--new", + description: "Directory containing new result files to compare against baseline") +{ IsRequired = true }; +newResultsOption.AddAlias("-n"); + +Option compareOutputOption = new( + name: "--output", + description: "Output file for comparison report", + getDefaultValue: () => new FileInfo(Path.Combine(Environment.CurrentDirectory, "comparison_report.txt"))); +compareOutputOption.AddAlias("-o"); + +Option toleranceOption = new( + name: "--tolerance", + description: "Relative tolerance for numeric comparisons (default: 0.01 = 1%)", + getDefaultValue: () => 0.01); +toleranceOption.AddAlias("-t"); + +compareCommand.AddOption(baselineOption); +compareCommand.AddOption(newResultsOption); +compareCommand.AddOption(compareOutputOption); +compareCommand.AddOption(toleranceOption); + +compareCommand.SetHandler((baselineDir, newDir, outputFile, tolerance) => +{ + try + { + Console.WriteLine("FDA Testing Utility - Compare"); + Console.WriteLine("============================="); + Console.WriteLine(); + + if (!baselineDir.Exists) { - Console.WriteLine(ex.StackTrace); + Console.WriteLine($"Error: Baseline directory not found: {baselineDir.FullName}"); + Environment.Exit(1); } + + if (!newDir.Exists) + { + Console.WriteLine($"Error: New results directory not found: {newDir.FullName}"); + Environment.Exit(1); + } + + CompareRunner runner = new( + baselineDir.FullName, + newDir.FullName, + outputFile.FullName, + tolerance); + + int exitCode = runner.Run(); + Environment.Exit(exitCode); + } + catch (Exception ex) + { + Console.WriteLine($"Fatal error: {ex.Message}"); + Console.WriteLine(ex.StackTrace); Environment.Exit(1); } -}, configOption, outputOption, verboseOption, studyOption); +}, baselineOption, newResultsOption, compareOutputOption, toleranceOption); + +// Add subcommands to root +rootCommand.AddCommand(computeCommand); +rootCommand.AddCommand(compareCommand); // Run return await rootCommand.InvokeAsync(args); From e253bbb6eae341439a092b3dd5de7c7a1c6c87c7 Mon Sep 17 00:00:00 2001 From: Brennan Beam <64556723+Brennan1994@users.noreply.github.com> Date: Tue, 23 Dec 2025 17:31:58 -0800 Subject: [PATCH 10/13] fix compare to deserialize into model objects --- HEC.FDA.TestingUtility/CompareRunner.cs | 613 +++++++++++++++++++++--- 1 file changed, 540 insertions(+), 73 deletions(-) diff --git a/HEC.FDA.TestingUtility/CompareRunner.cs b/HEC.FDA.TestingUtility/CompareRunner.cs index 6a74f8cef..374ea4836 100644 --- a/HEC.FDA.TestingUtility/CompareRunner.cs +++ b/HEC.FDA.TestingUtility/CompareRunner.cs @@ -1,10 +1,13 @@ using System.Text; using System.Xml.Linq; +using HEC.FDA.Model.metrics; +using HEC.FDA.Model.paireddata; namespace HEC.FDA.TestingUtility; /// -/// Compares two sets of FDA computation results and generates a comparison report. +/// Compares two sets of FDA computation results by deserializing XML into model objects +/// and comparing the same statistics that CsvReportFactory outputs. /// public class CompareRunner { @@ -12,6 +15,7 @@ public class CompareRunner private readonly string _newDir; private readonly string _outputPath; private readonly double _tolerance; + private const double MinimumAbsoluteDifference = 1.0; // $1 minimum to report a difference public CompareRunner(string baselineDir, string newDir, string outputPath, double tolerance) { @@ -45,8 +49,8 @@ public int Run() } StringBuilder report = new(); - report.AppendLine("FDA Results Comparison Report"); - report.AppendLine("============================="); + report.AppendLine("FDA Results Comparison Report (Model-Based)"); + report.AppendLine("============================================"); report.AppendLine($"Generated: {DateTime.Now:yyyy-MM-dd HH:mm:ss}"); report.AppendLine($"Baseline: {_baselineDir}"); report.AppendLine($"New: {_newDir}"); @@ -123,27 +127,27 @@ private int CompareFiles(string baselinePath, string newPath, StringBuilder repo try { - XElement baseline = XElement.Load(baselinePath); - XElement newResults = XElement.Load(newPath); + XElement baselineDoc = XElement.Load(baselinePath); + XElement newDoc = XElement.Load(newPath); - string studyId = baseline.Attribute("studyId")?.Value ?? Path.GetFileNameWithoutExtension(baselinePath); + string studyId = baselineDoc.Attribute("studyId")?.Value ?? Path.GetFileNameWithoutExtension(baselinePath); report.AppendLine($"=== {studyId} ==="); // Compare scenarios - differences += CompareElements(baseline, newResults, "ScenarioResults", "name", report); + differences += CompareScenarioResults(baselineDoc, newDoc, report); // Compare alternatives - differences += CompareElements(baseline, newResults, "AlternativeResults", "name", report); + differences += CompareAlternativeResults(baselineDoc, newDoc, report); // Compare stage damage - differences += CompareElements(baseline, newResults, "StageDamage", "name", report); + differences += CompareStageDamageResults(baselineDoc, newDoc, report); // Compare alternative comparison reports - differences += CompareElements(baseline, newResults, "AlternativeComparisonReport", "name", report); + differences += CompareAlternativeComparisonResults(baselineDoc, newDoc, report); if (differences == 0) { - report.AppendLine(" All values match within tolerance."); + report.AppendLine(" All statistics match within tolerance."); } report.AppendLine(); } @@ -156,120 +160,583 @@ private int CompareFiles(string baselinePath, string newPath, StringBuilder repo return differences; } - private int CompareElements(XElement baseline, XElement newResults, string elementType, string nameAttribute, StringBuilder report) + #region Scenario Comparison + + private int CompareScenarioResults(XElement baselineDoc, XElement newDoc, StringBuilder report) { int differences = 0; + var baselineElements = baselineDoc.Elements("ScenarioResults").ToList(); + var newElements = newDoc.Elements("ScenarioResults").ToList(); + + foreach (var baselineWrapper in baselineElements) + { + string name = baselineWrapper.Attribute("name")?.Value ?? "unknown"; + var newWrapper = newElements.FirstOrDefault(e => e.Attribute("name")?.Value == name); + + if (newWrapper == null) + { + report.AppendLine($" MISSING: Scenario '{name}' not in new results"); + differences++; + continue; + } + + // Extract inner ScenarioResults element and deserialize + var baselineInner = baselineWrapper.Element("ScenarioResults"); + var newInner = newWrapper.Element("ScenarioResults"); + + if (baselineInner == null || newInner == null) + { + report.AppendLine($" ERROR: Invalid XML structure for Scenario '{name}'"); + differences++; + continue; + } - IEnumerable baselineElements = baseline.Elements(elementType); - IEnumerable newElements = newResults.Elements(elementType); + try + { + var baselineResults = ScenarioResults.ReadFromXML(baselineInner); + var newResults = ScenarioResults.ReadFromXML(newInner); + differences += CompareScenarioStatistics(name, baselineResults, newResults, report); + } + catch (Exception ex) + { + report.AppendLine($" ERROR: Failed to deserialize Scenario '{name}': {ex.Message}"); + differences++; + } + } - foreach (XElement baselineElem in baselineElements) + // Check for scenarios in new that aren't in baseline + foreach (var newWrapper in newElements) { - string name = baselineElem.Attribute(nameAttribute)?.Value ?? "unknown"; - XElement? newElem = newElements.FirstOrDefault(e => e.Attribute(nameAttribute)?.Value == name); + string name = newWrapper.Attribute("name")?.Value ?? "unknown"; + if (!baselineElements.Any(e => e.Attribute("name")?.Value == name)) + { + report.AppendLine($" NEW: Scenario '{name}' only in new results"); + } + } + + return differences; + } - if (newElem == null) + private int CompareScenarioStatistics(string scenarioName, ScenarioResults baseline, ScenarioResults actual, StringBuilder report) + { + int differences = 0; + report.AppendLine($" Scenario: {scenarioName}"); + + foreach (var iaResult in baseline.ResultsList) + { + int iaId = iaResult.ImpactAreaID; + + // Find corresponding impact area in actual + var actualIaResult = actual.ResultsList.FirstOrDefault(r => r.ImpactAreaID == iaId); + if (actualIaResult == null) { - report.AppendLine($" MISSING: {elementType} '{name}' not in new results"); + report.AppendLine($" MISSING: ImpactArea[{iaId}] not in new results"); differences++; continue; } - // Compare numeric attributes - List<(string path, double baseline, double newVal)> numericDiffs = new(); - CompareNumericValues(baselineElem, newElem, "", numericDiffs); + // Compare aggregate EAD metrics + differences += CompareValue($"ImpactArea[{iaId}].MeanEAD", + iaResult.MeanExpectedAnnualConsequences(), + actualIaResult.MeanExpectedAnnualConsequences(), + report); + + differences += CompareValue($"ImpactArea[{iaId}].EAD_25thPct", + iaResult.ConsequencesExceededWithProbabilityQ(0.75), + actualIaResult.ConsequencesExceededWithProbabilityQ(0.75), + report); + + differences += CompareValue($"ImpactArea[{iaId}].EAD_50thPct", + iaResult.ConsequencesExceededWithProbabilityQ(0.50), + actualIaResult.ConsequencesExceededWithProbabilityQ(0.50), + report); + + differences += CompareValue($"ImpactArea[{iaId}].EAD_75thPct", + iaResult.ConsequencesExceededWithProbabilityQ(0.25), + actualIaResult.ConsequencesExceededWithProbabilityQ(0.25), + report); + + // Compare damage by category + foreach (var consequence in iaResult.ConsequenceResults.ConsequenceResultList) + { + string damCat = consequence.DamageCategory; + string assetCat = consequence.AssetCategory; + + double baselineCatEAD = iaResult.MeanExpectedAnnualConsequences(iaId, damCat, assetCat); + double actualCatEAD = actualIaResult.MeanExpectedAnnualConsequences(iaId, damCat, assetCat); - foreach ((string path, double baselineVal, double newVal) in numericDiffs) + differences += CompareValue($"ImpactArea[{iaId}].{damCat}.{assetCat}.MeanEAD", + baselineCatEAD, actualCatEAD, report); + } + + // Compare performance metrics for each threshold + foreach (var threshold in iaResult.PerformanceByThresholds.ListOfThresholds) { - double absDiff = Math.Abs(baselineVal - newVal); - double relDiff = baselineVal != 0 ? absDiff / Math.Abs(baselineVal) : absDiff; + int thresholdId = threshold.ThresholdID; - if (relDiff > _tolerance && absDiff > 1.0) // At least $1 difference + try { - differences++; - report.AppendLine($" DIFF: {elementType} '{name}' {path}"); - report.AppendLine($" Baseline: {baselineVal:F4}"); - report.AppendLine($" New: {newVal:F4}"); - report.AppendLine($" Diff: {absDiff:F4} ({relDiff:P2})"); + differences += CompareValue($"ImpactArea[{iaId}].Threshold[{thresholdId}].MeanAEP", + iaResult.MeanAEP(thresholdId), + actualIaResult.MeanAEP(thresholdId), + report); + + differences += CompareValue($"ImpactArea[{iaId}].Threshold[{thresholdId}].MedianAEP", + iaResult.MedianAEP(thresholdId), + actualIaResult.MedianAEP(thresholdId), + report); + + // Assurance metrics + differences += CompareValue($"ImpactArea[{iaId}].Threshold[{thresholdId}].Assurance_0.10", + iaResult.AssuranceOfEvent(thresholdId, 0.10), + actualIaResult.AssuranceOfEvent(thresholdId, 0.10), + report); + + differences += CompareValue($"ImpactArea[{iaId}].Threshold[{thresholdId}].Assurance_0.04", + iaResult.AssuranceOfEvent(thresholdId, 0.04), + actualIaResult.AssuranceOfEvent(thresholdId, 0.04), + report); + + differences += CompareValue($"ImpactArea[{iaId}].Threshold[{thresholdId}].Assurance_0.02", + iaResult.AssuranceOfEvent(thresholdId, 0.02), + actualIaResult.AssuranceOfEvent(thresholdId, 0.02), + report); + + differences += CompareValue($"ImpactArea[{iaId}].Threshold[{thresholdId}].Assurance_0.01", + iaResult.AssuranceOfEvent(thresholdId, 0.01), + actualIaResult.AssuranceOfEvent(thresholdId, 0.01), + report); + + // Long-term risk + differences += CompareValue($"ImpactArea[{iaId}].Threshold[{thresholdId}].LTRisk_10yr", + iaResult.LongTermExceedanceProbability(thresholdId, 10), + actualIaResult.LongTermExceedanceProbability(thresholdId, 10), + report); + + differences += CompareValue($"ImpactArea[{iaId}].Threshold[{thresholdId}].LTRisk_30yr", + iaResult.LongTermExceedanceProbability(thresholdId, 30), + actualIaResult.LongTermExceedanceProbability(thresholdId, 30), + report); + + differences += CompareValue($"ImpactArea[{iaId}].Threshold[{thresholdId}].LTRisk_50yr", + iaResult.LongTermExceedanceProbability(thresholdId, 50), + actualIaResult.LongTermExceedanceProbability(thresholdId, 50), + report); + } + catch (Exception) + { + // Threshold may not exist in actual - skip silently } } } - // Check for elements in new that aren't in baseline - foreach (XElement newElem in newElements) + return differences; + } + + #endregion + + #region Alternative Comparison + + private int CompareAlternativeResults(XElement baselineDoc, XElement newDoc, StringBuilder report) + { + int differences = 0; + var baselineElements = baselineDoc.Elements("AlternativeResults").ToList(); + var newElements = newDoc.Elements("AlternativeResults").ToList(); + + foreach (var baselineWrapper in baselineElements) { - string name = newElem.Attribute(nameAttribute)?.Value ?? "unknown"; - XElement? baselineElem = baselineElements.FirstOrDefault(e => e.Attribute(nameAttribute)?.Value == name); + string name = baselineWrapper.Attribute("name")?.Value ?? "unknown"; + var newWrapper = newElements.FirstOrDefault(e => e.Attribute("name")?.Value == name); + + if (newWrapper == null) + { + report.AppendLine($" MISSING: Alternative '{name}' not in new results"); + differences++; + continue; + } - if (baselineElem == null) + try { - report.AppendLine($" NEW: {elementType} '{name}' only in new results"); + // AlternativeResults stores BaseYearResults and FutureYearResults as ScenarioResults + // We compare the underlying scenarios since EqAD is derived from them + differences += CompareAlternativeStatistics(name, baselineWrapper, newWrapper, report); + } + catch (Exception ex) + { + report.AppendLine($" ERROR: Failed to compare Alternative '{name}': {ex.Message}"); + differences++; } } return differences; } - private void CompareNumericValues(XElement baseline, XElement newElem, string path, List<(string, double, double)> diffs) + private int CompareAlternativeStatistics(string altName, XElement baselineWrapper, XElement newWrapper, StringBuilder report) { - // Compare attributes - foreach (XAttribute attr in baseline.Attributes()) + int differences = 0; + report.AppendLine($" Alternative: {altName}"); + + // Deserialize the base year and future year scenario results + var baselineBaseYearXml = baselineWrapper.Element("BaseYearResults")?.Element("ScenarioResults"); + var newBaseYearXml = newWrapper.Element("BaseYearResults")?.Element("ScenarioResults"); + var baselineFutureYearXml = baselineWrapper.Element("FutureYearResults")?.Element("ScenarioResults"); + var newFutureYearXml = newWrapper.Element("FutureYearResults")?.Element("ScenarioResults"); + + if (baselineBaseYearXml == null || newBaseYearXml == null) + { + report.AppendLine($" ERROR: Missing BaseYearResults"); + return 1; + } + + if (baselineFutureYearXml == null || newFutureYearXml == null) + { + report.AppendLine($" ERROR: Missing FutureYearResults"); + return 1; + } + + var baselineBaseYear = ScenarioResults.ReadFromXML(baselineBaseYearXml); + var newBaseYear = ScenarioResults.ReadFromXML(newBaseYearXml); + var baselineFutureYear = ScenarioResults.ReadFromXML(baselineFutureYearXml); + var newFutureYear = ScenarioResults.ReadFromXML(newFutureYearXml); + + // Compare base year results + report.AppendLine($" BaseYear:"); + foreach (var iaResult in baselineBaseYear.ResultsList) { - if (double.TryParse(attr.Value, out double baselineVal)) + int iaId = iaResult.ImpactAreaID; + var actualIaResult = newBaseYear.ResultsList.FirstOrDefault(r => r.ImpactAreaID == iaId); + + if (actualIaResult == null) { - XAttribute? newAttr = newElem.Attribute(attr.Name); - if (newAttr != null && double.TryParse(newAttr.Value, out double newVal)) - { - string attrPath = string.IsNullOrEmpty(path) ? $"@{attr.Name}" : $"{path}/@{attr.Name}"; - diffs.Add((attrPath, baselineVal, newVal)); - } + report.AppendLine($" MISSING: ImpactArea[{iaId}] not in new results"); + differences++; + continue; + } + + differences += CompareValue($"ImpactArea[{iaId}].MeanEAD", + iaResult.MeanExpectedAnnualConsequences(), + actualIaResult.MeanExpectedAnnualConsequences(), + report); + + // Compare by category + foreach (var consequence in iaResult.ConsequenceResults.ConsequenceResultList) + { + string damCat = consequence.DamageCategory; + string assetCat = consequence.AssetCategory; + + differences += CompareValue($"ImpactArea[{iaId}].{damCat}.{assetCat}.MeanEAD", + iaResult.MeanExpectedAnnualConsequences(iaId, damCat, assetCat), + actualIaResult.MeanExpectedAnnualConsequences(iaId, damCat, assetCat), + report); } } - // Compare child elements recursively - foreach (XElement baselineChild in baseline.Elements()) + // Compare future year results + report.AppendLine($" FutureYear:"); + foreach (var iaResult in baselineFutureYear.ResultsList) { - string childName = baselineChild.Name.LocalName; + int iaId = iaResult.ImpactAreaID; + var actualIaResult = newFutureYear.ResultsList.FirstOrDefault(r => r.ImpactAreaID == iaId); - // Try to find matching child by element name and any identifying attributes - XElement? matchingChild = FindMatchingChild(newElem, baselineChild); + if (actualIaResult == null) + { + report.AppendLine($" MISSING: ImpactArea[{iaId}] not in new results"); + differences++; + continue; + } + + differences += CompareValue($"ImpactArea[{iaId}].MeanEAD", + iaResult.MeanExpectedAnnualConsequences(), + actualIaResult.MeanExpectedAnnualConsequences(), + report); - if (matchingChild != null) + // Compare by category + foreach (var consequence in iaResult.ConsequenceResults.ConsequenceResultList) { - string childPath = string.IsNullOrEmpty(path) ? childName : $"{path}/{childName}"; + string damCat = consequence.DamageCategory; + string assetCat = consequence.AssetCategory; - // Add identifying attributes to path if present - string? id = baselineChild.Attribute("id")?.Value ?? baselineChild.Attribute("name")?.Value; - if (id != null) - { - childPath += $"[{id}]"; - } + differences += CompareValue($"ImpactArea[{iaId}].{damCat}.{assetCat}.MeanEAD", + iaResult.MeanExpectedAnnualConsequences(iaId, damCat, assetCat), + actualIaResult.MeanExpectedAnnualConsequences(iaId, damCat, assetCat), + report); + } + } + + return differences; + } + + #endregion + + #region Stage Damage Comparison + + private int CompareStageDamageResults(XElement baselineDoc, XElement newDoc, StringBuilder report) + { + int differences = 0; + var baselineElements = baselineDoc.Elements("StageDamage").ToList(); + var newElements = newDoc.Elements("StageDamage").ToList(); + + foreach (var baselineWrapper in baselineElements) + { + string name = baselineWrapper.Attribute("name")?.Value ?? "unknown"; + var newWrapper = newElements.FirstOrDefault(e => e.Attribute("name")?.Value == name); + + if (newWrapper == null) + { + report.AppendLine($" MISSING: StageDamage '{name}' not in new results"); + differences++; + continue; + } + + try + { + var baselineCurves = DeserializeStageDamageCurves(baselineWrapper); + var newCurves = DeserializeStageDamageCurves(newWrapper); - CompareNumericValues(baselineChild, matchingChild, childPath, diffs); + differences += CompareStageDamageStatistics(name, baselineCurves, newCurves, report); + } + catch (Exception ex) + { + report.AppendLine($" ERROR: Failed to deserialize StageDamage '{name}': {ex.Message}"); + differences++; } } + + return differences; + } + + private List DeserializeStageDamageCurves(XElement wrapper) + { + var curves = new List(); + var curvesElement = wrapper.Element("Curves"); + + if (curvesElement == null) + return curves; + + foreach (var curveElement in curvesElement.Elements("UncertainPairedData")) + { + curves.Add(UncertainPairedData.ReadFromXML(curveElement)); + } + + return curves; } - private static XElement? FindMatchingChild(XElement parent, XElement target) + private int CompareStageDamageStatistics(string name, List baseline, List actual, StringBuilder report) { - string targetName = target.Name.LocalName; - IEnumerable candidates = parent.Elements(targetName); + int differences = 0; + report.AppendLine($" StageDamage: {name}"); - // Try to match by id or name attribute - string? targetId = target.Attribute("id")?.Value; - string? targetNameAttr = target.Attribute("name")?.Value; + if (baseline.Count != actual.Count) + { + report.AppendLine($" DIFF: CurveCount - Baseline: {baseline.Count}, New: {actual.Count}"); + differences++; + return differences; + } - if (targetId != null) + for (int i = 0; i < baseline.Count; i++) { - return candidates.FirstOrDefault(c => c.Attribute("id")?.Value == targetId); + var baselineCurve = baseline[i]; + var actualCurve = actual[i]; + + string curveId = $"Curve[IA={baselineCurve.ImpactAreaID},Dam={baselineCurve.DamageCategory},Asset={baselineCurve.AssetCategory}]"; + + // Compare point count + if (baselineCurve.Xvals.Length != actualCurve.Xvals.Length) + { + report.AppendLine($" DIFF: {curveId}.PointCount - Baseline: {baselineCurve.Xvals.Length}, New: {actualCurve.Xvals.Length}"); + differences++; + continue; + } + + // Compare min/max stages + if (baselineCurve.Xvals.Length > 0) + { + differences += CompareValue($"{curveId}.MinStage", + baselineCurve.Xvals.Min(), + actualCurve.Xvals.Min(), + report); + + differences += CompareValue($"{curveId}.MaxStage", + baselineCurve.Xvals.Max(), + actualCurve.Xvals.Max(), + report); + + // Compare median damage at each stage + for (int j = 0; j < baselineCurve.Xvals.Length; j++) + { + double stage = baselineCurve.Xvals[j]; + double baselineMedian = baselineCurve.Yvals[j].InverseCDF(0.5); + double actualMedian = actualCurve.Yvals[j].InverseCDF(0.5); + + differences += CompareValue($"{curveId}.Stage[{stage:F2}].MedianDamage", + baselineMedian, actualMedian, report); + } + } + } + + return differences; + } + + #endregion + + #region Alternative Comparison Report + + private int CompareAlternativeComparisonResults(XElement baselineDoc, XElement newDoc, StringBuilder report) + { + int differences = 0; + var baselineElements = baselineDoc.Elements("AlternativeComparisonReport").ToList(); + var newElements = newDoc.Elements("AlternativeComparisonReport").ToList(); + + foreach (var baselineWrapper in baselineElements) + { + string name = baselineWrapper.Attribute("name")?.Value ?? "unknown"; + var newWrapper = newElements.FirstOrDefault(e => e.Attribute("name")?.Value == name); + + if (newWrapper == null) + { + report.AppendLine($" MISSING: AlternativeComparisonReport '{name}' not in new results"); + differences++; + continue; + } + + try + { + // Compare statistics directly from XML attributes (stored pre-computed) + differences += CompareAlternativeComparisonStatistics(name, baselineWrapper, newWrapper, report); + } + catch (Exception ex) + { + report.AppendLine($" ERROR: Failed to compare AlternativeComparisonReport '{name}': {ex.Message}"); + differences++; + } } - if (targetNameAttr != null) + return differences; + } + + private int CompareAlternativeComparisonStatistics(string reportName, XElement baselineWrapper, XElement newWrapper, StringBuilder report) + { + int differences = 0; + report.AppendLine($" AlternativeComparisonReport: {reportName}"); + + // Iterate over with-project alternatives in baseline + foreach (var baselineAltElement in baselineWrapper.Elements("WithProjectAlternative")) { - return candidates.FirstOrDefault(c => c.Attribute("name")?.Value == targetNameAttr); + string altId = baselineAltElement.Attribute("id")?.Value ?? "0"; + string altName = baselineAltElement.Attribute("name")?.Value ?? $"Alt{altId}"; + + var newAltElement = newWrapper.Elements("WithProjectAlternative") + .FirstOrDefault(e => e.Attribute("id")?.Value == altId); + + if (newAltElement == null) + { + report.AppendLine($" MISSING: Alternative '{altName}' not in new results"); + differences++; + continue; + } + + // Compare each impact area + foreach (var baselineIaElement in baselineAltElement.Elements("ImpactArea")) + { + string iaId = baselineIaElement.Attribute("id")?.Value ?? "0"; + + var newIaElement = newAltElement.Elements("ImpactArea") + .FirstOrDefault(e => e.Attribute("id")?.Value == iaId); + + if (newIaElement == null) + { + report.AppendLine($" MISSING: Alt[{altName}].ImpactArea[{iaId}] not in new results"); + differences++; + continue; + } + + string prefix = $"Alt[{altName}].IA[{iaId}]"; + + // Compare aggregate metrics stored as attributes + differences += CompareXmlAttribute($"{prefix}.EqADReduced", + baselineIaElement, newIaElement, "eqadReduced", report); + + differences += CompareXmlAttribute($"{prefix}.BaseEADReduced", + baselineIaElement, newIaElement, "baseEadReduced", report); + + differences += CompareXmlAttribute($"{prefix}.FutureEADReduced", + baselineIaElement, newIaElement, "futureEadReduced", report); + + // Compare category breakdowns + foreach (var baselineCatElement in baselineIaElement.Elements("Category")) + { + string damCat = baselineCatElement.Attribute("damageCategory")?.Value ?? ""; + string assetCat = baselineCatElement.Attribute("assetCategory")?.Value ?? ""; + + var newCatElement = newIaElement.Elements("Category") + .FirstOrDefault(e => + e.Attribute("damageCategory")?.Value == damCat && + e.Attribute("assetCategory")?.Value == assetCat); + + if (newCatElement == null) + { + report.AppendLine($" MISSING: {prefix}.{damCat}.{assetCat} not in new results"); + differences++; + continue; + } + + differences += CompareXmlAttribute($"{prefix}.{damCat}.{assetCat}.EqADReduced", + baselineCatElement, newCatElement, "eqadReduced", report); + + differences += CompareXmlAttribute($"{prefix}.{damCat}.{assetCat}.BaseEADReduced", + baselineCatElement, newCatElement, "baseEadReduced", report); + + differences += CompareXmlAttribute($"{prefix}.{damCat}.{assetCat}.FutureEADReduced", + baselineCatElement, newCatElement, "futureEadReduced", report); + } + } } - // If no identifying attributes, just take the first one with the same name - return candidates.FirstOrDefault(); + return differences; } + + private int CompareXmlAttribute(string metricName, XElement baseline, XElement actual, string attrName, StringBuilder report) + { + if (!double.TryParse(baseline.Attribute(attrName)?.Value, out double baselineVal)) + return 0; + if (!double.TryParse(actual.Attribute(attrName)?.Value, out double actualVal)) + return 0; + + return CompareValue(metricName, baselineVal, actualVal, report); + } + + #endregion + + #region Value Comparison Helpers + + private int CompareValue(string metricName, double baseline, double actual, StringBuilder report) + { + if (ValuesAreEqual(baseline, actual)) + return 0; + + double absDiff = Math.Abs(baseline - actual); + double relDiff = baseline != 0 ? absDiff / Math.Abs(baseline) : (actual != 0 ? 1.0 : 0); + + report.AppendLine($" DIFF: {metricName}"); + report.AppendLine($" Baseline: {baseline:F4}"); + report.AppendLine($" New: {actual:F4}"); + report.AppendLine($" Diff: {absDiff:F4} ({relDiff:P2})"); + + return 1; + } + + private bool ValuesAreEqual(double baseline, double actual) + { + double absoluteDiff = Math.Abs(baseline - actual); + + // If the absolute difference is less than minimum threshold, consider equal + if (absoluteDiff <= MinimumAbsoluteDifference) + return true; + + // Check relative tolerance + double relativeDiff = baseline != 0 ? absoluteDiff / Math.Abs(baseline) : absoluteDiff; + return relativeDiff <= _tolerance; + } + + #endregion } From 1578789ccc7eadb22d28b5b719937889831f7f66 Mon Sep 17 00:00:00 2001 From: Brennan Beam <64556723+Brennan1994@users.noreply.github.com> Date: Tue, 23 Dec 2025 22:56:06 -0800 Subject: [PATCH 11/13] remove comparison. focus on report. --- HEC.FDA.TestingUtility/CompareRunner.cs | 742 ------------------ .../Comparison/ComparisonResult.cs | 42 - .../Comparison/StudyBaselineWriter.cs | 110 --- .../Comparison/XmlResultComparer.cs | 361 --------- HEC.FDA.TestingUtility/ComputeRunner.cs | 43 +- HEC.FDA.TestingUtility/Program.cs | 72 +- .../Reporting/CsvReportFactory.cs | 360 +++++---- .../Services/StudyLoader.cs | 8 + .../Results/AssuranceOfAEPRowItem.cs | 15 + .../Results/ScenarioPerformanceRowItem.cs | 17 + 10 files changed, 253 insertions(+), 1517 deletions(-) delete mode 100644 HEC.FDA.TestingUtility/CompareRunner.cs delete mode 100644 HEC.FDA.TestingUtility/Comparison/ComparisonResult.cs delete mode 100644 HEC.FDA.TestingUtility/Comparison/StudyBaselineWriter.cs delete mode 100644 HEC.FDA.TestingUtility/Comparison/XmlResultComparer.cs diff --git a/HEC.FDA.TestingUtility/CompareRunner.cs b/HEC.FDA.TestingUtility/CompareRunner.cs deleted file mode 100644 index 374ea4836..000000000 --- a/HEC.FDA.TestingUtility/CompareRunner.cs +++ /dev/null @@ -1,742 +0,0 @@ -using System.Text; -using System.Xml.Linq; -using HEC.FDA.Model.metrics; -using HEC.FDA.Model.paireddata; - -namespace HEC.FDA.TestingUtility; - -/// -/// Compares two sets of FDA computation results by deserializing XML into model objects -/// and comparing the same statistics that CsvReportFactory outputs. -/// -public class CompareRunner -{ - private readonly string _baselineDir; - private readonly string _newDir; - private readonly string _outputPath; - private readonly double _tolerance; - private const double MinimumAbsoluteDifference = 1.0; // $1 minimum to report a difference - - public CompareRunner(string baselineDir, string newDir, string outputPath, double tolerance) - { - _baselineDir = baselineDir; - _newDir = newDir; - _outputPath = outputPath; - _tolerance = tolerance; - } - - public int Run() - { - Console.WriteLine($"Baseline directory: {_baselineDir}"); - Console.WriteLine($"New results directory: {_newDir}"); - Console.WriteLine($"Tolerance: {_tolerance:P1}"); - Console.WriteLine(); - - // Find all XML result files in both directories - string[] baselineFiles = Directory.GetFiles(_baselineDir, "*_results.xml"); - string[] newFiles = Directory.GetFiles(_newDir, "*_results.xml"); - - if (baselineFiles.Length == 0) - { - Console.WriteLine("No baseline result files (*_results.xml) found."); - return 1; - } - - if (newFiles.Length == 0) - { - Console.WriteLine("No new result files (*_results.xml) found."); - return 1; - } - - StringBuilder report = new(); - report.AppendLine("FDA Results Comparison Report (Model-Based)"); - report.AppendLine("============================================"); - report.AppendLine($"Generated: {DateTime.Now:yyyy-MM-dd HH:mm:ss}"); - report.AppendLine($"Baseline: {_baselineDir}"); - report.AppendLine($"New: {_newDir}"); - report.AppendLine($"Tolerance: {_tolerance:P1}"); - report.AppendLine(); - - int totalDifferences = 0; - int filesCompared = 0; - - // Match files by study ID - foreach (string baselineFile in baselineFiles) - { - string fileName = Path.GetFileName(baselineFile); - string newFile = Path.Combine(_newDir, fileName); - - if (!File.Exists(newFile)) - { - report.AppendLine($"MISSING: {fileName} not found in new results"); - Console.WriteLine($"MISSING: {fileName}"); - totalDifferences++; - continue; - } - - Console.WriteLine($"Comparing: {fileName}"); - int differences = CompareFiles(baselineFile, newFile, report); - totalDifferences += differences; - filesCompared++; - - if (differences == 0) - { - Console.WriteLine($" MATCH: No differences found"); - } - else - { - Console.WriteLine($" DIFF: {differences} difference(s) found"); - } - } - - // Check for files in new that aren't in baseline - foreach (string newFile in newFiles) - { - string fileName = Path.GetFileName(newFile); - string baselineFile = Path.Combine(_baselineDir, fileName); - - if (!File.Exists(baselineFile)) - { - report.AppendLine($"NEW: {fileName} exists only in new results"); - Console.WriteLine($"NEW: {fileName}"); - } - } - - // Summary - report.AppendLine(); - report.AppendLine("=== Summary ==="); - report.AppendLine($"Files compared: {filesCompared}"); - report.AppendLine($"Total differences: {totalDifferences}"); - report.AppendLine($"Result: {(totalDifferences == 0 ? "PASS" : "FAIL")}"); - - // Save report - File.WriteAllText(_outputPath, report.ToString()); - Console.WriteLine(); - Console.WriteLine($"Report saved to: {_outputPath}"); - Console.WriteLine(); - Console.WriteLine($"=== Result: {(totalDifferences == 0 ? "PASS" : "FAIL")} ==="); - Console.WriteLine($"Files compared: {filesCompared}"); - Console.WriteLine($"Total differences: {totalDifferences}"); - - return totalDifferences > 0 ? 1 : 0; - } - - private int CompareFiles(string baselinePath, string newPath, StringBuilder report) - { - int differences = 0; - - try - { - XElement baselineDoc = XElement.Load(baselinePath); - XElement newDoc = XElement.Load(newPath); - - string studyId = baselineDoc.Attribute("studyId")?.Value ?? Path.GetFileNameWithoutExtension(baselinePath); - report.AppendLine($"=== {studyId} ==="); - - // Compare scenarios - differences += CompareScenarioResults(baselineDoc, newDoc, report); - - // Compare alternatives - differences += CompareAlternativeResults(baselineDoc, newDoc, report); - - // Compare stage damage - differences += CompareStageDamageResults(baselineDoc, newDoc, report); - - // Compare alternative comparison reports - differences += CompareAlternativeComparisonResults(baselineDoc, newDoc, report); - - if (differences == 0) - { - report.AppendLine(" All statistics match within tolerance."); - } - report.AppendLine(); - } - catch (Exception ex) - { - report.AppendLine($" ERROR: {ex.Message}"); - differences++; - } - - return differences; - } - - #region Scenario Comparison - - private int CompareScenarioResults(XElement baselineDoc, XElement newDoc, StringBuilder report) - { - int differences = 0; - var baselineElements = baselineDoc.Elements("ScenarioResults").ToList(); - var newElements = newDoc.Elements("ScenarioResults").ToList(); - - foreach (var baselineWrapper in baselineElements) - { - string name = baselineWrapper.Attribute("name")?.Value ?? "unknown"; - var newWrapper = newElements.FirstOrDefault(e => e.Attribute("name")?.Value == name); - - if (newWrapper == null) - { - report.AppendLine($" MISSING: Scenario '{name}' not in new results"); - differences++; - continue; - } - - // Extract inner ScenarioResults element and deserialize - var baselineInner = baselineWrapper.Element("ScenarioResults"); - var newInner = newWrapper.Element("ScenarioResults"); - - if (baselineInner == null || newInner == null) - { - report.AppendLine($" ERROR: Invalid XML structure for Scenario '{name}'"); - differences++; - continue; - } - - try - { - var baselineResults = ScenarioResults.ReadFromXML(baselineInner); - var newResults = ScenarioResults.ReadFromXML(newInner); - differences += CompareScenarioStatistics(name, baselineResults, newResults, report); - } - catch (Exception ex) - { - report.AppendLine($" ERROR: Failed to deserialize Scenario '{name}': {ex.Message}"); - differences++; - } - } - - // Check for scenarios in new that aren't in baseline - foreach (var newWrapper in newElements) - { - string name = newWrapper.Attribute("name")?.Value ?? "unknown"; - if (!baselineElements.Any(e => e.Attribute("name")?.Value == name)) - { - report.AppendLine($" NEW: Scenario '{name}' only in new results"); - } - } - - return differences; - } - - private int CompareScenarioStatistics(string scenarioName, ScenarioResults baseline, ScenarioResults actual, StringBuilder report) - { - int differences = 0; - report.AppendLine($" Scenario: {scenarioName}"); - - foreach (var iaResult in baseline.ResultsList) - { - int iaId = iaResult.ImpactAreaID; - - // Find corresponding impact area in actual - var actualIaResult = actual.ResultsList.FirstOrDefault(r => r.ImpactAreaID == iaId); - if (actualIaResult == null) - { - report.AppendLine($" MISSING: ImpactArea[{iaId}] not in new results"); - differences++; - continue; - } - - // Compare aggregate EAD metrics - differences += CompareValue($"ImpactArea[{iaId}].MeanEAD", - iaResult.MeanExpectedAnnualConsequences(), - actualIaResult.MeanExpectedAnnualConsequences(), - report); - - differences += CompareValue($"ImpactArea[{iaId}].EAD_25thPct", - iaResult.ConsequencesExceededWithProbabilityQ(0.75), - actualIaResult.ConsequencesExceededWithProbabilityQ(0.75), - report); - - differences += CompareValue($"ImpactArea[{iaId}].EAD_50thPct", - iaResult.ConsequencesExceededWithProbabilityQ(0.50), - actualIaResult.ConsequencesExceededWithProbabilityQ(0.50), - report); - - differences += CompareValue($"ImpactArea[{iaId}].EAD_75thPct", - iaResult.ConsequencesExceededWithProbabilityQ(0.25), - actualIaResult.ConsequencesExceededWithProbabilityQ(0.25), - report); - - // Compare damage by category - foreach (var consequence in iaResult.ConsequenceResults.ConsequenceResultList) - { - string damCat = consequence.DamageCategory; - string assetCat = consequence.AssetCategory; - - double baselineCatEAD = iaResult.MeanExpectedAnnualConsequences(iaId, damCat, assetCat); - double actualCatEAD = actualIaResult.MeanExpectedAnnualConsequences(iaId, damCat, assetCat); - - differences += CompareValue($"ImpactArea[{iaId}].{damCat}.{assetCat}.MeanEAD", - baselineCatEAD, actualCatEAD, report); - } - - // Compare performance metrics for each threshold - foreach (var threshold in iaResult.PerformanceByThresholds.ListOfThresholds) - { - int thresholdId = threshold.ThresholdID; - - try - { - differences += CompareValue($"ImpactArea[{iaId}].Threshold[{thresholdId}].MeanAEP", - iaResult.MeanAEP(thresholdId), - actualIaResult.MeanAEP(thresholdId), - report); - - differences += CompareValue($"ImpactArea[{iaId}].Threshold[{thresholdId}].MedianAEP", - iaResult.MedianAEP(thresholdId), - actualIaResult.MedianAEP(thresholdId), - report); - - // Assurance metrics - differences += CompareValue($"ImpactArea[{iaId}].Threshold[{thresholdId}].Assurance_0.10", - iaResult.AssuranceOfEvent(thresholdId, 0.10), - actualIaResult.AssuranceOfEvent(thresholdId, 0.10), - report); - - differences += CompareValue($"ImpactArea[{iaId}].Threshold[{thresholdId}].Assurance_0.04", - iaResult.AssuranceOfEvent(thresholdId, 0.04), - actualIaResult.AssuranceOfEvent(thresholdId, 0.04), - report); - - differences += CompareValue($"ImpactArea[{iaId}].Threshold[{thresholdId}].Assurance_0.02", - iaResult.AssuranceOfEvent(thresholdId, 0.02), - actualIaResult.AssuranceOfEvent(thresholdId, 0.02), - report); - - differences += CompareValue($"ImpactArea[{iaId}].Threshold[{thresholdId}].Assurance_0.01", - iaResult.AssuranceOfEvent(thresholdId, 0.01), - actualIaResult.AssuranceOfEvent(thresholdId, 0.01), - report); - - // Long-term risk - differences += CompareValue($"ImpactArea[{iaId}].Threshold[{thresholdId}].LTRisk_10yr", - iaResult.LongTermExceedanceProbability(thresholdId, 10), - actualIaResult.LongTermExceedanceProbability(thresholdId, 10), - report); - - differences += CompareValue($"ImpactArea[{iaId}].Threshold[{thresholdId}].LTRisk_30yr", - iaResult.LongTermExceedanceProbability(thresholdId, 30), - actualIaResult.LongTermExceedanceProbability(thresholdId, 30), - report); - - differences += CompareValue($"ImpactArea[{iaId}].Threshold[{thresholdId}].LTRisk_50yr", - iaResult.LongTermExceedanceProbability(thresholdId, 50), - actualIaResult.LongTermExceedanceProbability(thresholdId, 50), - report); - } - catch (Exception) - { - // Threshold may not exist in actual - skip silently - } - } - } - - return differences; - } - - #endregion - - #region Alternative Comparison - - private int CompareAlternativeResults(XElement baselineDoc, XElement newDoc, StringBuilder report) - { - int differences = 0; - var baselineElements = baselineDoc.Elements("AlternativeResults").ToList(); - var newElements = newDoc.Elements("AlternativeResults").ToList(); - - foreach (var baselineWrapper in baselineElements) - { - string name = baselineWrapper.Attribute("name")?.Value ?? "unknown"; - var newWrapper = newElements.FirstOrDefault(e => e.Attribute("name")?.Value == name); - - if (newWrapper == null) - { - report.AppendLine($" MISSING: Alternative '{name}' not in new results"); - differences++; - continue; - } - - try - { - // AlternativeResults stores BaseYearResults and FutureYearResults as ScenarioResults - // We compare the underlying scenarios since EqAD is derived from them - differences += CompareAlternativeStatistics(name, baselineWrapper, newWrapper, report); - } - catch (Exception ex) - { - report.AppendLine($" ERROR: Failed to compare Alternative '{name}': {ex.Message}"); - differences++; - } - } - - return differences; - } - - private int CompareAlternativeStatistics(string altName, XElement baselineWrapper, XElement newWrapper, StringBuilder report) - { - int differences = 0; - report.AppendLine($" Alternative: {altName}"); - - // Deserialize the base year and future year scenario results - var baselineBaseYearXml = baselineWrapper.Element("BaseYearResults")?.Element("ScenarioResults"); - var newBaseYearXml = newWrapper.Element("BaseYearResults")?.Element("ScenarioResults"); - var baselineFutureYearXml = baselineWrapper.Element("FutureYearResults")?.Element("ScenarioResults"); - var newFutureYearXml = newWrapper.Element("FutureYearResults")?.Element("ScenarioResults"); - - if (baselineBaseYearXml == null || newBaseYearXml == null) - { - report.AppendLine($" ERROR: Missing BaseYearResults"); - return 1; - } - - if (baselineFutureYearXml == null || newFutureYearXml == null) - { - report.AppendLine($" ERROR: Missing FutureYearResults"); - return 1; - } - - var baselineBaseYear = ScenarioResults.ReadFromXML(baselineBaseYearXml); - var newBaseYear = ScenarioResults.ReadFromXML(newBaseYearXml); - var baselineFutureYear = ScenarioResults.ReadFromXML(baselineFutureYearXml); - var newFutureYear = ScenarioResults.ReadFromXML(newFutureYearXml); - - // Compare base year results - report.AppendLine($" BaseYear:"); - foreach (var iaResult in baselineBaseYear.ResultsList) - { - int iaId = iaResult.ImpactAreaID; - var actualIaResult = newBaseYear.ResultsList.FirstOrDefault(r => r.ImpactAreaID == iaId); - - if (actualIaResult == null) - { - report.AppendLine($" MISSING: ImpactArea[{iaId}] not in new results"); - differences++; - continue; - } - - differences += CompareValue($"ImpactArea[{iaId}].MeanEAD", - iaResult.MeanExpectedAnnualConsequences(), - actualIaResult.MeanExpectedAnnualConsequences(), - report); - - // Compare by category - foreach (var consequence in iaResult.ConsequenceResults.ConsequenceResultList) - { - string damCat = consequence.DamageCategory; - string assetCat = consequence.AssetCategory; - - differences += CompareValue($"ImpactArea[{iaId}].{damCat}.{assetCat}.MeanEAD", - iaResult.MeanExpectedAnnualConsequences(iaId, damCat, assetCat), - actualIaResult.MeanExpectedAnnualConsequences(iaId, damCat, assetCat), - report); - } - } - - // Compare future year results - report.AppendLine($" FutureYear:"); - foreach (var iaResult in baselineFutureYear.ResultsList) - { - int iaId = iaResult.ImpactAreaID; - var actualIaResult = newFutureYear.ResultsList.FirstOrDefault(r => r.ImpactAreaID == iaId); - - if (actualIaResult == null) - { - report.AppendLine($" MISSING: ImpactArea[{iaId}] not in new results"); - differences++; - continue; - } - - differences += CompareValue($"ImpactArea[{iaId}].MeanEAD", - iaResult.MeanExpectedAnnualConsequences(), - actualIaResult.MeanExpectedAnnualConsequences(), - report); - - // Compare by category - foreach (var consequence in iaResult.ConsequenceResults.ConsequenceResultList) - { - string damCat = consequence.DamageCategory; - string assetCat = consequence.AssetCategory; - - differences += CompareValue($"ImpactArea[{iaId}].{damCat}.{assetCat}.MeanEAD", - iaResult.MeanExpectedAnnualConsequences(iaId, damCat, assetCat), - actualIaResult.MeanExpectedAnnualConsequences(iaId, damCat, assetCat), - report); - } - } - - return differences; - } - - #endregion - - #region Stage Damage Comparison - - private int CompareStageDamageResults(XElement baselineDoc, XElement newDoc, StringBuilder report) - { - int differences = 0; - var baselineElements = baselineDoc.Elements("StageDamage").ToList(); - var newElements = newDoc.Elements("StageDamage").ToList(); - - foreach (var baselineWrapper in baselineElements) - { - string name = baselineWrapper.Attribute("name")?.Value ?? "unknown"; - var newWrapper = newElements.FirstOrDefault(e => e.Attribute("name")?.Value == name); - - if (newWrapper == null) - { - report.AppendLine($" MISSING: StageDamage '{name}' not in new results"); - differences++; - continue; - } - - try - { - var baselineCurves = DeserializeStageDamageCurves(baselineWrapper); - var newCurves = DeserializeStageDamageCurves(newWrapper); - - differences += CompareStageDamageStatistics(name, baselineCurves, newCurves, report); - } - catch (Exception ex) - { - report.AppendLine($" ERROR: Failed to deserialize StageDamage '{name}': {ex.Message}"); - differences++; - } - } - - return differences; - } - - private List DeserializeStageDamageCurves(XElement wrapper) - { - var curves = new List(); - var curvesElement = wrapper.Element("Curves"); - - if (curvesElement == null) - return curves; - - foreach (var curveElement in curvesElement.Elements("UncertainPairedData")) - { - curves.Add(UncertainPairedData.ReadFromXML(curveElement)); - } - - return curves; - } - - private int CompareStageDamageStatistics(string name, List baseline, List actual, StringBuilder report) - { - int differences = 0; - report.AppendLine($" StageDamage: {name}"); - - if (baseline.Count != actual.Count) - { - report.AppendLine($" DIFF: CurveCount - Baseline: {baseline.Count}, New: {actual.Count}"); - differences++; - return differences; - } - - for (int i = 0; i < baseline.Count; i++) - { - var baselineCurve = baseline[i]; - var actualCurve = actual[i]; - - string curveId = $"Curve[IA={baselineCurve.ImpactAreaID},Dam={baselineCurve.DamageCategory},Asset={baselineCurve.AssetCategory}]"; - - // Compare point count - if (baselineCurve.Xvals.Length != actualCurve.Xvals.Length) - { - report.AppendLine($" DIFF: {curveId}.PointCount - Baseline: {baselineCurve.Xvals.Length}, New: {actualCurve.Xvals.Length}"); - differences++; - continue; - } - - // Compare min/max stages - if (baselineCurve.Xvals.Length > 0) - { - differences += CompareValue($"{curveId}.MinStage", - baselineCurve.Xvals.Min(), - actualCurve.Xvals.Min(), - report); - - differences += CompareValue($"{curveId}.MaxStage", - baselineCurve.Xvals.Max(), - actualCurve.Xvals.Max(), - report); - - // Compare median damage at each stage - for (int j = 0; j < baselineCurve.Xvals.Length; j++) - { - double stage = baselineCurve.Xvals[j]; - double baselineMedian = baselineCurve.Yvals[j].InverseCDF(0.5); - double actualMedian = actualCurve.Yvals[j].InverseCDF(0.5); - - differences += CompareValue($"{curveId}.Stage[{stage:F2}].MedianDamage", - baselineMedian, actualMedian, report); - } - } - } - - return differences; - } - - #endregion - - #region Alternative Comparison Report - - private int CompareAlternativeComparisonResults(XElement baselineDoc, XElement newDoc, StringBuilder report) - { - int differences = 0; - var baselineElements = baselineDoc.Elements("AlternativeComparisonReport").ToList(); - var newElements = newDoc.Elements("AlternativeComparisonReport").ToList(); - - foreach (var baselineWrapper in baselineElements) - { - string name = baselineWrapper.Attribute("name")?.Value ?? "unknown"; - var newWrapper = newElements.FirstOrDefault(e => e.Attribute("name")?.Value == name); - - if (newWrapper == null) - { - report.AppendLine($" MISSING: AlternativeComparisonReport '{name}' not in new results"); - differences++; - continue; - } - - try - { - // Compare statistics directly from XML attributes (stored pre-computed) - differences += CompareAlternativeComparisonStatistics(name, baselineWrapper, newWrapper, report); - } - catch (Exception ex) - { - report.AppendLine($" ERROR: Failed to compare AlternativeComparisonReport '{name}': {ex.Message}"); - differences++; - } - } - - return differences; - } - - private int CompareAlternativeComparisonStatistics(string reportName, XElement baselineWrapper, XElement newWrapper, StringBuilder report) - { - int differences = 0; - report.AppendLine($" AlternativeComparisonReport: {reportName}"); - - // Iterate over with-project alternatives in baseline - foreach (var baselineAltElement in baselineWrapper.Elements("WithProjectAlternative")) - { - string altId = baselineAltElement.Attribute("id")?.Value ?? "0"; - string altName = baselineAltElement.Attribute("name")?.Value ?? $"Alt{altId}"; - - var newAltElement = newWrapper.Elements("WithProjectAlternative") - .FirstOrDefault(e => e.Attribute("id")?.Value == altId); - - if (newAltElement == null) - { - report.AppendLine($" MISSING: Alternative '{altName}' not in new results"); - differences++; - continue; - } - - // Compare each impact area - foreach (var baselineIaElement in baselineAltElement.Elements("ImpactArea")) - { - string iaId = baselineIaElement.Attribute("id")?.Value ?? "0"; - - var newIaElement = newAltElement.Elements("ImpactArea") - .FirstOrDefault(e => e.Attribute("id")?.Value == iaId); - - if (newIaElement == null) - { - report.AppendLine($" MISSING: Alt[{altName}].ImpactArea[{iaId}] not in new results"); - differences++; - continue; - } - - string prefix = $"Alt[{altName}].IA[{iaId}]"; - - // Compare aggregate metrics stored as attributes - differences += CompareXmlAttribute($"{prefix}.EqADReduced", - baselineIaElement, newIaElement, "eqadReduced", report); - - differences += CompareXmlAttribute($"{prefix}.BaseEADReduced", - baselineIaElement, newIaElement, "baseEadReduced", report); - - differences += CompareXmlAttribute($"{prefix}.FutureEADReduced", - baselineIaElement, newIaElement, "futureEadReduced", report); - - // Compare category breakdowns - foreach (var baselineCatElement in baselineIaElement.Elements("Category")) - { - string damCat = baselineCatElement.Attribute("damageCategory")?.Value ?? ""; - string assetCat = baselineCatElement.Attribute("assetCategory")?.Value ?? ""; - - var newCatElement = newIaElement.Elements("Category") - .FirstOrDefault(e => - e.Attribute("damageCategory")?.Value == damCat && - e.Attribute("assetCategory")?.Value == assetCat); - - if (newCatElement == null) - { - report.AppendLine($" MISSING: {prefix}.{damCat}.{assetCat} not in new results"); - differences++; - continue; - } - - differences += CompareXmlAttribute($"{prefix}.{damCat}.{assetCat}.EqADReduced", - baselineCatElement, newCatElement, "eqadReduced", report); - - differences += CompareXmlAttribute($"{prefix}.{damCat}.{assetCat}.BaseEADReduced", - baselineCatElement, newCatElement, "baseEadReduced", report); - - differences += CompareXmlAttribute($"{prefix}.{damCat}.{assetCat}.FutureEADReduced", - baselineCatElement, newCatElement, "futureEadReduced", report); - } - } - } - - return differences; - } - - private int CompareXmlAttribute(string metricName, XElement baseline, XElement actual, string attrName, StringBuilder report) - { - if (!double.TryParse(baseline.Attribute(attrName)?.Value, out double baselineVal)) - return 0; - if (!double.TryParse(actual.Attribute(attrName)?.Value, out double actualVal)) - return 0; - - return CompareValue(metricName, baselineVal, actualVal, report); - } - - #endregion - - #region Value Comparison Helpers - - private int CompareValue(string metricName, double baseline, double actual, StringBuilder report) - { - if (ValuesAreEqual(baseline, actual)) - return 0; - - double absDiff = Math.Abs(baseline - actual); - double relDiff = baseline != 0 ? absDiff / Math.Abs(baseline) : (actual != 0 ? 1.0 : 0); - - report.AppendLine($" DIFF: {metricName}"); - report.AppendLine($" Baseline: {baseline:F4}"); - report.AppendLine($" New: {actual:F4}"); - report.AppendLine($" Diff: {absDiff:F4} ({relDiff:P2})"); - - return 1; - } - - private bool ValuesAreEqual(double baseline, double actual) - { - double absoluteDiff = Math.Abs(baseline - actual); - - // If the absolute difference is less than minimum threshold, consider equal - if (absoluteDiff <= MinimumAbsoluteDifference) - return true; - - // Check relative tolerance - double relativeDiff = baseline != 0 ? absoluteDiff / Math.Abs(baseline) : absoluteDiff; - return relativeDiff <= _tolerance; - } - - #endregion -} diff --git a/HEC.FDA.TestingUtility/Comparison/ComparisonResult.cs b/HEC.FDA.TestingUtility/Comparison/ComparisonResult.cs deleted file mode 100644 index 35db2910d..000000000 --- a/HEC.FDA.TestingUtility/Comparison/ComparisonResult.cs +++ /dev/null @@ -1,42 +0,0 @@ -namespace HEC.FDA.TestingUtility.Comparison; - -public class ComparisonResult -{ - public string ElementName { get; set; } = string.Empty; - public string ElementType { get; set; } = string.Empty; - public bool Passed { get; set; } = true; - public string? ErrorMessage { get; set; } - public List Differences { get; } = new(); - - public string Summary => Passed - ? "All values match" - : ErrorMessage ?? $"{Differences.Count} difference(s) found"; -} - -public class Difference -{ - public string Metric { get; set; } = string.Empty; - public double? Expected { get; set; } - public double? Actual { get; set; } - public string? ExpectedDescription { get; set; } - public string? ActualDescription { get; set; } - public double? AbsoluteDifference => Expected.HasValue && Actual.HasValue - ? Math.Abs(Expected.Value - Actual.Value) - : null; - public double? PercentDifference => Expected.HasValue && Actual.HasValue && Expected.Value != 0 - ? Math.Abs((Expected.Value - Actual.Value) / Expected.Value) * 100 - : null; - - public override string ToString() - { - if (Expected.HasValue && Actual.HasValue) - { - return $"{Metric}: Expected={Expected:F4}, Actual={Actual:F4}, Diff={AbsoluteDifference:F4} ({PercentDifference:F2}%)"; - } - if (ExpectedDescription != null || ActualDescription != null) - { - return $"{Metric}: Expected={ExpectedDescription ?? "null"}, Actual={ActualDescription ?? "null"}"; - } - return $"{Metric}: Expected={Expected}, Actual={Actual}"; - } -} diff --git a/HEC.FDA.TestingUtility/Comparison/StudyBaselineWriter.cs b/HEC.FDA.TestingUtility/Comparison/StudyBaselineWriter.cs deleted file mode 100644 index 681353896..000000000 --- a/HEC.FDA.TestingUtility/Comparison/StudyBaselineWriter.cs +++ /dev/null @@ -1,110 +0,0 @@ -using System.Xml.Linq; -using HEC.FDA.Model.metrics; -using HEC.FDA.Model.paireddata; - -namespace HEC.FDA.TestingUtility.Comparison; - -public static class StudyBaselineWriter -{ - public static XElement CreateStudyBaseline(string studyId, string studyName) - { - return new XElement("StudyBaseline", - new XAttribute("studyId", studyId), - new XAttribute("studyName", studyName), - new XAttribute("createdDate", DateTime.Now.ToString("yyyy-MM-dd"))); - } - - public static void AddScenarioResults(XElement baseline, string name, ScenarioResults results) - { - XElement wrapper = new("ScenarioResults", - new XAttribute("name", name), - results.WriteToXML()); - baseline.Add(wrapper); - } - - public static void AddAlternativeResults(XElement baseline, string name, AlternativeResults results) - { - XElement wrapper = new("AlternativeResults", - new XAttribute("name", name), - new XElement("BaseYearResults", results.BaseYearScenarioResults.WriteToXML()), - new XElement("FutureYearResults", results.FutureYearScenarioResults.WriteToXML())); - baseline.Add(wrapper); - } - - public static void AddStageDamage(XElement baseline, string name, List curves) - { - XElement curvesElement = new("Curves"); - foreach (UncertainPairedData curve in curves) - { - curvesElement.Add(curve.WriteToXML()); - } - - XElement wrapper = new("StageDamage", - new XAttribute("name", name), - new XAttribute("curveCount", curves.Count), - curvesElement); - baseline.Add(wrapper); - } - - public static void AddAlternativeComparisonResults(XElement baseline, string name, AlternativeComparisonReportResults results, List<(int altId, string altName)> withProjectAlternatives) - { - if (results == null) return; - - XElement wrapper = new("AlternativeComparisonReport", - new XAttribute("name", name)); - - List impactAreaIds = results.GetImpactAreaIDs(); - List damageCategories = results.GetDamageCategories(); - List assetCategories = results.GetAssetCategories(); - - foreach ((int altId, string altName) in withProjectAlternatives) - { - XElement altElement = new("WithProjectAlternative", - new XAttribute("id", altId), - new XAttribute("name", altName)); - - foreach (int impactAreaId in impactAreaIds) - { - XElement iaElement = new("ImpactArea", - new XAttribute("id", impactAreaId), - new XAttribute("eqadReduced", results.SampleMeanEqadReduced(altId, impactAreaId)), - new XAttribute("baseEadReduced", results.SampleMeanBaseYearEADReduced(altId, impactAreaId)), - new XAttribute("futureEadReduced", results.SampleMeanFutureYearEADReduced(altId, impactAreaId))); - - // Add category breakdowns - foreach (string damCat in damageCategories) - { - foreach (string assetCat in assetCategories) - { - double eqadReduced = results.SampleMeanEqadReduced(altId, impactAreaId, damCat, assetCat); - if (eqadReduced != 0) - { - iaElement.Add(new XElement("Category", - new XAttribute("damageCategory", damCat), - new XAttribute("assetCategory", assetCat), - new XAttribute("eqadReduced", eqadReduced), - new XAttribute("baseEadReduced", results.SampleMeanBaseYearEADReduced(altId, impactAreaId, damCat, assetCat)), - new XAttribute("futureEadReduced", results.SampleMeanFutureYearEADReduced(altId, impactAreaId, damCat, assetCat)))); - } - } - } - - altElement.Add(iaElement); - } - - wrapper.Add(altElement); - } - - baseline.Add(wrapper); - } - - public static void Save(XElement baseline, string path) - { - string? directory = Path.GetDirectoryName(path); - if (!string.IsNullOrEmpty(directory) && !Directory.Exists(directory)) - { - Directory.CreateDirectory(directory); - } - baseline.Save(path); - } -} diff --git a/HEC.FDA.TestingUtility/Comparison/XmlResultComparer.cs b/HEC.FDA.TestingUtility/Comparison/XmlResultComparer.cs deleted file mode 100644 index 0350c4532..000000000 --- a/HEC.FDA.TestingUtility/Comparison/XmlResultComparer.cs +++ /dev/null @@ -1,361 +0,0 @@ -using System.Xml.Linq; -using HEC.FDA.Model.metrics; -using HEC.FDA.Model.paireddata; - -namespace HEC.FDA.TestingUtility.Comparison; - -public class XmlResultComparer -{ - private const double RelativeTolerance = 0.01; // 1% relative tolerance - private const double MinimumAbsoluteDifference = 1.0; // $1 minimum to consider a difference - - private XElement? _baselineDoc; - - public void LoadBaseline(string baselinePath) - { - if (!File.Exists(baselinePath)) - { - throw new FileNotFoundException($"Baseline file not found: {baselinePath}"); - } - _baselineDoc = XElement.Load(baselinePath); - } - - public ComparisonResult CompareScenarioResults(string elementName, ScenarioResults actual) - { - ComparisonResult result = new() { ElementName = elementName, ElementType = "Scenario" }; - - if (_baselineDoc == null) - { - result.Passed = false; - result.ErrorMessage = "Baseline not loaded"; - return result; - } - - // Find the baseline element by name - var baselineElement = _baselineDoc.Elements("ScenarioResults") - .FirstOrDefault(e => e.Attribute("name")?.Value == elementName); - - if (baselineElement == null) - { - result.Passed = false; - result.ErrorMessage = $"Baseline not found for Scenario '{elementName}'"; - return result; - } - - // Get the inner ScenarioResults XML - var innerXml = baselineElement.Element("ScenarioResults"); - if (innerXml == null) - { - result.Passed = false; - result.ErrorMessage = $"Invalid baseline format for Scenario '{elementName}'"; - return result; - } - - ScenarioResults baseline = ScenarioResults.ReadFromXML(innerXml); - - // Use built-in Equals method - result.Passed = actual.Equals(baseline); - - if (!result.Passed) - { - GenerateScenarioDiff(baseline, actual, result); - } - - return result; - } - - public ComparisonResult CompareAlternativeResults(string elementName, AlternativeResults actual) - { - ComparisonResult result = new() { ElementName = elementName, ElementType = "Alternative" }; - - if (_baselineDoc == null) - { - result.Passed = false; - result.ErrorMessage = "Baseline not loaded"; - return result; - } - - var baselineElement = _baselineDoc.Elements("AlternativeResults") - .FirstOrDefault(e => e.Attribute("name")?.Value == elementName); - - if (baselineElement == null) - { - result.Passed = false; - result.ErrorMessage = $"Baseline not found for Alternative '{elementName}'"; - return result; - } - - result.Passed = true; - - // Compare base year scenario results - var baseYearXml = baselineElement.Element("BaseYearResults")?.Element("ScenarioResults"); - if (baseYearXml != null) - { - var baselineBaseYear = ScenarioResults.ReadFromXML(baseYearXml); - bool baseYearMatch = actual.BaseYearScenarioResults.Equals(baselineBaseYear); - result.Passed &= baseYearMatch; - - if (!baseYearMatch) - { - result.Differences.Add(new Difference - { - Metric = "BaseYearResults", - Expected = null, - Actual = null - }); - } - } - - // Compare future year scenario results - var futureYearXml = baselineElement.Element("FutureYearResults")?.Element("ScenarioResults"); - if (futureYearXml != null) - { - var baselineFutureYear = ScenarioResults.ReadFromXML(futureYearXml); - bool futureYearMatch = actual.FutureYearScenarioResults.Equals(baselineFutureYear); - result.Passed &= futureYearMatch; - - if (!futureYearMatch) - { - result.Differences.Add(new Difference - { - Metric = "FutureYearResults", - Expected = null, - Actual = null - }); - } - } - - return result; - } - - public ComparisonResult CompareStageDamage(string elementName, List actualCurves) - { - ComparisonResult result = new() { ElementName = elementName, ElementType = "StageDamage" }; - - if (_baselineDoc == null) - { - result.Passed = false; - result.ErrorMessage = "Baseline not loaded"; - return result; - } - - var baselineElement = _baselineDoc.Elements("StageDamage") - .FirstOrDefault(e => e.Attribute("name")?.Value == elementName); - - if (baselineElement == null) - { - result.Passed = false; - result.ErrorMessage = $"Baseline not found for StageDamage '{elementName}'"; - return result; - } - - // Get baseline curves - var curvesElement = baselineElement.Element("Curves"); - if (curvesElement == null) - { - result.Passed = false; - result.ErrorMessage = $"Invalid baseline format for StageDamage '{elementName}' - no Curves element"; - return result; - } - - var baselineCurveElements = curvesElement.Elements("UncertainPairedData").ToList(); - - // Compare curve counts - if (baselineCurveElements.Count != actualCurves.Count) - { - result.Passed = false; - result.Differences.Add(new Difference - { - Metric = "CurveCount", - Expected = baselineCurveElements.Count, - Actual = actualCurves.Count - }); - return result; - } - - result.Passed = true; - - // Compare each curve - for (int i = 0; i < actualCurves.Count; i++) - { - var baselineCurve = UncertainPairedData.ReadFromXML(baselineCurveElements[i]); - var actualCurve = actualCurves[i]; - - // Compare metadata - if (baselineCurve.ImpactAreaID != actualCurve.ImpactAreaID || - baselineCurve.DamageCategory != actualCurve.DamageCategory || - baselineCurve.AssetCategory != actualCurve.AssetCategory) - { - result.Passed = false; - result.Differences.Add(new Difference - { - Metric = $"Curve[{i}].Metadata", - ExpectedDescription = $"IA={baselineCurve.ImpactAreaID}, DamCat={baselineCurve.DamageCategory}, Asset={baselineCurve.AssetCategory}", - ActualDescription = $"IA={actualCurve.ImpactAreaID}, DamCat={actualCurve.DamageCategory}, Asset={actualCurve.AssetCategory}" - }); - continue; - } - - // Compare X values (stages) - if (baselineCurve.Xvals.Length != actualCurve.Xvals.Length) - { - result.Passed = false; - result.Differences.Add(new Difference - { - Metric = $"Curve[{i}].XValueCount", - Expected = baselineCurve.Xvals.Length, - Actual = actualCurve.Xvals.Length - }); - continue; - } - - // Compare mean damage values at each stage - for (int j = 0; j < baselineCurve.Xvals.Length; j++) - { - double baselineMean = baselineCurve.Yvals[j].InverseCDF(0.5); - double actualMean = actualCurve.Yvals[j].InverseCDF(0.5); - - if (!ValuesAreEqual(baselineMean, actualMean)) - { - result.Passed = false; - result.Differences.Add(new Difference - { - Metric = $"Curve[{i}].Stage[{baselineCurve.Xvals[j]:F2}].MedianDamage", - Expected = baselineMean, - Actual = actualMean - }); - } - } - } - - return result; - } - - private static bool ValuesAreEqual(double expected, double actual) - { - double absoluteDiff = Math.Abs(expected - actual); - if (absoluteDiff <= MinimumAbsoluteDifference) return true; - - double relativeDiff = expected != 0 ? absoluteDiff / Math.Abs(expected) : absoluteDiff; - return relativeDiff <= RelativeTolerance; - } - - private static void GenerateScenarioDiff(ScenarioResults baseline, ScenarioResults actual, ComparisonResult result) - { - foreach (int iaId in baseline.GetImpactAreaIDs()) - { - foreach (string damCat in baseline.GetDamageCategories()) - { - foreach (string assetCat in baseline.GetAssetCategories()) - { - double baselineMean = baseline.SampleMeanExpectedAnnualConsequences(iaId, damCat, assetCat); - double actualMean = actual.SampleMeanExpectedAnnualConsequences(iaId, damCat, assetCat); - - if (!ValuesAreEqual(baselineMean, actualMean)) - { - result.Differences.Add(new Difference - { - Metric = $"MeanEAD[IA={iaId},DamCat={damCat},Asset={assetCat}]", - Expected = baselineMean, - Actual = actualMean - }); - } - } - } - } - } - - public ComparisonResult CompareAlternativeComparisonResults(string elementName, AlternativeComparisonReportResults actual, List<(int altId, string altName)> withProjectAlternatives) - { - ComparisonResult result = new() { ElementName = elementName, ElementType = "AlternativeComparisonReport" }; - - if (_baselineDoc == null) - { - result.Passed = false; - result.ErrorMessage = "Baseline not loaded"; - return result; - } - - var baselineElement = _baselineDoc.Elements("AlternativeComparisonReport") - .FirstOrDefault(e => e.Attribute("name")?.Value == elementName); - - if (baselineElement == null) - { - result.Passed = false; - result.ErrorMessage = $"Baseline not found for AlternativeComparisonReport '{elementName}'"; - return result; - } - - result.Passed = true; - - foreach (var (altId, altName) in withProjectAlternatives) - { - var baselineAltElement = baselineElement.Elements("WithProjectAlternative") - .FirstOrDefault(e => e.Attribute("id")?.Value == altId.ToString()); - - if (baselineAltElement == null) - { - result.Passed = false; - result.Differences.Add(new Difference - { - Metric = $"WithProjectAlternative[{altName}]", - ExpectedDescription = "present in baseline", - ActualDescription = "missing" - }); - continue; - } - - foreach (var baselineIaElement in baselineAltElement.Elements("ImpactArea")) - { - int impactAreaId = int.Parse(baselineIaElement.Attribute("id")?.Value ?? "0"); - - // Compare EqAD Reduced - double baselineEqadReduced = double.Parse(baselineIaElement.Attribute("eqadReduced")?.Value ?? "0"); - double actualEqadReduced = actual.SampleMeanEqadReduced(altId, impactAreaId); - - if (!ValuesAreEqual(baselineEqadReduced, actualEqadReduced)) - { - result.Passed = false; - result.Differences.Add(new Difference - { - Metric = $"EqadReduced[Alt={altName},IA={impactAreaId}]", - Expected = baselineEqadReduced, - Actual = actualEqadReduced - }); - } - - // Compare Base EAD Reduced - double baselineBaseEadReduced = double.Parse(baselineIaElement.Attribute("baseEadReduced")?.Value ?? "0"); - double actualBaseEadReduced = actual.SampleMeanBaseYearEADReduced(altId, impactAreaId); - - if (!ValuesAreEqual(baselineBaseEadReduced, actualBaseEadReduced)) - { - result.Passed = false; - result.Differences.Add(new Difference - { - Metric = $"BaseEadReduced[Alt={altName},IA={impactAreaId}]", - Expected = baselineBaseEadReduced, - Actual = actualBaseEadReduced - }); - } - - // Compare Future EAD Reduced - double baselineFutureEadReduced = double.Parse(baselineIaElement.Attribute("futureEadReduced")?.Value ?? "0"); - double actualFutureEadReduced = actual.SampleMeanFutureYearEADReduced(altId, impactAreaId); - - if (!ValuesAreEqual(baselineFutureEadReduced, actualFutureEadReduced)) - { - result.Passed = false; - result.Differences.Add(new Difference - { - Metric = $"FutureEadReduced[Alt={altName},IA={impactAreaId}]", - Expected = baselineFutureEadReduced, - Actual = actualFutureEadReduced - }); - } - } - } - - return result; - } -} diff --git a/HEC.FDA.TestingUtility/ComputeRunner.cs b/HEC.FDA.TestingUtility/ComputeRunner.cs index 2a0ecdf60..45c033f30 100644 --- a/HEC.FDA.TestingUtility/ComputeRunner.cs +++ b/HEC.FDA.TestingUtility/ComputeRunner.cs @@ -1,8 +1,6 @@ using System.Diagnostics; -using System.Xml.Linq; using HEC.FDA.Model.metrics; using HEC.FDA.Model.paireddata; -using HEC.FDA.TestingUtility.Comparison; using HEC.FDA.TestingUtility.Configuration; using HEC.FDA.TestingUtility.Reporting; using HEC.FDA.TestingUtility.Services; @@ -19,8 +17,7 @@ namespace HEC.FDA.TestingUtility; /// -/// Runs FDA computations and generates result files (XML and CSV). -/// Does not perform comparisons - use CompareRunner for that. +/// Runs FDA computations and generates CSV result reports. /// public class ComputeRunner { @@ -80,8 +77,6 @@ public async Task RunAsync() using StudyLoader loader = new(); loader.LoadStudy(study.NetworkSourcePath, _config.GlobalSettings.LocalTempDirectory); - XElement computedResults = StudyBaselineWriter.CreateStudyBaseline(study.StudyId, study.StudyName); - List computations = BuildComputationList(study); Console.WriteLine($" Found {computations.Count} computations to run."); @@ -97,27 +92,23 @@ public async Task RunAsync() case "stagedamage": List sdCurves = StageDamageRunner.RunStageDamage(compute.ElementName); SaveStageDamageResults(compute.ElementName, sdCurves); - StudyBaselineWriter.AddStageDamage(computedResults, compute.ElementName, sdCurves); _csvReportFactory.AddStageDamageSummary(study.StudyId, compute.ElementName, sdCurves); break; case "scenario": ScenarioResults scenarioResults = ScenarioRunner.RunScenario(compute.ElementName, _cts.Token); - SaveScenarioResults(compute.ElementName, scenarioResults); - StudyBaselineWriter.AddScenarioResults(computedResults, compute.ElementName, scenarioResults); - _csvReportFactory.AddScenarioResults(study.StudyId, compute.ElementName, scenarioResults); + IASElement scenarioElement = SaveScenarioResults(compute.ElementName, scenarioResults); + _csvReportFactory.AddScenarioResults(study.StudyId, scenarioElement); break; case "alternative": AlternativeResults altResults = AlternativeRunner.RunAlternative(compute.ElementName, _cts.Token); - SaveAlternativeResults(compute.ElementName, altResults); - StudyBaselineWriter.AddAlternativeResults(computedResults, compute.ElementName, altResults); - _csvReportFactory.AddAlternativeResults(study.StudyId, compute.ElementName, altResults); + AlternativeElement altElement = SaveAlternativeResults(compute.ElementName, altResults); + _csvReportFactory.AddAlternativeResults(study.StudyId, altElement); break; case "alternativecomparison": (AlternativeComparisonReportResults compResults, List<(int altId, string altName)> withProjAlts) = RunAlternativeComparisonWithMetadata(compute.ElementName, _cts.Token); - StudyBaselineWriter.AddAlternativeComparisonResults(computedResults, compute.ElementName, compResults, withProjAlts); _csvReportFactory.AddAlternativeComparisonResults(study.StudyId, compute.ElementName, compResults, withProjAlts); break; @@ -139,9 +130,6 @@ public async Task RunAsync() Console.WriteLine($" {ex.StackTrace}"); } } - - // Save results - SaveResults(computedResults, study); } catch (OperationCanceledException) { @@ -177,14 +165,6 @@ public async Task RunAsync() string csvPath = Path.Combine(_outputDir, "results_report.csv"); _csvReportFactory.SaveReport(csvPath); - Console.WriteLine(); - Console.WriteLine("Generated files:"); - foreach ((string studyId, _, _, _) in studyTimings) - { - Console.WriteLine($" {studyId}_results.xml"); - } - Console.WriteLine($" results_report.csv"); - return errors > 0 ? 1 : 0; } @@ -299,27 +279,22 @@ private static (AlternativeComparisonReportResults results, List<(int altId, str return (results, withProjectAlternatives); } - private void SaveResults(XElement computedResults, StudyConfiguration study) - { - string outputPath = Path.Combine(_outputDir, $"{study.StudyId}_results.xml"); - StudyBaselineWriter.Save(computedResults, outputPath); - Console.WriteLine($" Results saved to: {outputPath}"); - } - - private static void SaveScenarioResults(string elementName, ScenarioResults results) + private static IASElement SaveScenarioResults(string elementName, ScenarioResults results) { IASElement element = ScenarioRunner.FindElement(elementName); element.Results = results; PersistenceFactory.GetIASManager().SaveExisting(element); Console.WriteLine($" Saved to temp database."); + return element; } - private static void SaveAlternativeResults(string elementName, AlternativeResults results) + private static AlternativeElement SaveAlternativeResults(string elementName, AlternativeResults results) { AlternativeElement element = ScenarioRunner.FindElement(elementName); element.Results = results; PersistenceFactory.GetElementManager().SaveExisting(element); Console.WriteLine($" Saved to temp database."); + return element; } private static void SaveStageDamageResults(string elementName, List curves) diff --git a/HEC.FDA.TestingUtility/Program.cs b/HEC.FDA.TestingUtility/Program.cs index c887b7d95..f27b6ffe0 100644 --- a/HEC.FDA.TestingUtility/Program.cs +++ b/HEC.FDA.TestingUtility/Program.cs @@ -10,7 +10,7 @@ RootCommand rootCommand = new("FDA Testing Utility - Regression Testing Tool for FDA Studies"); // ============ COMPUTE COMMAND ============ -Command computeCommand = new("compute", "Run computations on FDA studies and generate result files"); +Command computeCommand = new("compute", "Run computations on FDA studies and generate CSV result reports"); Option computeConfigOption = new( name: "--config", @@ -72,78 +72,8 @@ } }, computeConfigOption, computeOutputOption, computeStudyOption); -// ============ COMPARE COMMAND ============ -Command compareCommand = new("compare", "Compare two sets of computed results and generate a comparison report"); - -Option baselineOption = new( - name: "--baseline", - description: "Directory containing baseline result files") -{ IsRequired = true }; -baselineOption.AddAlias("-b"); - -Option newResultsOption = new( - name: "--new", - description: "Directory containing new result files to compare against baseline") -{ IsRequired = true }; -newResultsOption.AddAlias("-n"); - -Option compareOutputOption = new( - name: "--output", - description: "Output file for comparison report", - getDefaultValue: () => new FileInfo(Path.Combine(Environment.CurrentDirectory, "comparison_report.txt"))); -compareOutputOption.AddAlias("-o"); - -Option toleranceOption = new( - name: "--tolerance", - description: "Relative tolerance for numeric comparisons (default: 0.01 = 1%)", - getDefaultValue: () => 0.01); -toleranceOption.AddAlias("-t"); - -compareCommand.AddOption(baselineOption); -compareCommand.AddOption(newResultsOption); -compareCommand.AddOption(compareOutputOption); -compareCommand.AddOption(toleranceOption); - -compareCommand.SetHandler((baselineDir, newDir, outputFile, tolerance) => -{ - try - { - Console.WriteLine("FDA Testing Utility - Compare"); - Console.WriteLine("============================="); - Console.WriteLine(); - - if (!baselineDir.Exists) - { - Console.WriteLine($"Error: Baseline directory not found: {baselineDir.FullName}"); - Environment.Exit(1); - } - - if (!newDir.Exists) - { - Console.WriteLine($"Error: New results directory not found: {newDir.FullName}"); - Environment.Exit(1); - } - - CompareRunner runner = new( - baselineDir.FullName, - newDir.FullName, - outputFile.FullName, - tolerance); - - int exitCode = runner.Run(); - Environment.Exit(exitCode); - } - catch (Exception ex) - { - Console.WriteLine($"Fatal error: {ex.Message}"); - Console.WriteLine(ex.StackTrace); - Environment.Exit(1); - } -}, baselineOption, newResultsOption, compareOutputOption, toleranceOption); - // Add subcommands to root rootCommand.AddCommand(computeCommand); -rootCommand.AddCommand(compareCommand); // Run return await rootCommand.InvokeAsync(args); diff --git a/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs b/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs index 56faa2b86..968612730 100644 --- a/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs +++ b/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs @@ -1,181 +1,61 @@ using System.Text; using HEC.FDA.Model.metrics; using HEC.FDA.Model.paireddata; +using HEC.FDA.ViewModel.Alternatives; +using HEC.FDA.ViewModel.Alternatives.Results.BatchCompute; +using HEC.FDA.ViewModel.ImpactAreaScenario; +using HEC.FDA.ViewModel.Results; namespace HEC.FDA.TestingUtility.Reporting; /// /// Factory class that produces a comprehensive CSV report containing all tabularly /// visualized results from FDA computations across all studies. +/// Uses ViewModel row item classes for scenario and alternative results. /// public class CsvReportFactory { - private readonly StringBuilder _scenarioResults = new(); - private readonly StringBuilder _scenarioDamageByCategory = new(); - private readonly StringBuilder _scenarioPerformance = new(); - private readonly StringBuilder _alternativeResults = new(); - private readonly StringBuilder _alternativeDamageByCategory = new(); + private readonly List<(string StudyId, IASElement Element)> _scenarioElements = []; + private readonly List<(string StudyId, AlternativeElement Element)> _alternativeElements = []; private readonly StringBuilder _stageDamageSummary = new(); private readonly StringBuilder _altComparisonSummary = new(); private readonly StringBuilder _altComparisonByCategory = new(); public CsvReportFactory() { - WriteHeaders(); + WriteStageDamageHeader(); + WriteAltComparisonHeaders(); } - private void WriteHeaders() + private void WriteStageDamageHeader() { - // Scenario Results header - _scenarioResults.AppendLine("Study ID,Scenario Name,Impact Area ID,Mean EAD,EAD 25th Pct,EAD 50th Pct,EAD 75th Pct"); - - // Scenario Damage by Category header - _scenarioDamageByCategory.AppendLine("Study ID,Scenario Name,Impact Area ID,Damage Category,Asset Category,Mean EAD"); - - // Scenario Performance header - _scenarioPerformance.AppendLine("Study ID,Scenario Name,Impact Area ID,Threshold ID,Mean AEP,Median AEP,Assurance 0.10,Assurance 0.04,Assurance 0.02,Assurance 0.01,LT Risk 10yr,LT Risk 30yr,LT Risk 50yr"); - - // Alternative Results header - _alternativeResults.AppendLine("Study ID,Alternative Name,Impact Area ID,Base Year,Future Year,Period of Analysis,Mean Base EAD,Mean Future EAD,Mean EqAD,EqAD 25th Pct,EqAD 50th Pct,EqAD 75th Pct"); - - // Alternative Damage by Category header - _alternativeDamageByCategory.AppendLine("Study ID,Alternative Name,Impact Area ID,Damage Category,Asset Category,Mean EqAD"); - - // Stage Damage Summary header - _stageDamageSummary.AppendLine("Study ID,Element Name,Impact Area ID,Impact Area Name,Damage Category,Asset Category,Point Count,Min Stage,Max Stage"); + _stageDamageSummary.AppendLine("Study ID,Element Name,Impact Area ID,Impact Area Name,Damage Category,Asset Category,Point Count,Min Stage,Max Stage, Median Sample Integral"); + } - // Alternative Comparison Summary header + private void WriteAltComparisonHeaders() + { _altComparisonSummary.AppendLine("Study ID,Report Name,With Project Alt,Impact Area ID,Without Proj EqAD,With Proj EqAD,EqAD Reduced,EqAD Reduced 25th Pct,EqAD Reduced 50th Pct,EqAD Reduced 75th Pct,Without Proj Base EAD,With Proj Base EAD,Base EAD Reduced,Without Proj Future EAD,With Proj Future EAD,Future EAD Reduced"); - - // Alternative Comparison by Category header _altComparisonByCategory.AppendLine("Study ID,Report Name,With Project Alt,Impact Area ID,Damage Category,Asset Category,EqAD Reduced,Base Year EAD Reduced,Future Year EAD Reduced"); } /// - /// Adds scenario results from a computation to the report. - /// Iterates over ResultsList for direct access to each ImpactAreaScenarioResults. + /// Adds a scenario element to be included in the report. + /// Uses ScenarioDamageRowItem, ScenarioDamCatRowItem, and ScenarioPerformanceRowItem for data extraction. /// - public void AddScenarioResults(string studyId, string scenarioName, ScenarioResults results) - { - if (results == null) return; - - try - { - // Iterate over the ResultsList directly for better access to impact area data - foreach (var iaResult in results.ResultsList) - { - int impactAreaId = iaResult.ImpactAreaID; - - // Write aggregate EAD results for this impact area - double meanEAD = iaResult.MeanExpectedAnnualConsequences(); - double ead25 = iaResult.ConsequencesExceededWithProbabilityQ(0.75); // 25th percentile = exceeded by 75% - double ead50 = iaResult.ConsequencesExceededWithProbabilityQ(0.50); - double ead75 = iaResult.ConsequencesExceededWithProbabilityQ(0.25); // 75th percentile = exceeded by 25% - - _scenarioResults.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(scenarioName)},{impactAreaId},{meanEAD:F2},{ead25:F2},{ead50:F2},{ead75:F2}"); - - // Write damage by category from ConsequenceResults - foreach (var consequence in iaResult.ConsequenceResults.ConsequenceResultList) - { - string damCat = consequence.DamageCategory; - string assetCat = consequence.AssetCategory; - double categoryMeanEAD = iaResult.MeanExpectedAnnualConsequences(impactAreaId, damCat, assetCat); - - _scenarioDamageByCategory.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(scenarioName)},{impactAreaId},{EscapeCsv(damCat)},{EscapeCsv(assetCat)},{categoryMeanEAD:F2}"); - } - - // Write performance metrics for each threshold in this impact area - WritePerformanceMetrics(studyId, scenarioName, iaResult); - } - } - catch (Exception ex) - { - Console.WriteLine($" Warning: Error extracting scenario results for CSV: {ex.Message}"); - } - } - - private void WritePerformanceMetrics(string studyId, string scenarioName, ImpactAreaScenarioResults iaResult) + public void AddScenarioResults(string studyId, IASElement element) { - int impactAreaId = iaResult.ImpactAreaID; - - // Iterate over all available thresholds for this impact area - foreach (var threshold in iaResult.PerformanceByThresholds.ListOfThresholds) - { - try - { - int thresholdId = threshold.ThresholdID; - - double meanAEP = iaResult.MeanAEP(thresholdId); - double medianAEP = iaResult.MedianAEP(thresholdId); - - double assurance10 = iaResult.AssuranceOfEvent(thresholdId, 0.10); - double assurance04 = iaResult.AssuranceOfEvent(thresholdId, 0.04); - double assurance02 = iaResult.AssuranceOfEvent(thresholdId, 0.02); - double assurance01 = iaResult.AssuranceOfEvent(thresholdId, 0.01); - - double ltRisk10 = iaResult.LongTermExceedanceProbability(thresholdId, 10); - double ltRisk30 = iaResult.LongTermExceedanceProbability(thresholdId, 30); - double ltRisk50 = iaResult.LongTermExceedanceProbability(thresholdId, 50); - - _scenarioPerformance.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(scenarioName)},{impactAreaId},{thresholdId},{meanAEP:F6},{medianAEP:F6},{assurance10:F4},{assurance04:F4},{assurance02:F4},{assurance01:F4},{ltRisk10:F4},{ltRisk30:F4},{ltRisk50:F4}"); - } - catch (Exception ex) - { - Console.WriteLine($" Warning: Could not extract performance metrics for threshold {threshold.ThresholdID}: {ex.Message}"); - } - } + if (element?.Results == null) return; + _scenarioElements.Add((studyId, element)); } /// - /// Adds alternative results from a computation to the report. + /// Adds an alternative element to be included in the report. + /// Uses AlternativeDamageRowItem and AlternativeDamCatRowItem for data extraction. /// - public void AddAlternativeResults(string studyId, string alternativeName, AlternativeResults results) + public void AddAlternativeResults(string studyId, AlternativeElement element) { - if (results == null || results.IsNull) return; - - try - { - var impactAreaIds = results.GetImpactAreaIDs(); - var damageCategories = results.GetDamageCategories(); - var assetCategories = results.GetAssetCategories(); - - int baseYear = results.AnalysisYears.Count > 0 ? results.AnalysisYears[0] : 0; - int futureYear = results.AnalysisYears.Count > 1 ? results.AnalysisYears[1] : 0; - int periodOfAnalysis = results.PeriodOfAnalysis; - - // Write aggregate results per impact area - foreach (int impactAreaId in impactAreaIds) - { - double meanBaseEAD = results.SampleMeanBaseYearEAD(impactAreaId); - double meanFutureEAD = results.SampleMeanFutureYearEAD(impactAreaId); - double meanEqAD = results.SampleMeanEqad(impactAreaId); - double eqad25 = results.EqadExceededWithProbabilityQ(0.75, impactAreaId); - double eqad50 = results.EqadExceededWithProbabilityQ(0.50, impactAreaId); - double eqad75 = results.EqadExceededWithProbabilityQ(0.25, impactAreaId); - - _alternativeResults.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(alternativeName)},{impactAreaId},{baseYear},{futureYear},{periodOfAnalysis},{meanBaseEAD:F2},{meanFutureEAD:F2},{meanEqAD:F2},{eqad25:F2},{eqad50:F2},{eqad75:F2}"); - } - - // Write damage by category - foreach (int impactAreaId in impactAreaIds) - { - foreach (string damCat in damageCategories) - { - foreach (string assetCat in assetCategories) - { - double meanEqAD = results.SampleMeanEqad(impactAreaId, damCat, assetCat); - if (meanEqAD != 0) // Only write non-zero values - { - _alternativeDamageByCategory.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(alternativeName)},{impactAreaId},{EscapeCsv(damCat)},{EscapeCsv(assetCat)},{meanEqAD:F2}"); - } - } - } - } - } - catch (Exception ex) - { - Console.WriteLine($" Warning: Error extracting alternative results for CSV: {ex.Message}"); - } + if (element?.Results == null || element.Results.IsNull) return; + _alternativeElements.Add((studyId, element)); } /// @@ -193,12 +73,14 @@ public void AddStageDamageSummary(string studyId, string elementName, List 0 ? curve.Xvals!.Min() : 0; double maxStage = pointCount > 0 ? curve.Xvals!.Max() : 0; - _stageDamageSummary.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(elementName)},{impactAreaId},,{EscapeCsv(damCat)},{EscapeCsv(assetCat)},{pointCount},{minStage:F2},{maxStage:F2}"); + PairedData medianCurve = curve.SamplePairedData(.5); + double integral = medianCurve.integrate(); + + _stageDamageSummary.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(elementName)},{impactAreaId},{EscapeCsv(damCat)},{EscapeCsv(assetCat)},{pointCount},{minStage:F2},{maxStage:F2},{integral:F2}"); } } catch (Exception ex) @@ -222,10 +104,8 @@ public void AddAlternativeComparisonResults(string studyId, string reportName, A foreach (var (altId, altName) in withProjectAlternatives) { - // Write aggregated summary per impact area foreach (int impactAreaId in impactAreaIds) { - // EqAD values double withoutProjEqad = results.SampleMeanWithoutProjectEqad(impactAreaId); double withProjEqad = results.SampleMeanWithProjectEqad(altId, impactAreaId); double eqadReduced = results.SampleMeanEqadReduced(altId, impactAreaId); @@ -233,12 +113,10 @@ public void AddAlternativeComparisonResults(string studyId, string reportName, A double eqadReduced50 = results.EqadReducedExceededWithProbabilityQ(0.50, altId, impactAreaId); double eqadReduced75 = results.EqadReducedExceededWithProbabilityQ(0.25, altId, impactAreaId); - // Base year EAD values double withoutProjBaseEad = results.SampleMeanWithoutProjectBaseYearEAD(impactAreaId); double withProjBaseEad = results.SampleMeanWithProjectBaseYearEAD(altId, impactAreaId); double baseEadReduced = results.SampleMeanBaseYearEADReduced(altId, impactAreaId); - // Future year EAD values double withoutProjFutureEad = results.SampleMeanWithoutProjectFutureYearEAD(impactAreaId); double withProjFutureEad = results.SampleMeanWithProjectFutureYearEAD(altId, impactAreaId); double futureEadReduced = results.SampleMeanFutureYearEADReduced(altId, impactAreaId); @@ -246,7 +124,6 @@ public void AddAlternativeComparisonResults(string studyId, string reportName, A _altComparisonSummary.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(reportName)},{EscapeCsv(altName)},{impactAreaId},{withoutProjEqad:F2},{withProjEqad:F2},{eqadReduced:F2},{eqadReduced25:F2},{eqadReduced50:F2},{eqadReduced75:F2},{withoutProjBaseEad:F2},{withProjBaseEad:F2},{baseEadReduced:F2},{withoutProjFutureEad:F2},{withProjFutureEad:F2},{futureEadReduced:F2}"); } - // Write by category breakdown foreach (int impactAreaId in impactAreaIds) { foreach (string damCat in damageCategories) @@ -257,7 +134,6 @@ public void AddAlternativeComparisonResults(string studyId, string reportName, A double baseEadReduced = results.SampleMeanBaseYearEADReduced(altId, impactAreaId, damCat, assetCat); double futureEadReduced = results.SampleMeanFutureYearEADReduced(altId, impactAreaId, damCat, assetCat); - // Only write non-zero values if (eqadReduced != 0 || baseEadReduced != 0 || futureEadReduced != 0) { _altComparisonByCategory.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(reportName)},{EscapeCsv(altName)},{impactAreaId},{EscapeCsv(damCat)},{EscapeCsv(assetCat)},{eqadReduced:F2},{baseEadReduced:F2},{futureEadReduced:F2}"); @@ -285,23 +161,27 @@ public void SaveReport(string outputPath) report.AppendLine(); report.AppendLine("=== SCENARIO RESULTS ==="); - report.Append(_scenarioResults); + report.Append(BuildScenarioResultsSection()); report.AppendLine(); report.AppendLine("=== SCENARIO DAMAGE BY CATEGORY ==="); - report.Append(_scenarioDamageByCategory); + report.Append(BuildScenarioDamCatSection()); report.AppendLine(); report.AppendLine("=== SCENARIO PERFORMANCE ==="); - report.Append(_scenarioPerformance); + report.Append(BuildScenarioPerformanceSection()); + report.AppendLine(); + + report.AppendLine("=== SCENARIO ASSURANCE OF AEP ==="); + report.Append(BuildScenarioAssuranceSection()); report.AppendLine(); report.AppendLine("=== ALTERNATIVE RESULTS ==="); - report.Append(_alternativeResults); + report.Append(BuildAlternativeResultsSection()); report.AppendLine(); report.AppendLine("=== ALTERNATIVE DAMAGE BY CATEGORY ==="); - report.Append(_alternativeDamageByCategory); + report.Append(BuildAlternativeDamCatSection()); report.AppendLine(); report.AppendLine("=== STAGE DAMAGE SUMMARY ==="); @@ -319,6 +199,172 @@ public void SaveReport(string outputPath) Console.WriteLine($"CSV report saved to: {outputPath}"); } + #region Scenario Section Builders + + private StringBuilder BuildScenarioResultsSection() + { + StringBuilder sb = new(); + sb.AppendLine("Study ID,Name,Analysis Year,Impact Area,Mean EAD,25th Percentile EAD,50th Percentile EAD,75th Percentile EAD"); + + foreach (var (studyId, element) in _scenarioElements) + { + try + { + var rows = ScenarioDamageRowItem.CreateScenarioDamageRowItems(element); + foreach (var row in rows) + { + sb.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(row.Name)},{EscapeCsv(row.AnalysisYear)},{EscapeCsv(row.ImpactArea)},{row.Mean:F2},{row.Point25:F2},{row.Point5:F2},{row.Point75:F2}"); + } + } + catch (Exception ex) + { + Console.WriteLine($" Warning: Error extracting scenario results for {element.Name}: {ex.Message}"); + } + } + + return sb; + } + + private StringBuilder BuildScenarioDamCatSection() + { + StringBuilder sb = new(); + sb.AppendLine("Study ID,Scenario Name,Analysis Year,Impact Area,Damage Category,Asset Category,Mean EAD"); + + foreach (var (studyId, element) in _scenarioElements) + { + try + { + var rows = ScenarioDamCatRowItem.CreateScenarioDamCatRowItems(element); + foreach (var row in rows) + { + sb.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(row.Name)},{EscapeCsv(row.AnalysisYear)},{EscapeCsv(row.ImpactAreaName)},{EscapeCsv(row.DamCat)},{EscapeCsv(row.AssetCat)},{row.MeanDamage:F2}"); + } + } + catch (Exception ex) + { + Console.WriteLine($" Warning: Error extracting scenario damage categories for {element.Name}: {ex.Message}"); + } + } + + return sb; + } + + private StringBuilder BuildScenarioPerformanceSection() + { + StringBuilder sb = new(); + sb.AppendLine("Study ID,Name,Analysis Year,Impact Area,Threshold Type,Threshold Value,LT Risk 10yr,LT Risk 30yr,LT Risk 50yr,Mean AEP,Median AEP,Assurance 0.10,Assurance 0.04,Assurance 0.02,Assurance 0.01,Assurance 0.004,Assurance 0.002"); + + foreach (var (studyId, element) in _scenarioElements) + { + try + { + foreach (var iaResult in element.Results.ResultsList) + { + int iasID = iaResult.ImpactAreaID; + SpecificIAS? ias = element.SpecificIASElements.FirstOrDefault(s => s.ImpactAreaID == iasID); + if (ias == null) continue; + + foreach (var threshold in iaResult.PerformanceByThresholds.ListOfThresholds) + { + var row = new ScenarioPerformanceRowItem(element, ias, threshold); + sb.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(row.Name)},{EscapeCsv(row.AnalysisYear)},{EscapeCsv(row.ImpactArea)},{EscapeCsv(row.ThresholdType)},{row.ThresholdValue:F2},{row.LongTerm10:F4},{row.LongTerm30:F4},{row.LongTerm50:F4},{row.Mean:F6},{row.Median:F6},{row.Threshold1:F4},{row.Threshold04:F4},{row.Threshold02:F4},{row.Threshold01:F4},{row.Threshold004:F4},{row.Threshold002:F4}"); + } + } + } + catch (Exception ex) + { + Console.WriteLine($" Warning: Error extracting scenario performance for {element.Name}: {ex.Message}"); + } + } + + return sb; + } + + private StringBuilder BuildScenarioAssuranceSection() + { + StringBuilder sb = new(); + sb.AppendLine("Study ID,Name,Analysis Year,Impact Area,Threshold Type,Threshold Value,Mean AEP,Median AEP,90% Assurance AEP,Assurance of 0.10 AEP,Assurance of 0.04 AEP,Assurance of 0.02 AEP,Assurance of 0.01 AEP,Assurance of 0.004 AEP,Assurance of 0.002 AEP"); + + foreach (var (studyId, element) in _scenarioElements) + { + try + { + foreach (var iaResult in element.Results.ResultsList) + { + int iasID = iaResult.ImpactAreaID; + SpecificIAS? ias = element.SpecificIASElements.FirstOrDefault(s => s.ImpactAreaID == iasID); + if (ias == null) continue; + + foreach (var threshold in iaResult.PerformanceByThresholds.ListOfThresholds) + { + var row = new AssuranceOfAEPRowItem(element, ias, threshold); + sb.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(row.Name)},{EscapeCsv(row.AnalysisYear)},{EscapeCsv(row.ImpactArea)},{EscapeCsv(row.ThresholdType)},{row.ThresholdValue:F2},{row.Mean:F6},{row.Median:F6},{row.NinetyPercentAssurance:F6},{row.AEP1:F4},{row.AEP04:F4},{row.AEP02:F4},{row.AEP01:F4},{row.AEP004:F4},{row.AEP002:F4}"); + } + } + } + catch (Exception ex) + { + Console.WriteLine($" Warning: Error extracting scenario assurance for {element.Name}: {ex.Message}"); + } + } + + return sb; + } + + #endregion + + #region Alternative Section Builders + + private StringBuilder BuildAlternativeResultsSection() + { + StringBuilder sb = new(); + sb.AppendLine("Study ID,Name,Impact Area,Base Year,Future Year,Discount Rate,Period of Analysis,Mean EqAD,25th Percentile EqAD,50th Percentile EqAD,75th Percentile EqAD"); + + foreach (var (studyId, element) in _alternativeElements) + { + try + { + var rows = AlternativeDamageRowItem.CreateAlternativeDamageRowItems(element); + foreach (var row in rows) + { + sb.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(row.Name)},{EscapeCsv(row.ImpactArea)},{row.BaseYear},{row.FutureYear},{row.DiscountRate:F4},{row.PeriodOfAnalysis},{row.Mean:F2},{row.Point75:F2},{row.Point5:F2},{row.Point25:F2}"); + } + } + catch (Exception ex) + { + Console.WriteLine($" Warning: Error extracting alternative results for {element.Name}: {ex.Message}"); + } + } + + return sb; + } + + private StringBuilder BuildAlternativeDamCatSection() + { + StringBuilder sb = new(); + sb.AppendLine("Study ID,Scenario Name,Impact Area,Damage Category,Asset Category,Mean EqAD"); + + foreach (var (studyId, element) in _alternativeElements) + { + try + { + var rows = AlternativeDamCatRowItem.CreateAlternativeDamCatRowItems(element); + foreach (var row in rows) + { + sb.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(row.Name)},{EscapeCsv(row.ImpactAreaName)},{EscapeCsv(row.DamCat)},{EscapeCsv(row.AssetCat)},{row.MeanDamage:F2}"); + } + } + catch (Exception ex) + { + Console.WriteLine($" Warning: Error extracting alternative damage categories for {element.Name}: {ex.Message}"); + } + } + + return sb; + } + + #endregion + /// /// Escapes a value for CSV output (handles commas and quotes). /// diff --git a/HEC.FDA.TestingUtility/Services/StudyLoader.cs b/HEC.FDA.TestingUtility/Services/StudyLoader.cs index 89b8b1606..e2ab84ad5 100644 --- a/HEC.FDA.TestingUtility/Services/StudyLoader.cs +++ b/HEC.FDA.TestingUtility/Services/StudyLoader.cs @@ -1,3 +1,4 @@ +using System.Data.SQLite; using HEC.FDA.TestingUtility.Configuration; using HEC.FDA.ViewModel; using HEC.FDA.ViewModel.AggregatedStageDamage; @@ -138,6 +139,13 @@ public void Cleanup() Connection.Instance.Close(); } + // Clear SQLite connection pool to release file handles + SQLiteConnection.ClearAllPools(); + + // Force garbage collection to release any remaining handles + GC.Collect(); + GC.WaitForPendingFinalizers(); + Directory.Delete(_localStudyPath, recursive: true); Console.WriteLine($" Cleaned up temp study folder: {_localStudyPath}"); } diff --git a/HEC.FDA.ViewModel/Results/AssuranceOfAEPRowItem.cs b/HEC.FDA.ViewModel/Results/AssuranceOfAEPRowItem.cs index def0e2c99..68da345db 100644 --- a/HEC.FDA.ViewModel/Results/AssuranceOfAEPRowItem.cs +++ b/HEC.FDA.ViewModel/Results/AssuranceOfAEPRowItem.cs @@ -1,25 +1,40 @@ using HEC.FDA.Model.metrics; using HEC.FDA.ViewModel.ImpactAreaScenario; +using HEC.FDA.ViewModel.TableWithPlot.Rows.Attributes; namespace HEC.FDA.ViewModel.Results { public class AssuranceOfAEPRowItem { + [DisplayAsColumn("Name")] public string Name { get; set; } + [DisplayAsColumn("Analysis Year")] public string AnalysisYear { get; set; } + [DisplayAsColumn("Impact Area")] public string ImpactArea { get; set; } + [DisplayAsColumn("Threshold Type")] public string ThresholdType { get; set; } + [DisplayAsColumn("Threshold Value")] public double ThresholdValue { get; set; } + [DisplayAsColumn("Mean AEP")] public double Mean { get; set; } + [DisplayAsColumn("Median AEP")] public double Median { get; set; } + [DisplayAsColumn("90% Assurance AEP")] public double NinetyPercentAssurance { get; set; } + [DisplayAsColumn("Assurance of 0.10 AEP")] public double AEP1 { get; set; } + [DisplayAsColumn("Assurance of 0.04 AEP")] public double AEP04 { get; set; } + [DisplayAsColumn("Assurance of 0.02 AEP")] public double AEP02 { get; set; } + [DisplayAsColumn("Assurance of 0.01 AEP")] public double AEP01 { get; set; } + [DisplayAsColumn("Assurance of 0.004 AEP")] public double AEP004 { get; set; } + [DisplayAsColumn("Assurance of 0.002 AEP")] public double AEP002 { get; set; } public AssuranceOfAEPRowItem(IASElement scenario, SpecificIAS ias, Threshold threshold) diff --git a/HEC.FDA.ViewModel/Results/ScenarioPerformanceRowItem.cs b/HEC.FDA.ViewModel/Results/ScenarioPerformanceRowItem.cs index 3f8a389ec..cb10dbcba 100644 --- a/HEC.FDA.ViewModel/Results/ScenarioPerformanceRowItem.cs +++ b/HEC.FDA.ViewModel/Results/ScenarioPerformanceRowItem.cs @@ -1,29 +1,46 @@ using HEC.FDA.Model.metrics; using HEC.FDA.ViewModel.ImpactAreaScenario; +using HEC.FDA.ViewModel.TableWithPlot.Rows.Attributes; namespace HEC.FDA.ViewModel.Results { public class ScenarioPerformanceRowItem { + [DisplayAsColumn("Name")] public string Name { get; set; } + [DisplayAsColumn("Analysis Year")] public string AnalysisYear { get; set; } + [DisplayAsColumn("Impact Area")] public string ImpactArea { get; set; } + [DisplayAsColumn("Threshold Type")] public string ThresholdType { get; set; } + [DisplayAsColumn("Threshold Value")] public double ThresholdValue { get; set; } + [DisplayAsColumn("LT Risk 10yr")] public double LongTerm10 { get; set; } + [DisplayAsColumn("LT Risk 30yr")] public double LongTerm30 { get; set; } + [DisplayAsColumn("LT Risk 50yr")] public double LongTerm50 { get; set; } + [DisplayAsColumn("Mean AEP")] public double Mean { get; set; } + [DisplayAsColumn("Median AEP")] public double Median { get; set; } + [DisplayAsColumn("Assurance 0.10")] public double Threshold1 { get; set; } + [DisplayAsColumn("Assurance 0.04")] public double Threshold04 { get; set; } + [DisplayAsColumn("Assurance 0.02")] public double Threshold02 { get; set; } + [DisplayAsColumn("Assurance 0.01")] public double Threshold01 { get; set; } + [DisplayAsColumn("Assurance 0.004")] public double Threshold004 { get; set; } + [DisplayAsColumn("Assurance 0.002")] public double Threshold002 { get; set; } public ScenarioPerformanceRowItem(IASElement scenario, SpecificIAS ias, Threshold threshold) From 3004e97034db86c6fcdc03d801b9a15ff8a30323 Mon Sep 17 00:00:00 2001 From: Brennan Beam <64556723+Brennan1994@users.noreply.github.com> Date: Tue, 10 Feb 2026 10:49:23 -0800 Subject: [PATCH 12/13] add examples. --- .../Configuration/TestConfiguration.cs | 2 +- HEC.FDA.TestingUtility/README.md | 262 ++++++++++++++++++ HEC.FDA.TestingUtility/example-config.json | 19 ++ 3 files changed, 282 insertions(+), 1 deletion(-) create mode 100644 HEC.FDA.TestingUtility/README.md create mode 100644 HEC.FDA.TestingUtility/example-config.json diff --git a/HEC.FDA.TestingUtility/Configuration/TestConfiguration.cs b/HEC.FDA.TestingUtility/Configuration/TestConfiguration.cs index 1e4865c55..569a1a9a5 100644 --- a/HEC.FDA.TestingUtility/Configuration/TestConfiguration.cs +++ b/HEC.FDA.TestingUtility/Configuration/TestConfiguration.cs @@ -28,7 +28,7 @@ public class GlobalSettings public string LocalTempDirectory { get; set; } = Path.Combine(Path.GetTempPath(), "FDATests"); [JsonPropertyName("timeoutMinutes")] - public int TimeoutMinutes { get; set; } = 30; + public int TimeoutMinutes { get; set; } = 120; } public class StudyConfiguration diff --git a/HEC.FDA.TestingUtility/README.md b/HEC.FDA.TestingUtility/README.md new file mode 100644 index 000000000..64a81bbfb --- /dev/null +++ b/HEC.FDA.TestingUtility/README.md @@ -0,0 +1,262 @@ +# HEC.FDA.TestingUtility + +A command-line regression testing tool for HEC-FDA studies. It loads FDA study databases, runs computations (stage damage, scenarios, alternatives, and alternative comparisons), and produces a comprehensive CSV report of the results. + +## Purpose + +This utility runs FDA computations outside the GUI so that results can be validated programmatically. Typical use cases include: + +- **Regression testing** - Run computations on known studies and compare the CSV output against baseline results to detect unintended changes. +- **Batch computation** - Execute all computations in one or more studies without manual interaction. +- **CI/CD integration** - Automate computation verification as part of a build pipeline. + +## Quick Start + +### 1. Build the project + +```bash +dotnet build HEC.FDA.TestingUtility/HEC.FDA.TestingUtility.csproj +``` + +### 2. Create a configuration file + +Copy `example-config.json` from this project directory and update `networkSourcePath` to point to your FDA study folder (the folder containing the `.sqlite` or `.db` file): + +```json +{ + "testSuiteId": "quick-start", + "globalSettings": { + "localTempDirectory": "C:/temp/FDATests", + "timeoutMinutes": 60 + }, + "studies": [ + { + "studyId": "my-study", + "studyName": "My Study", + "networkSourcePath": "C:/path/to/my/study/folder", + "runAllStageDamage": true, + "runAllScenarios": true, + "runAllAlternatives": true, + "runAllAlternativeComparisons": true, + "computations": [] + } + ] +} +``` + +Setting the `runAll*` flags to `true` auto-discovers every element of that type in the study, so you don't need to list them individually. If you only want to run specific elements, set the flags to `false` and use `computations` instead: + +```json +{ + "testSuiteId": "targeted-run", + "globalSettings": {}, + "studies": [ + { + "studyId": "my-study", + "studyName": "My Study", + "networkSourcePath": "C:/path/to/my/study/folder", + "runAllStageDamage": false, + "runAllScenarios": false, + "runAllAlternatives": false, + "runAllAlternativeComparisons": false, + "computations": [ + { "type": "stagedamage", "elementName": "My Stage Damage" }, + { "type": "scenario", "elementName": "Existing Conditions" }, + { "type": "alternative", "elementName": "Proposed Levee" } + ] + } + ] +} +``` + +You can list computations in any order -- the utility automatically sorts them by dependency (stage damage first, then scenarios, then alternatives, then alternative comparisons). + +### 3. Run the utility + +```bash +# Run all studies in the config, write results to a "results" folder +dotnet run --project HEC.FDA.TestingUtility -- compute -c my-test.json -o results + +# Run only a single study by its studyId +dotnet run --project HEC.FDA.TestingUtility -- compute -c my-test.json -o results -s "my-study" + +# Filter to multiple studies +dotnet run --project HEC.FDA.TestingUtility -- compute -c my-test.json -s "study-a" -s "study-b" +``` + +### 4. Check the output + +After the run completes, look for `results_report.csv` in the output directory. It contains separate sections for scenario EAD, damage by category, performance metrics, alternative EqAD, stage damage summaries, and alternative comparison results. + +The console output will also show a summary: + +``` +=== Summary === +Completed: 5 +Errors: 0 +Duration: 2m 34.1s + +CSV report saved to: results/results_report.csv +``` + +An exit code of `0` means all computations passed. An exit code of `1` means at least one failed -- scroll up in the console output to find lines marked `ERROR`. + +## Prerequisites + +- .NET 9.0 SDK (Windows, targets `net9.0-windows`) +- Access to FDA study databases (`.sqlite` or `.db` files) located on a network share or local directory +- NuGet sources configured as described in the root `CLAUDE.md` + +## Building + +```bash +dotnet build HEC.FDA.TestingUtility/HEC.FDA.TestingUtility.csproj +``` + +## Usage + +The utility uses the `System.CommandLine` library and exposes a `compute` subcommand: + +```bash +dotnet run --project HEC.FDA.TestingUtility -- compute --config [options] +``` + +### Options + +| Option | Alias | Required | Description | +|---|---|---|---| +| `--config` | `-c` | Yes | Path to a JSON configuration file defining the studies and computations to run. | +| `--output` | `-o` | No | Output directory for generated files. Defaults to the current working directory. | +| `--study` | `-s` | No | Filter to one or more specific study IDs. Can be specified multiple times. | + +### Examples + +Run all studies defined in a config file: + +```bash +dotnet run --project HEC.FDA.TestingUtility -- compute -c tests/regression.json -o results/ +``` + +Run only a specific study: + +```bash +dotnet run --project HEC.FDA.TestingUtility -- compute -c tests/regression.json -s "muncie" +``` + +## Configuration File + +The configuration is a JSON file with the following structure: + +```json +{ + "testSuiteId": "regression-v1", + "globalSettings": { + "localTempDirectory": "C:/temp/FDATests", + "timeoutMinutes": 30 + }, + "studies": [ + { + "studyId": "muncie", + "studyName": "Muncie Indiana", + "networkSourcePath": "\\\\server\\share\\studies\\Muncie", + "runAllStageDamage": true, + "runAllScenarios": true, + "runAllAlternatives": true, + "runAllAlternativeComparisons": true, + "computations": [] + } + ] +} +``` + +### Global Settings + +| Field | Default | Description | +|---|---|---| +| `localTempDirectory` | System temp + `FDATests` | Directory where studies are copied locally before computation. | +| `timeoutMinutes` | `30` | Maximum wall-clock time for all computations before cancellation. | + +### Study Configuration + +| Field | Description | +|---|---| +| `studyId` | Short identifier used in the CSV report and for the `--study` filter. | +| `studyName` | Human-readable name printed during execution. | +| `networkSourcePath` | Path to the study folder containing the `.sqlite` or `.db` database file. | +| `runAllStageDamage` | Auto-discover and run all stage damage elements in the study. | +| `runAllScenarios` | Auto-discover and run all scenario elements. | +| `runAllAlternatives` | Auto-discover and run all alternative elements. | +| `runAllAlternativeComparisons` | Auto-discover and run all alternative comparison report elements. | +| `computations` | Explicit list of computations (see below). Combined with auto-discovered elements. | + +### Computation Entry + +Each entry in the `computations` array targets a specific element: + +```json +{ + "type": "scenario", + "elementName": "Existing Conditions" +} +``` + +Valid `type` values (case-insensitive): + +| Type | Description | +|---|---| +| `stagedamage` | Computes stage-damage curves from hydraulics and inventory data. | +| `scenario` | Runs a Monte Carlo scenario simulation using convergence criteria from study properties. | +| `alternative` | Computes annualized damages for an alternative using its base and future scenario results. | +| `alternativecomparison` | Compares with-project alternatives against a without-project alternative. | + +Computations are automatically sorted in dependency order: stage damage -> scenario -> alternative -> alternative comparison. This means you can list them in any order and the utility will execute them correctly. + +## How It Works + +1. **Study Loading** - The study folder is copied from `networkSourcePath` to a local temp directory to avoid locking network files. The SQLite database is opened and all element types are loaded into the `StudyCache` in dependency order (terrains, impact areas, hydraulics, frequencies, inventories, stage damage, scenarios, alternatives, etc.). + +2. **Computation** - Each computation is dispatched to the appropriate runner: + - `StageDamageRunner` - Builds stage-damage configuration from hydraulics and inventory, then calls `ScenarioStageDamage.Compute()`. + - `ScenarioRunner` - Creates `ImpactAreaScenarioSimulation` objects and runs `Scenario.Compute()` with the study's convergence criteria. + - `AlternativeRunner` - Calls `Alternative.AnnualizationCompute()` using the base/future scenario results and study discount rate / period of analysis. + - `AlternativeComparisonRunner` - Computes alternatives and then runs `AlternativeComparisonReport.ComputeAlternativeComparisonReport()`. + +3. **Result Saving** - Computed results are saved back to the temp study database so that downstream computations (e.g., alternatives depending on scenario results) can access them. + +4. **CSV Report** - A single `results_report.csv` file is written to the output directory containing sections for: + - Scenario results (mean and percentile EAD by impact area) + - Scenario damage by category + - Scenario performance (long-term risk, AEP, assurance) + - Scenario assurance of AEP + - Alternative results (mean and percentile EqAD) + - Alternative damage by category + - Stage damage summary (point counts, stage ranges, median integrals) + - Alternative comparison summary (EqAD reduced, base/future EAD reduced) + - Alternative comparison by damage category + +5. **Cleanup** - When a study finishes, the temp copy is deleted automatically. + +## Project Structure + +``` +HEC.FDA.TestingUtility/ + Program.cs # CLI entry point (System.CommandLine) + ComputeRunner.cs # Orchestrates study loading, computation, and reporting + Configuration/ + TestConfiguration.cs # JSON config deserialization models + Services/ + StudyLoader.cs # Copies study to temp, opens DB, loads all elements + StageDamageRunner.cs # Stage damage computation + ScenarioRunner.cs # Scenario computation + element lookup helpers + AlternativeRunner.cs # Alternative annualization computation + AlternativeComparisonRunner.cs # Alternative comparison report computation + Reporting/ + CsvReportFactory.cs # Builds the multi-section CSV report +``` + +## Exit Codes + +| Code | Meaning | +|---|---| +| `0` | All computations completed successfully. | +| `1` | One or more computations failed, or the configuration was invalid. | diff --git a/HEC.FDA.TestingUtility/example-config.json b/HEC.FDA.TestingUtility/example-config.json new file mode 100644 index 000000000..88e461a27 --- /dev/null +++ b/HEC.FDA.TestingUtility/example-config.json @@ -0,0 +1,19 @@ +{ + "testSuiteId": "quick-start", + "globalSettings": { + "localTempDirectory": "C:/temp/FDATests", + "timeoutMinutes": 60 + }, + "studies": [ + { + "studyId": "my-study", + "studyName": "My Study", + "networkSourcePath": "C:/path/to/my/study/folder", + "runAllStageDamage": true, + "runAllScenarios": true, + "runAllAlternatives": true, + "runAllAlternativeComparisons": true, + "computations": [] + } + ] +} From c59d8eddc623e90861007a1942262f7ed9ba6da6 Mon Sep 17 00:00:00 2001 From: Brennan Beam <64556723+Brennan1994@users.noreply.github.com> Date: Thu, 26 Mar 2026 10:56:47 -0700 Subject: [PATCH 13/13] test utils update --- HEC.FDA.TestingUtility/Program.cs | 125 +++++++++--------- .../Reporting/CsvReportFactory.cs | 2 +- .../Services/StageDamageRunner.cs | 1 + .../Services/StudyLoader.cs | 1 + .../Beam/ValidationAndVerification.cs | 20 +++ ScratchSpace/Beam/output/results_report.csv | 64 +++++++++ ScratchSpace/Beam/west-sac-config.json | 19 +++ ScratchSpace/EntryPoints/Beam.cs | 5 +- ScratchSpace/ScratchSpace.csproj | 1 + 9 files changed, 175 insertions(+), 63 deletions(-) create mode 100644 ScratchSpace/Beam/ValidationAndVerification.cs create mode 100644 ScratchSpace/Beam/output/results_report.csv create mode 100644 ScratchSpace/Beam/west-sac-config.json diff --git a/HEC.FDA.TestingUtility/Program.cs b/HEC.FDA.TestingUtility/Program.cs index f27b6ffe0..d90653df9 100644 --- a/HEC.FDA.TestingUtility/Program.cs +++ b/HEC.FDA.TestingUtility/Program.cs @@ -1,79 +1,84 @@ using System.CommandLine; using Geospatial.GDALAssist; -using HEC.FDA.TestingUtility; using HEC.FDA.TestingUtility.Configuration; -// Initialize GDAL early -GDALSetup.InitializeMultiplatform(); +namespace HEC.FDA.TestingUtility; -// Create root command -RootCommand rootCommand = new("FDA Testing Utility - Regression Testing Tool for FDA Studies"); +public class Program +{ + public static async Task Main(string[] args) + { + // Initialize GDAL early + GDALSetup.InitializeMultiplatform(); -// ============ COMPUTE COMMAND ============ -Command computeCommand = new("compute", "Run computations on FDA studies and generate CSV result reports"); + // Create root command + RootCommand rootCommand = new("FDA Testing Utility - Regression Testing Tool for FDA Studies"); -Option computeConfigOption = new( - name: "--config", - description: "Path to JSON configuration file") -{ IsRequired = true }; -computeConfigOption.AddAlias("-c"); + // ============ COMPUTE COMMAND ============ + Command computeCommand = new("compute", "Run computations on FDA studies and generate CSV result reports"); -Option computeOutputOption = new( - name: "--output", - description: "Output directory for generated files", - getDefaultValue: () => new DirectoryInfo(Environment.CurrentDirectory)); -computeOutputOption.AddAlias("-o"); + Option computeConfigOption = new( + name: "--config", + description: "Path to JSON configuration file") + { IsRequired = true }; + computeConfigOption.AddAlias("-c"); -Option computeStudyOption = new( - name: "--study", - description: "Filter to specific study IDs (can specify multiple)") -{ AllowMultipleArgumentsPerToken = true }; -computeStudyOption.AddAlias("-s"); + Option computeOutputOption = new( + name: "--output", + description: "Output directory for generated files", + getDefaultValue: () => new DirectoryInfo(Environment.CurrentDirectory)); + computeOutputOption.AddAlias("-o"); -computeCommand.AddOption(computeConfigOption); -computeCommand.AddOption(computeOutputOption); -computeCommand.AddOption(computeStudyOption); + Option computeStudyOption = new( + name: "--study", + description: "Filter to specific study IDs (can specify multiple)") + { AllowMultipleArgumentsPerToken = true }; + computeStudyOption.AddAlias("-s"); -computeCommand.SetHandler(async (configFile, outputDir, studyFilter) => -{ - try - { - Console.WriteLine("FDA Testing Utility - Compute"); - Console.WriteLine("============================="); - Console.WriteLine(); + computeCommand.AddOption(computeConfigOption); + computeCommand.AddOption(computeOutputOption); + computeCommand.AddOption(computeStudyOption); - if (!configFile.Exists) + computeCommand.SetHandler(async (configFile, outputDir, studyFilter) => { - Console.WriteLine($"Error: Configuration file not found: {configFile.FullName}"); - Environment.Exit(1); - } + try + { + Console.WriteLine("FDA Testing Utility - Compute"); + Console.WriteLine("============================="); + Console.WriteLine(); - Console.WriteLine($"Loading configuration: {configFile.FullName}"); - TestConfiguration config = TestConfiguration.LoadFromFile(configFile.FullName); + if (!configFile.Exists) + { + Console.WriteLine($"Error: Configuration file not found: {configFile.FullName}"); + return; + } - if (!outputDir.Exists) - { - outputDir.Create(); - } + Console.WriteLine($"Loading configuration: {configFile.FullName}"); + TestConfiguration config = TestConfiguration.LoadFromFile(configFile.FullName); - ComputeRunner runner = new( - config, - outputDir.FullName, - studyFilter?.Length > 0 ? studyFilter : null); + if (!outputDir.Exists) + { + outputDir.Create(); + } - int exitCode = await runner.RunAsync(); - Environment.Exit(exitCode); - } - catch (Exception ex) - { - Console.WriteLine($"Fatal error: {ex.Message}"); - Console.WriteLine(ex.StackTrace); - Environment.Exit(1); - } -}, computeConfigOption, computeOutputOption, computeStudyOption); + ComputeRunner runner = new( + config, + outputDir.FullName, + studyFilter?.Length > 0 ? studyFilter : null); -// Add subcommands to root -rootCommand.AddCommand(computeCommand); + await runner.RunAsync(); + } + catch (Exception ex) + { + Console.WriteLine($"Fatal error: {ex.Message}"); + Console.WriteLine(ex.StackTrace); + } + }, computeConfigOption, computeOutputOption, computeStudyOption); -// Run -return await rootCommand.InvokeAsync(args); + // Add subcommands to root + rootCommand.AddCommand(computeCommand); + + // Run + return await rootCommand.InvokeAsync(args); + } +} \ No newline at end of file diff --git a/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs b/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs index 968612730..167dd832d 100644 --- a/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs +++ b/HEC.FDA.TestingUtility/Reporting/CsvReportFactory.cs @@ -78,7 +78,7 @@ public void AddStageDamageSummary(string studyId, string elementName, List 0 ? curve.Xvals!.Max() : 0; PairedData medianCurve = curve.SamplePairedData(.5); - double integral = medianCurve.integrate(); + double integral = medianCurve.Integrate(false); _stageDamageSummary.AppendLine($"{EscapeCsv(studyId)},{EscapeCsv(elementName)},{impactAreaId},{EscapeCsv(damCat)},{EscapeCsv(assetCat)},{pointCount},{minStage:F2},{maxStage:F2},{integral:F2}"); } diff --git a/HEC.FDA.TestingUtility/Services/StageDamageRunner.cs b/HEC.FDA.TestingUtility/Services/StageDamageRunner.cs index 69cad01a8..c7e1c5232 100644 --- a/HEC.FDA.TestingUtility/Services/StageDamageRunner.cs +++ b/HEC.FDA.TestingUtility/Services/StageDamageRunner.cs @@ -2,6 +2,7 @@ using HEC.FDA.Model.stageDamage; using HEC.FDA.ViewModel; using HEC.FDA.ViewModel.AggregatedStageDamage; +using HEC.FDA.ViewModel.Hydraulics; using HEC.FDA.ViewModel.Hydraulics.GriddedData; using HEC.FDA.ViewModel.ImpactArea; using HEC.FDA.ViewModel.Inventory; diff --git a/HEC.FDA.TestingUtility/Services/StudyLoader.cs b/HEC.FDA.TestingUtility/Services/StudyLoader.cs index e2ab84ad5..e8b5300b4 100644 --- a/HEC.FDA.TestingUtility/Services/StudyLoader.cs +++ b/HEC.FDA.TestingUtility/Services/StudyLoader.cs @@ -6,6 +6,7 @@ using HEC.FDA.ViewModel.FlowTransforms; using HEC.FDA.ViewModel.FrequencyRelationships; using HEC.FDA.ViewModel.GeoTech; +using HEC.FDA.ViewModel.Hydraulics; using HEC.FDA.ViewModel.Hydraulics.GriddedData; using HEC.FDA.ViewModel.ImpactArea; using HEC.FDA.ViewModel.ImpactAreaScenario; diff --git a/ScratchSpace/Beam/ValidationAndVerification.cs b/ScratchSpace/Beam/ValidationAndVerification.cs new file mode 100644 index 000000000..8959fb9eb --- /dev/null +++ b/ScratchSpace/Beam/ValidationAndVerification.cs @@ -0,0 +1,20 @@ +using System; +using System.IO; +using System.Threading.Tasks; + +namespace ScratchSpace.Beam; + +public static class ValidationAndVerification +{ + private static readonly string ConfigPath = Path.GetFullPath( + Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "Beam", "west-sac-config.json")); + + private static readonly string OutputPath = Path.GetFullPath( + Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "Beam", "output")); + + public static async Task RunValidationAndVerificationReport() + { + string[] args = ["compute", "-c", ConfigPath, "-o", OutputPath]; + return await HEC.FDA.TestingUtility.Program.Main(args); + } +} diff --git a/ScratchSpace/Beam/output/results_report.csv b/ScratchSpace/Beam/output/results_report.csv new file mode 100644 index 000000000..01b7340c6 --- /dev/null +++ b/ScratchSpace/Beam/output/results_report.csv @@ -0,0 +1,64 @@ +=== FDA COMPUTATION RESULTS REPORT === +Generated: 2026-03-23 11:45:55 + +=== SCENARIO RESULTS === +Study ID,Name,Analysis Year,Impact Area,Mean EAD,25th Percentile EAD,50th Percentile EAD,75th Percentile EAD +west-sac-mixed,WOP,2025,Sole,1905043.32,1332887.55,1804764.20,2373226.31 + +=== SCENARIO DAMAGE BY CATEGORY === +Study ID,Scenario Name,Analysis Year,Impact Area,Damage Category,Asset Category,Mean EAD +west-sac-mixed,WOP,2025,Sole,RES,Structure,171487.20 +west-sac-mixed,WOP,2025,Sole,RES,Content,110842.82 +west-sac-mixed,WOP,2025,Sole,RES,Other,0.00 +west-sac-mixed,WOP,2025,Sole,RES,Vehicle,0.00 +west-sac-mixed,WOP,2025,Sole,COM,Structure,351976.86 +west-sac-mixed,WOP,2025,Sole,COM,Content,767616.45 +west-sac-mixed,WOP,2025,Sole,COM,Other,0.00 +west-sac-mixed,WOP,2025,Sole,COM,Vehicle,0.00 +west-sac-mixed,WOP,2025,Sole,IND,Structure,93064.37 +west-sac-mixed,WOP,2025,Sole,IND,Content,383460.04 +west-sac-mixed,WOP,2025,Sole,IND,Other,0.00 +west-sac-mixed,WOP,2025,Sole,IND,Vehicle,0.00 +west-sac-mixed,WOP,2025,Sole,PUB,Structure,14586.62 +west-sac-mixed,WOP,2025,Sole,PUB,Content,12008.96 +west-sac-mixed,WOP,2025,Sole,PUB,Other,0.00 +west-sac-mixed,WOP,2025,Sole,PUB,Vehicle,0.00 + +=== SCENARIO PERFORMANCE === +Study ID,Name,Analysis Year,Impact Area,Threshold Type,Threshold Value,LT Risk 10yr,LT Risk 30yr,LT Risk 50yr,Mean AEP,Median AEP,Assurance 0.10,Assurance 0.04,Assurance 0.02,Assurance 0.01,Assurance 0.004,Assurance 0.002 +west-sac-mixed,WOP,2025,Sole,Default Exterior Stage,12.50,1.0000,1.0000,1.0000,0.998995,0.999097,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 + +=== SCENARIO ASSURANCE OF AEP === +Study ID,Name,Analysis Year,Impact Area,Threshold Type,Threshold Value,Mean AEP,Median AEP,90% Assurance AEP,Assurance of 0.10 AEP,Assurance of 0.04 AEP,Assurance of 0.02 AEP,Assurance of 0.01 AEP,Assurance of 0.004 AEP,Assurance of 0.002 AEP +west-sac-mixed,WOP,2025,Sole,Default Exterior Stage,12.50,0.998995,0.999097,0.999179,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 + +=== ALTERNATIVE RESULTS === +Study ID,Name,Impact Area,Base Year,Future Year,Discount Rate,Period of Analysis,Mean EqAD,25th Percentile EqAD,50th Percentile EqAD,75th Percentile EqAD + +=== ALTERNATIVE DAMAGE BY CATEGORY === +Study ID,Scenario Name,Impact Area,Damage Category,Asset Category,Mean EqAD + +=== STAGE DAMAGE SUMMARY === +Study ID,Element Name,Impact Area ID,Impact Area Name,Damage Category,Asset Category,Point Count,Min Stage,Max Stage, Median Sample Integral +west-sac-mixed,WOP,0,RES,Structure,124,11.33,41.57,19532836.97 +west-sac-mixed,WOP,0,RES,Content,124,11.33,41.57,11072203.33 +west-sac-mixed,WOP,0,RES,Other,124,11.33,41.57,0.00 +west-sac-mixed,WOP,0,RES,Vehicle,124,11.33,41.57,0.00 +west-sac-mixed,WOP,0,COM,Structure,124,11.33,41.57,19349898.65 +west-sac-mixed,WOP,0,COM,Content,124,11.33,41.57,35406768.31 +west-sac-mixed,WOP,0,COM,Other,124,11.33,41.57,0.00 +west-sac-mixed,WOP,0,COM,Vehicle,124,11.33,41.57,0.00 +west-sac-mixed,WOP,0,IND,Structure,124,11.33,41.57,4336490.28 +west-sac-mixed,WOP,0,IND,Content,124,11.33,41.57,16141781.26 +west-sac-mixed,WOP,0,IND,Other,124,11.33,41.57,0.00 +west-sac-mixed,WOP,0,IND,Vehicle,124,11.33,41.57,0.00 +west-sac-mixed,WOP,0,PUB,Structure,124,11.33,41.57,943265.66 +west-sac-mixed,WOP,0,PUB,Content,124,11.33,41.57,682918.17 +west-sac-mixed,WOP,0,PUB,Other,124,11.33,41.57,0.00 +west-sac-mixed,WOP,0,PUB,Vehicle,124,11.33,41.57,0.00 + +=== ALTERNATIVE COMPARISON SUMMARY === +Study ID,Report Name,With Project Alt,Impact Area ID,Without Proj EqAD,With Proj EqAD,EqAD Reduced,EqAD Reduced 25th Pct,EqAD Reduced 50th Pct,EqAD Reduced 75th Pct,Without Proj Base EAD,With Proj Base EAD,Base EAD Reduced,Without Proj Future EAD,With Proj Future EAD,Future EAD Reduced + +=== ALTERNATIVE COMPARISON BY CATEGORY === +Study ID,Report Name,With Project Alt,Impact Area ID,Damage Category,Asset Category,EqAD Reduced,Base Year EAD Reduced,Future Year EAD Reduced diff --git a/ScratchSpace/Beam/west-sac-config.json b/ScratchSpace/Beam/west-sac-config.json new file mode 100644 index 000000000..b9f9a8fc4 --- /dev/null +++ b/ScratchSpace/Beam/west-sac-config.json @@ -0,0 +1,19 @@ +{ + "testSuiteId": "west-sac-mixed", + "globalSettings": { + "localTempDirectory": "C:/temp/FDATests", + "timeoutMinutes": 60 + }, + "studies": [ + { + "studyId": "west-sac-mixed", + "studyName": "West Sac Mixed", + "networkSourcePath": "C:/nexus_share/_HEYDUMMY/2.0.2", + "runAllStageDamage": true, + "runAllScenarios": true, + "runAllAlternatives": true, + "runAllAlternativeComparisons": true, + "computations": [] + } + ] +} diff --git a/ScratchSpace/EntryPoints/Beam.cs b/ScratchSpace/EntryPoints/Beam.cs index 6a5c61733..acbb44317 100644 --- a/ScratchSpace/EntryPoints/Beam.cs +++ b/ScratchSpace/EntryPoints/Beam.cs @@ -33,6 +33,7 @@ using Statistics.Histograms; using ScottPlot; using HEC.FDA.Model.utilities; +using ScratchSpace.Beam; namespace ScratchSpace.EntryPoints; @@ -44,9 +45,9 @@ public static class Beam private static readonly string STUDY_PATH = @"C:\Users\HEC\Projects\AlaiWai2\AlaiWai2\AlaiWai2.sqlite"; private static readonly string SCENARIO_NAME = "FWOP"; // Leave empty to compute all scenarios, or set a specific name - public static void EntryPoint() + public static async void EntryPoint() { - RunScenarioCompute(); + int success = await ValidationAndVerification.RunValidationAndVerificationReport(); } /// diff --git a/ScratchSpace/ScratchSpace.csproj b/ScratchSpace/ScratchSpace.csproj index 6d8d008c4..a1e196057 100644 --- a/ScratchSpace/ScratchSpace.csproj +++ b/ScratchSpace/ScratchSpace.csproj @@ -18,6 +18,7 @@ +