test: Benchmark#15
Merged
Merged
Conversation
Contributor
There was a problem hiding this comment.
Pull request overview
Adds a BenchmarkDotNet-based benchmarks project to compare the in-tree OpenAPI C# generator against the latest released NuGet/tool version, and wires it into the solution and shared package versions.
Changes:
- Add
OpenApiCodeGenerator.Benchmarksproject (BenchmarkDotNet runner + side-by-side “current vs released” harness). - Add central package versions for BenchmarkDotNet and the released generator tool package.
- Include benchmark project in the solution and grant it internal access to the core generator assembly.
Reviewed changes
Copilot reviewed 10 out of 10 changed files in this pull request and generated 6 comments.
Show a summary per file
| File | Description |
|---|---|
| src/OpenApiCodeGenerator/OpenApiCodeGenerator.csproj | Adds InternalsVisibleTo for the benchmarks project. |
| OpenApiCodeGenerator.slnx | Adds a /benchmarks/ folder and includes the benchmarks project. |
| Directory.Packages.props | Adds centrally managed versions for BenchmarkDotNet + released tool package. |
| benchmarks/OpenApiCodeGenerator.Benchmarks/OpenApiCodeGenerator.Benchmarks.csproj | New benchmarks executable project; restores released package and copies its DLLs into an output subfolder. |
| benchmarks/OpenApiCodeGenerator.Benchmarks/Program.cs | Runs ComparisonBenchmarks via BenchmarkDotNet. |
| benchmarks/OpenApiCodeGenerator.Benchmarks/ComparisonBenchmarks.cs | Implements benchmark suite (Parse+Generate and GenerateOnly) for multiple real-world specs. |
| benchmarks/OpenApiCodeGenerator.Benchmarks/ReleasedGeneratorProxy.cs | Loads released generator into a separate AssemblyLoadContext and invokes it for comparison. |
| benchmarks/OpenApiCodeGenerator.Benchmarks/BenchmarkDotNet.Artifacts/results/ComparisonBenchmarks-report.html | Benchmark output artifact committed to repo. |
| benchmarks/OpenApiCodeGenerator.Benchmarks/BenchmarkDotNet.Artifacts/results/ComparisonBenchmarks-report.csv | Benchmark output artifact committed to repo. |
| benchmarks/OpenApiCodeGenerator.Benchmarks/BenchmarkDotNet.Artifacts/results/ComparisonBenchmarks-report-github.md | Benchmark output artifact committed to repo. |
💡 Add Copilot custom instructions for smarter, more guided reviews. Learn how to get started.
You can also share your feedback on Copilot code review. Take the survey.
Comment on lines
+40
to
+43
| public string GenerateFromText(string openApiText) | ||
| { | ||
| return (string) _generateFromText.Invoke(_generator, [openApiText])!; | ||
| } |
Comment on lines
+166
to
+167
| return result.Document | ||
| ?? throw new InvalidOperationException("Failed to parse OpenAPI document."); |
Comment on lines
+170
to
+177
| private static async Task<string> LoadSpecFromUrlAsync(Uri url) | ||
| { | ||
| Console.WriteLine($"Downloading spec from {url} ..."); | ||
|
|
||
| using var http = new HttpClient(); | ||
| http.DefaultRequestHeaders.UserAgent.ParseAdd("OpenApiCodeGenerator-Benchmarks/1.0"); | ||
|
|
||
| string spec = await http.GetStringAsync(url).ConfigureAwait(false); |
Comment on lines
+1
to
+5
| <!DOCTYPE html> | ||
| <html lang='en'> | ||
| <head> | ||
| <meta charset='utf-8' /> | ||
| <title>ComparisonBenchmarks-20260314-212521</title> |
Comment on lines
+1
to
+13
| Method;Job;AnalyzeLaunchVariance;EvaluateOverhead;MaxAbsoluteError;MaxRelativeError;MinInvokeCount;MinIterationTime;OutlierMode;Affinity;EnvironmentVariables;Jit;LargeAddressAware;Platform;PowerPlanMode;Runtime;AllowVeryLargeObjects;Concurrent;CpuGroups;Force;HeapAffinitizeMask;HeapCount;NoAffinitize;RetainVm;Server;Arguments;BuildConfiguration;Clock;EngineFactory;NuGetReferences;Toolchain;IsMutator;InvocationCount;IterationCount;IterationTime;LaunchCount;MaxIterationCount;MaxWarmupIterationCount;MemoryRandomization;MinIterationCount;MinWarmupIterationCount;RunStrategy;UnrollFactor;WarmupCount;Categories;Mean;Error;StdDev;Median;Ratio;RatioSD;Gen0;Gen1;Gen2;Allocated;Alloc Ratio | ||
| Current;DefaultJob;False;Default;Default;Default;Default;Default;Default;1111111111111111;Empty;RyuJit;Default;X64;8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c;.NET 10.0;False;True;False;True;Default;Default;False;False;False;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;16;Default;"GitHubApiNext,GenerateOnly";"19,590.73 μs";913.760 μs;"2,694.242 μs";"18,504.19 μs";?;?;687.5000;343.7500;15.6250;37497.21 KB;? | ||
| Released;DefaultJob;False;Default;Default;Default;Default;Default;Default;1111111111111111;Empty;RyuJit;Default;X64;8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c;.NET 10.0;False;True;False;True;Default;Default;False;False;False;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;16;Default;"GitHubApiNext,ParseAndGenerate";"4,275,059.67 μs";"82,573.361 μs";"113,027.358 μs";"4,264,655.55 μs";1.00;0.04;204000.0000;9000.0000;0.0000;10055299.83 KB;1.00 | ||
| Current;DefaultJob;False;Default;Default;Default;Default;Default;Default;1111111111111111;Empty;RyuJit;Default;X64;8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c;.NET 10.0;False;True;False;True;Default;Default;False;False;False;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;16;Default;"GitHubApiNext,ParseAndGenerate";"758,891.62 μs";"14,788.118 μs";"23,023.320 μs";"754,015.15 μs";0.18;0.01;13000.0000;8000.0000;0.0000;678324.84 KB;0.07 | ||
| Current;DefaultJob;False;Default;Default;Default;Default;Default;Default;1111111111111111;Empty;RyuJit;Default;X64;8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c;.NET 10.0;False;True;False;True;Default;Default;False;False;False;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;16;Default;"GitHubApi,GenerateOnly";"15,883.52 μs";308.995 μs;317.315 μs;"15,751.93 μs";?;?;718.7500;343.7500;0.0000;39430.52 KB;? | ||
| Released;DefaultJob;False;Default;Default;Default;Default;Default;Default;1111111111111111;Empty;RyuJit;Default;X64;8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c;.NET 10.0;False;True;False;True;Default;Default;False;False;False;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;16;Default;"GitHubApi,ParseAndGenerate";"610,237.06 μs";"14,185.529 μs";"40,928.499 μs";"601,908.25 μs";1.00;0.09;8000.0000;7000.0000;0.0000;435401.99 KB;1.00 | ||
| Current;DefaultJob;False;Default;Default;Default;Default;Default;Default;1111111111111111;Empty;RyuJit;Default;X64;8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c;.NET 10.0;False;True;False;True;Default;Default;False;False;False;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;16;Default;"GitHubApi,ParseAndGenerate";"548,678.89 μs";"10,787.996 μs";"19,175.653 μs";"548,303.45 μs";0.90;0.07;8000.0000;7000.0000;0.0000;412075.61 KB;0.95 | ||
| Current;DefaultJob;False;Default;Default;Default;Default;Default;Default;1111111111111111;Empty;RyuJit;Default;X64;8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c;.NET 10.0;False;True;False;True;Default;Default;False;False;False;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;16;Default;"Petstore,GenerateOnly";18.11 μs;0.732 μs;2.053 μs;17.33 μs;?;?;1.1292;0.0305;0.0000;56.34 KB;? | ||
| Released;DefaultJob;False;Default;Default;Default;Default;Default;Default;1111111111111111;Empty;RyuJit;Default;X64;8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c;.NET 10.0;False;True;False;True;Default;Default;False;False;False;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;16;Default;"Petstore,ParseAndGenerate";384.84 μs;7.691 μs;9.155 μs;382.91 μs;1.00;0.03;17.5781;7.8125;0.0000;883.69 KB;1.00 | ||
| Current;DefaultJob;False;Default;Default;Default;Default;Default;Default;1111111111111111;Empty;RyuJit;Default;X64;8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c;.NET 10.0;False;True;False;True;Default;Default;False;False;False;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;16;Default;"Petstore,ParseAndGenerate";474.75 μs;26.998 μs;79.604 μs;454.69 μs;1.23;0.21;16.6016;5.8594;0.0000;849.99 KB;0.96 | ||
| Current;DefaultJob;False;Default;Default;Default;Default;Default;Default;1111111111111111;Empty;RyuJit;Default;X64;8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c;.NET 10.0;False;True;False;True;Default;Default;False;False;False;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;16;Default;"StripeApi,GenerateOnly";"18,655.74 μs";923.037 μs;"2,721.595 μs";"18,123.93 μs";?;?;625.0000;250.0000;31.2500;34779.54 KB;? | ||
| Released;DefaultJob;False;Default;Default;Default;Default;Default;Default;1111111111111111;Empty;RyuJit;Default;X64;8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c;.NET 10.0;False;True;False;True;Default;Default;False;False;False;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;16;Default;"StripeApi,ParseAndGenerate";"419,754.60 μs";"15,256.846 μs";"44,745.693 μs";"417,412.80 μs";1.01;0.15;5000.0000;4000.0000;0.0000;275602.76 KB;1.00 | ||
| Current;DefaultJob;False;Default;Default;Default;Default;Default;Default;1111111111111111;Empty;RyuJit;Default;X64;8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c;.NET 10.0;False;True;False;True;Default;Default;False;False;False;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;Default;16;Default;"StripeApi,ParseAndGenerate";"393,432.72 μs";"16,806.124 μs";"49,553.240 μs";"377,968.55 μs";0.95;0.15;4000.0000;3000.0000;0.0000;255438.53 KB;0.93 |
Comment on lines
+1
to
+20
| ``` | ||
|
|
||
| BenchmarkDotNet v0.15.8, Windows 11 (10.0.26200.7840/25H2/2025Update/HudsonValley2) | ||
| AMD Ryzen 7 7800X3D 4.20GHz, 1 CPU, 16 logical and 8 physical cores | ||
| .NET SDK 10.0.200-preview.0.26103.119 | ||
| [Host] : .NET 10.0.3 (10.0.3, 10.0.326.7603), X64 RyuJIT x86-64-v4 | ||
| DefaultJob : .NET 10.0.3 (10.0.3, 10.0.326.7603), X64 RyuJIT x86-64-v4 | ||
|
|
||
|
|
||
| ``` | ||
| | Method | Categories | Mean | Error | StdDev | Median | Ratio | RatioSD | Gen0 | Gen1 | Gen2 | Allocated | Alloc Ratio | | ||
| |--------- |------------------------------- |----------------:|--------------:|---------------:|----------------:|------:|--------:|------------:|----------:|--------:|---------------:|------------:| | ||
| | Current | GitHubApiNext,GenerateOnly | 19,590.73 μs | 913.760 μs | 2,694.242 μs | 18,504.19 μs | ? | ? | 687.5000 | 343.7500 | 15.6250 | 37497.21 KB | ? | | ||
| | | | | | | | | | | | | | | | ||
| | Released | GitHubApiNext,ParseAndGenerate | 4,275,059.67 μs | 82,573.361 μs | 113,027.358 μs | 4,264,655.55 μs | 1.00 | 0.04 | 204000.0000 | 9000.0000 | - | 10055299.83 KB | 1.00 | | ||
| | Current | GitHubApiNext,ParseAndGenerate | 758,891.62 μs | 14,788.118 μs | 23,023.320 μs | 754,015.15 μs | 0.18 | 0.01 | 13000.0000 | 8000.0000 | - | 678324.84 KB | 0.07 | | ||
| | | | | | | | | | | | | | | | ||
| | Current | GitHubApi,GenerateOnly | 15,883.52 μs | 308.995 μs | 317.315 μs | 15,751.93 μs | ? | ? | 718.7500 | 343.7500 | - | 39430.52 KB | ? | | ||
| | | | | | | | | | | | | | | | ||
| | Released | GitHubApi,ParseAndGenerate | 610,237.06 μs | 14,185.529 μs | 40,928.499 μs | 601,908.25 μs | 1.00 | 0.09 | 8000.0000 | 7000.0000 | - | 435401.99 KB | 1.00 | |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
No description provided.