Skip to content

Commit dded66f

Browse files
KSemenenkoKSemenenko
committed
Prepare release performance gates
1 parent ed2f4d2 commit dded66f

20 files changed

Lines changed: 482 additions & 206 deletions

‎.github/workflows/benchmarks.yml‎

Lines changed: 21 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,27 @@ env:
1010

1111
jobs:
1212
benchmark:
13-
name: BenchmarkDotNet full suite
13+
name: BenchmarkDotNet (${{ matrix.suite }})
1414
runs-on: ubuntu-latest
15-
timeout-minutes: 120
15+
timeout-minutes: 30
1616
permissions:
1717
contents: read
18+
strategy:
19+
fail-fast: false
20+
matrix:
21+
include:
22+
- suite: fuzzy-edit-distance
23+
filter: '*FuzzyEditDistanceBenchmarks*'
24+
- suite: graph-build
25+
filter: '*GraphBuildBenchmarks*'
26+
- suite: graph-search
27+
filter: '*GraphSearchBenchmarks*'
28+
- suite: tiktoken-search
29+
filter: '*TiktokenSearchBenchmarks*'
30+
- suite: graph-persistence
31+
filter: '*GraphPersistenceBenchmarks*'
32+
- suite: graph-lifecycle
33+
filter: '*GraphLifecycleBenchmarks*'
1834

1935
steps:
2036
- name: Checkout
@@ -33,28 +49,13 @@ jobs:
3349
- name: Build
3450
run: dotnet build MarkdownLd.Kb.slnx --configuration Release --no-restore
3551

36-
- name: Run fuzzy edit-distance benchmarks
37-
run: dotnet run --project benchmarks/MarkdownLd.Kb.Benchmarks -c Release -- --filter "*FuzzyEditDistanceBenchmarks*"
38-
39-
- name: Run graph build benchmarks
40-
run: dotnet run --project benchmarks/MarkdownLd.Kb.Benchmarks -c Release -- --filter "*GraphBuildBenchmarks*"
41-
42-
- name: Run graph search benchmarks
43-
run: dotnet run --project benchmarks/MarkdownLd.Kb.Benchmarks -c Release -- --filter "*GraphSearchBenchmarks*"
44-
45-
- name: Run Tiktoken search benchmarks
46-
run: dotnet run --project benchmarks/MarkdownLd.Kb.Benchmarks -c Release -- --filter "*TiktokenSearchBenchmarks*"
47-
48-
- name: Run graph persistence benchmarks
49-
run: dotnet run --project benchmarks/MarkdownLd.Kb.Benchmarks -c Release -- --filter "*GraphPersistenceBenchmarks*"
50-
51-
- name: Run graph lifecycle benchmarks
52-
run: dotnet run --project benchmarks/MarkdownLd.Kb.Benchmarks -c Release -- --filter "*GraphLifecycleBenchmarks*"
52+
- name: Run ${{ matrix.suite }} benchmarks
53+
run: dotnet run --project benchmarks/MarkdownLd.Kb.Benchmarks -c Release -- --filter "${{ matrix.filter }}"
5354

5455
- name: Upload benchmark artifacts
5556
uses: actions/upload-artifact@v7
5657
with:
57-
name: benchmarkdotnet-results
58+
name: benchmarkdotnet-results-${{ matrix.suite }}
5859
path: artifacts/benchmarks/results
5960
if-no-files-found: error
6061
retention-days: 14

‎.github/workflows/release.yml‎

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,9 +74,60 @@ jobs:
7474
path: ${{ env.ARTIFACTS_DIR }}/*.nupkg
7575
retention-days: 5
7676

77+
benchmark:
78+
name: BenchmarkDotNet (${{ matrix.suite }})
79+
runs-on: ubuntu-latest
80+
timeout-minutes: 30
81+
permissions:
82+
contents: read
83+
strategy:
84+
fail-fast: false
85+
matrix:
86+
include:
87+
- suite: fuzzy-edit-distance
88+
filter: '*FuzzyEditDistanceBenchmarks*'
89+
- suite: graph-build
90+
filter: '*GraphBuildBenchmarks*'
91+
- suite: graph-search
92+
filter: '*GraphSearchBenchmarks*'
93+
- suite: tiktoken-search
94+
filter: '*TiktokenSearchBenchmarks*'
95+
- suite: graph-persistence
96+
filter: '*GraphPersistenceBenchmarks*'
97+
- suite: graph-lifecycle
98+
filter: '*GraphLifecycleBenchmarks*'
99+
100+
steps:
101+
- name: Checkout
102+
uses: actions/checkout@v6
103+
with:
104+
submodules: true
105+
106+
- name: Setup .NET
107+
uses: actions/setup-dotnet@v5
108+
with:
109+
dotnet-version: ${{ env.DOTNET_VERSION }}
110+
111+
- name: Restore dependencies
112+
run: dotnet restore ${{ env.SOLUTION }}
113+
114+
- name: Build
115+
run: dotnet build ${{ env.SOLUTION }} --configuration Release --no-restore
116+
117+
- name: Run ${{ matrix.suite }} benchmarks
118+
run: dotnet run --project benchmarks/MarkdownLd.Kb.Benchmarks -c Release -- --filter "${{ matrix.filter }}"
119+
120+
- name: Upload benchmark artifacts
121+
uses: actions/upload-artifact@v7
122+
with:
123+
name: benchmarkdotnet-results-${{ matrix.suite }}
124+
path: artifacts/benchmarks/results
125+
if-no-files-found: error
126+
retention-days: 5
127+
77128
publish-nuget:
78129
name: Publish to NuGet
79-
needs: build
130+
needs: [build, benchmark]
80131
runs-on: ubuntu-latest
81132
if: github.ref == 'refs/heads/main'
82133
permissions:

‎.github/workflows/validation.yml‎

Lines changed: 52 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@ env:
1212

1313
jobs:
1414
validate:
15-
name: Build, test, benchmark, and pack
15+
name: Build, test, and pack
1616
runs-on: ubuntu-latest
17-
timeout-minutes: 120
17+
timeout-minutes: 30
1818
permissions:
1919
contents: read
2020

@@ -41,24 +41,6 @@ jobs:
4141
- name: Test with coverage
4242
run: dotnet test --solution MarkdownLd.Kb.slnx --configuration Release --verbosity normal -- --coverage --coverage-output-format cobertura --coverage-output "${{ github.workspace }}/TestResults/TUnitCoverage/coverage.cobertura.xml" --coverage-settings "${{ github.workspace }}/CodeCoverage.runsettings"
4343

44-
- name: Run fuzzy edit-distance benchmarks
45-
run: dotnet run --project benchmarks/MarkdownLd.Kb.Benchmarks -c Release -- --filter "*FuzzyEditDistanceBenchmarks*"
46-
47-
- name: Run graph build benchmarks
48-
run: dotnet run --project benchmarks/MarkdownLd.Kb.Benchmarks -c Release -- --filter "*GraphBuildBenchmarks*"
49-
50-
- name: Run graph search benchmarks
51-
run: dotnet run --project benchmarks/MarkdownLd.Kb.Benchmarks -c Release -- --filter "*GraphSearchBenchmarks*"
52-
53-
- name: Run Tiktoken search benchmarks
54-
run: dotnet run --project benchmarks/MarkdownLd.Kb.Benchmarks -c Release -- --filter "*TiktokenSearchBenchmarks*"
55-
56-
- name: Run graph persistence benchmarks
57-
run: dotnet run --project benchmarks/MarkdownLd.Kb.Benchmarks -c Release -- --filter "*GraphPersistenceBenchmarks*"
58-
59-
- name: Run graph lifecycle benchmarks
60-
run: dotnet run --project benchmarks/MarkdownLd.Kb.Benchmarks -c Release -- --filter "*GraphLifecycleBenchmarks*"
61-
6244
- name: Pack
6345
run: dotnet pack MarkdownLd.Kb.slnx --configuration Release -p:IncludeSymbols=false -p:SymbolPackageFormat=snupkg --output ./artifacts
6446

@@ -69,17 +51,60 @@ jobs:
6951
path: TestResults/TUnitCoverage/coverage.cobertura.xml
7052
retention-days: 5
7153

72-
- name: Upload benchmark artifact
54+
- name: Upload package artifact
7355
uses: actions/upload-artifact@v7
7456
with:
75-
name: benchmarkdotnet-results
76-
path: artifacts/benchmarks/results
77-
if-no-files-found: error
57+
name: nuget-validation-packages
58+
path: ./artifacts/*.nupkg
7859
retention-days: 5
7960

80-
- name: Upload package artifact
61+
benchmark:
62+
name: BenchmarkDotNet (${{ matrix.suite }})
63+
runs-on: ubuntu-latest
64+
timeout-minutes: 30
65+
permissions:
66+
contents: read
67+
strategy:
68+
fail-fast: false
69+
matrix:
70+
include:
71+
- suite: fuzzy-edit-distance
72+
filter: '*FuzzyEditDistanceBenchmarks*'
73+
- suite: graph-build
74+
filter: '*GraphBuildBenchmarks*'
75+
- suite: graph-search
76+
filter: '*GraphSearchBenchmarks*'
77+
- suite: tiktoken-search
78+
filter: '*TiktokenSearchBenchmarks*'
79+
- suite: graph-persistence
80+
filter: '*GraphPersistenceBenchmarks*'
81+
- suite: graph-lifecycle
82+
filter: '*GraphLifecycleBenchmarks*'
83+
84+
steps:
85+
- name: Checkout
86+
uses: actions/checkout@v6
87+
with:
88+
submodules: true
89+
90+
- name: Setup .NET
91+
uses: actions/setup-dotnet@v5
92+
with:
93+
dotnet-version: ${{ env.DOTNET_VERSION }}
94+
95+
- name: Restore dependencies
96+
run: dotnet restore MarkdownLd.Kb.slnx
97+
98+
- name: Build
99+
run: dotnet build MarkdownLd.Kb.slnx --configuration Release --no-restore
100+
101+
- name: Run ${{ matrix.suite }} benchmarks
102+
run: dotnet run --project benchmarks/MarkdownLd.Kb.Benchmarks -c Release -- --filter "${{ matrix.filter }}"
103+
104+
- name: Upload benchmark artifact
81105
uses: actions/upload-artifact@v7
82106
with:
83-
name: nuget-validation-packages
84-
path: ./artifacts/*.nupkg
107+
name: benchmarkdotnet-results-${{ matrix.suite }}
108+
path: artifacts/benchmarks/results
109+
if-no-files-found: error
85110
retention-days: 5

‎README.md‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1305,20 +1305,20 @@ dotnet test --solution MarkdownLd.Kb.slnx --configuration Release -- --coverage
13051305

13061306
Coverage is collected through `Microsoft.Testing.Extensions.CodeCoverage`. Cobertura is the XML output format used for line and branch reporting; the test project does not reference Coverlet.
13071307

1308-
BenchmarkDotNet performance runs are separate from TUnit correctness tests. Commands, workload profiles, profiler options, and full result tables live in [Performance Benchmarks](docs/Features/PerformanceBenchmarks.md). PR validation and the dedicated benchmark workflow both run the complete BenchmarkDotNet suite and upload the `benchmarkdotnet-results` artifact.
1308+
BenchmarkDotNet performance runs are separate from TUnit correctness tests. Commands, workload profiles, profiler options, and full result tables live in [Performance Benchmarks](docs/Features/PerformanceBenchmarks.md). The build/test/pack validation job stays separate; PR validation, release validation, and the dedicated benchmark workflow run the complete BenchmarkDotNet suite as parallel suite jobs and upload suite-specific `benchmarkdotnet-results-*` artifacts.
13091309

1310-
Current local headline numbers from the May 3, 2026 BenchmarkDotNet 0.15.8 run on Apple M2 Pro with .NET 10.0.5:
1310+
Current local headline numbers from the May 4, 2026 BenchmarkDotNet 0.15.8 run on Apple M2 Pro with .NET 10.0.5:
13111311

13121312
| Area | Current local result |
13131313
| --- | --- |
1314-
| Full suite | 118 BenchmarkDotNet cases using the `Default` job |
1315-
| Graph build | `LargeCorpus` builds in 47.851 ms with 57.73 MB allocated |
1316-
| Low-latency search | `ShortDocuments` exact ranked graph search is 1.092 ms / 2.17 MB; BM25 is 1.309 ms / 2.14 MB |
1317-
| Typo-tolerant search | BM25 fuzzy stays opt-in; `ShortDocuments` typo fuzzy search is 1.815 ms / 2.86 MB |
1318-
| RDF query paths | `ShortDocuments` exact schema SPARQL is 49.212 ms / 60.32 MB; local federated schema search is 41.243 ms / 62.3 MB |
1319-
| Tiktoken search | `LongDocuments` exact token-distance search is 159.8 us / 107.27 KB; typo correction is 225.7 us / 110.68 KB |
1320-
| Persistence | `LargeCorpus` Turtle file load is 34.787 ms / 28.10 MB; JSON-LD file load is 98.267 ms / 75.32 MB |
1321-
| Lifecycle | Build/search/save/load/export is 45.44 ms / 53.51 MB |
1322-
| Fuzzy edit distance | Long insertion is 368.69x faster than naive Levenshtein; long no-match is 176.19x faster, both with 0 B allocated |
1314+
| Full suite | 118 BenchmarkDotNet cases using the `Required` job; local sequential pass completed in 5 minutes 41 seconds (`real 341.12s`) |
1315+
| Graph build | `LargeCorpus` builds in 151.12 ms with 58.75 MB allocated |
1316+
| Low-latency search | `ShortDocuments` exact ranked graph search is 1.143 ms / 2.15 MB; BM25 is 2.503 ms / 2.14 MB |
1317+
| Typo-tolerant search | BM25 fuzzy stays opt-in; `ShortDocuments` typo fuzzy search is 7.366 ms / 2.77 MB |
1318+
| RDF query paths | `ShortDocuments` exact schema SPARQL is 94.422 ms / 61.25 MB; local federated schema search is 92.469 ms / 63.24 MB |
1319+
| Tiktoken search | `LongDocuments` exact token-distance search is 145.52 μs / 107.23 KB; typo correction is 250.10 μs / 110.17 KB |
1320+
| Persistence | `LargeCorpus` Turtle file load is 118.305 ms / 28.10 MB; JSON-LD file load is 163.824 ms / 75.51 MB |
1321+
| Lifecycle | Build/search/save/load/export is 154.3 ms / 53.7 MB |
1322+
| Fuzzy edit distance | Long insertion is 381.38x faster than naive Levenshtein; long no-match is 178.03x faster, both with 0 B allocated |
13231323

13241324
These numbers are local diagnostics, not a cross-machine performance contract.

‎benchmarks/MarkdownLd.Kb.Benchmarks/MarkdownLdBenchmarkConfig.cs‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
using BenchmarkDotNet.Exporters.Json;
77
using BenchmarkDotNet.Jobs;
88
using BenchmarkDotNet.Loggers;
9+
using Perfolizer.Horology;
910

1011
namespace ManagedCode.MarkdownLd.Kb.Benchmarks;
1112

@@ -18,12 +19,16 @@ public sealed class MarkdownLdBenchmarkConfig : ManualConfig
1819
private const string JitProfileValue = "jit";
1920
private const string JobLongOption = "--job";
2021
private const string JobShortOption = "-j";
22+
private const int RequiredLaunchCount = 1;
23+
private const int RequiredWarmupCount = 2;
24+
private const int RequiredIterationCount = 5;
25+
private static readonly TimeInterval RequiredMinIterationTime = TimeInterval.FromMilliseconds(100);
2126

2227
public MarkdownLdBenchmarkConfig(IReadOnlyList<string> args)
2328
{
2429
AddLogger(ConsoleLogger.Default);
2530
AddColumnProvider(DefaultColumnProviders.Instance);
26-
AddDefaultJob(args);
31+
AddRequiredJob(args);
2732
AddExporter(MarkdownExporter.GitHub);
2833
AddExporter(CsvExporter.Default);
2934
AddExporter(JsonExporter.Full);
@@ -33,14 +38,20 @@ public MarkdownLdBenchmarkConfig(IReadOnlyList<string> args)
3338
ArtifactsPath = Path.Combine(Directory.GetCurrentDirectory(), ArtifactsDirectory);
3439
}
3540

36-
private void AddDefaultJob(IReadOnlyList<string> args)
41+
private void AddRequiredJob(IReadOnlyList<string> args)
3742
{
3843
if (args.Any(IsJobOption))
3944
{
4045
return;
4146
}
4247

43-
AddJob(Job.Default.WithId("Default"));
48+
AddJob(
49+
Job.Default
50+
.WithId("Required")
51+
.WithLaunchCount(RequiredLaunchCount)
52+
.WithWarmupCount(RequiredWarmupCount)
53+
.WithIterationCount(RequiredIterationCount)
54+
.WithMinIterationTime(RequiredMinIterationTime));
4455
}
4556

4657
private static bool IsJobOption(string arg)

‎docs/Features/HybridGraphSearch.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ flowchart LR
5151
## Intended Library Boundary
5252

5353
- The library owns the graph-native candidate extraction and merge policy.
54-
- BM25 ranking is in-memory and uses the same candidate text as semantic indexing for the selected boundary: graph-native for graph-only callers, document-aware for build-result/facade callers. Exact BM25 keeps the allocation profile close to ranked graph search by avoiding full per-document dictionaries. Optional fuzzy token matching is implemented inside this library, does not add an external package dependency, and does not copy AGPL-licensed code from reviewed prior-art projects.
54+
- BM25 ranking is in-memory and uses the same candidate text as semantic indexing for the selected boundary: graph-native for graph-only callers, document-aware for build-result/facade callers. Exact BM25 keeps the allocation profile close to ranked graph search by avoiding full per-document dictionaries. Optional fuzzy token matching is implemented inside this library, does not add an external package dependency, and follows public bounded edit-distance algorithms with our own implementation.
5555
- The host application owns the concrete embedding provider and supplies it as `Microsoft.Extensions.AI.IEmbeddingGenerator<string, Embedding<float>>`.
5656
- The library does not own a vector database, gateway endpoint, or hosted ranking infrastructure.
5757

0 commit comments

Comments
 (0)