Skip to content

Commit 6131132

Browse files
SockworkOrangeSockworkOrange
committed
fix: more benchmark refactoring
1 parent e30d12f commit 6131132

23 files changed

Lines changed: 158 additions & 231 deletions

‎Makefile‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -46,12 +46,28 @@ $(WASM_PLUGIN_OUTPUT): $(WASM_SOURCES)
4646
./scripts/wasm/copy_plugin_to.sh dist
4747
./scripts/wasm/update_sha.sh sqlc.ci.yaml
4848

49-
run-benchmark: sqlc-generate
50-
./benchmark/scripts/run_benchmark_all_dbs.sh
51-
5249
run-end2end-tests:
5350
./end2end/scripts/run_tests.sh
54-
51+
52+
# Benchmarks
53+
run-benchmark-sqlite-reads: sqlc-generate
54+
./benchmark/scripts/run_single_benchmark.sh sqlite reads
55+
56+
run-benchmark-sqlite-writes: sqlc-generate
57+
./benchmark/scripts/run_single_benchmark.sh sqlite writes
58+
59+
run-benchmark-postgresql-reads: sqlc-generate
60+
./benchmark/scripts/run_single_benchmark.sh postgresql reads
61+
62+
run-benchmark-postgresql-writes: sqlc-generate
63+
./benchmark/scripts/run_single_benchmark.sh postgresql writes
64+
65+
run-benchmark-mysql-reads: sqlc-generate
66+
./benchmark/scripts/run_single_benchmark.sh mysql reads
67+
68+
run-benchmark-mysql-writes: sqlc-generate
69+
./benchmark/scripts/run_single_benchmark.sh mysql writes
70+
5571
# Manual
5672
generate-protobuf:
5773
./scripts/generate_protobuf.sh
@@ -60,4 +76,4 @@ dotnet-format:
6076
dotnet format \
6177
--exclude GeneratedProtobuf \
6278
--exclude examples \
63-
--exclude benchmark/*SqlcImpl
79+
--exclude benchmark/*SqlcImpl

‎benchmark/BenchmarkRunner/Benchmarks/BaseBenchmark.cs‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
public abstract class BaseBenchmark
2+
{
3+
protected bool _isInitialized = false;
4+
protected SemaphoreSlim _initLock = new(1, 1);
5+
6+
public async Task InitializeOnceAsync(Func<Task> task)
7+
{
8+
if (_isInitialized) return;
9+
await _initLock.WaitAsync();
10+
try
11+
{
12+
if (_isInitialized) return;
13+
await task.Invoke();
14+
_isInitialized = true;
15+
}
16+
finally
17+
{
18+
_initLock.Release();
19+
}
20+
}
21+
22+
public static void InvokeGarbageCollection()
23+
{
24+
GC.Collect();
25+
GC.WaitForPendingFinalizers();
26+
GC.Collect();
27+
}
28+
}

‎benchmark/BenchmarkRunner/Benchmarks/BaseReadBenchmark.cs‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
using BenchmarkDotNet.Attributes;
2+
3+
public abstract class BaseReadBenchmark : BaseBenchmark
4+
{
5+
[Params(1000)]
6+
public int QueriesToRun { get; set; }
7+
8+
[Params(500)]
9+
public int CustomerCount { get; set; }
10+
11+
[Params(100, 500)]
12+
public int Limit { get; set; }
13+
14+
[IterationSetup]
15+
public static void IterationSetup() => InvokeGarbageCollection();
16+
public abstract Task Sqlc_GetCustomerOrders();
17+
public abstract Task EFCore_NoTracking_GetCustomerOrders();
18+
public abstract Task EFCore_WithTracking_GetCustomerOrders();
19+
20+
private static int CalculateMaxConcurrency(int totalTasks, int maxConcurrency)
21+
{
22+
return new int[] {
23+
maxConcurrency, totalTasks, Environment.ProcessorCount
24+
}.Min(x => x);
25+
}
26+
public static async Task<List<T>> ExecuteConcurrentlyAsync<T>(
27+
int totalTasks,
28+
int maxConcurrency,
29+
Func<int, Task<List<T>>> taskFactory)
30+
{
31+
maxConcurrency = CalculateMaxConcurrency(totalTasks, maxConcurrency);
32+
using var semaphore = new SemaphoreSlim(maxConcurrency, maxConcurrency);
33+
var tasks = new List<Task<List<T>>>();
34+
for (int i = 0; i < totalTasks; i++)
35+
{
36+
var index = i; // Capture for closure
37+
tasks.Add(ExecuteWithThrottleAsync(semaphore, () => taskFactory(index)));
38+
}
39+
40+
var results = await Task.WhenAll([.. tasks]);
41+
return [.. results.SelectMany(r => r)];
42+
}
43+
44+
private static async Task<List<T>> ExecuteWithThrottleAsync<T>(SemaphoreSlim semaphore, Func<Task<List<T>>> taskFactory)
45+
{
46+
await semaphore.WaitAsync();
47+
try
48+
{
49+
return await taskFactory();
50+
}
51+
finally
52+
{
53+
semaphore.Release();
54+
}
55+
}
56+
}

benchmark/BenchmarkRunner/Benchmarks/WriteBenchmark.cs renamed to benchmark/BenchmarkRunner/Benchmarks/BaseWriteBenchmark.cs

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,7 @@
11
using BenchmarkDotNet.Attributes;
22

3-
public abstract class WriteBenchmark
3+
public abstract class BaseWriteBenchmark : BaseBenchmark
44
{
5-
protected bool _isInitialized = false;
6-
protected SemaphoreSlim _initLock = new(1, 1);
7-
85
[Params(2000000)]
96
public int TotalRecords { get; set; }
107

‎benchmark/BenchmarkRunner/Benchmarks/Consts.cs‎

Lines changed: 0 additions & 3 deletions
This file was deleted.

‎benchmark/BenchmarkRunner/Benchmarks/MysqlReadBenchmark.cs‎

Lines changed: 6 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ namespace BenchmarkRunner.Benchmarks;
1212
[MarkdownExporterAttribute.GitHub]
1313
[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)]
1414
[CategoriesColumn]
15-
public class MysqlReadBenchmark : ReadBenchmark
15+
public sealed class MysqlReadBenchmark : BaseReadBenchmark
1616
{
1717
private static readonly string _connectionString = Config.GetMysqlConnectionString();
1818
private readonly QuerySql _sqlcImpl = new(_connectionString);
@@ -23,12 +23,8 @@ public class MysqlReadBenchmark : ReadBenchmark
2323
[GlobalSetup]
2424
public async Task GlobalSetup()
2525
{
26-
if (_isInitialized) return;
27-
await _initLock.WaitAsync();
28-
try
26+
await InitializeOnceAsync(async () =>
2927
{
30-
if (_isInitialized) return;
31-
3228
await MysqlDatabaseHelper.CleanupDatabaseAsync(_connectionString);
3329
var seeder = new MysqlDatabaseSeeder(_connectionString);
3430
await seeder.SeedAsync(
@@ -38,22 +34,14 @@ await seeder.SeedAsync(
3834
itemsPerOrder: 20
3935
// 20 * 1000 = 20,000 possible rows returned
4036
);
41-
_isInitialized = true;
42-
}
43-
finally
44-
{
45-
_initLock.Release();
46-
}
37+
});
4738
}
4839

49-
[IterationSetup]
50-
public static void IterationSetup() => Helpers.InvokeGarbageCollection();
51-
5240
[BenchmarkCategory("Read")]
5341
[Benchmark(Baseline = true, Description = "SQLC - GetCustomerOrders")]
5442
public override async Task Sqlc_GetCustomerOrders()
5543
{
56-
await Helpers.ExecuteConcurrentlyAsync(QueriesToRun, ConcurrentQueries, _ =>
44+
await ExecuteConcurrentlyAsync(QueriesToRun, ConcurrentQueries, _ =>
5745
{
5846
return _sqlcImpl.GetCustomerOrdersAsync(new QuerySql.GetCustomerOrdersArgs(
5947
CustomerId: Random.Shared.Next(1, CustomerCount),
@@ -67,7 +55,7 @@ await Helpers.ExecuteConcurrentlyAsync(QueriesToRun, ConcurrentQueries, _ =>
6755
[Benchmark(Description = "EFCore (NoTracking) - GetCustomerOrders")]
6856
public override async Task EFCore_NoTracking_GetCustomerOrders()
6957
{
70-
await Helpers.ExecuteConcurrentlyAsync(QueriesToRun, ConcurrentQueries, async _ =>
58+
await ExecuteConcurrentlyAsync(QueriesToRun, ConcurrentQueries, async _ =>
7159
{
7260
await using var dbContext = new SalesDbContext(_connectionString);
7361
var queries = new Queries(dbContext, useTracking: false);
@@ -83,7 +71,7 @@ await Helpers.ExecuteConcurrentlyAsync(QueriesToRun, ConcurrentQueries, async _
8371
[Benchmark(Description = "EFCore (WithTracking) - GetCustomerOrders")]
8472
public override async Task EFCore_WithTracking_GetCustomerOrders()
8573
{
86-
await Helpers.ExecuteConcurrentlyAsync(QueriesToRun, ConcurrentQueries, async _ =>
74+
await ExecuteConcurrentlyAsync(QueriesToRun, ConcurrentQueries, async _ =>
8775
{
8876
await using var dbContext = new SalesDbContext(_connectionString);
8977
var queries = new Queries(dbContext, useTracking: true);

‎benchmark/BenchmarkRunner/Benchmarks/MysqlWriteBenchmark.cs‎

Lines changed: 4 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ namespace BenchmarkRunner.Benchmarks;
1313
[MarkdownExporterAttribute.GitHub]
1414
[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)]
1515
[CategoriesColumn]
16-
public class MysqlWriteBenchmark : WriteBenchmark
16+
public sealed class MysqlWriteBenchmark : BaseWriteBenchmark
1717
{
1818
private static readonly string _connectionString = Config.GetMysqlConnectionString();
1919
private readonly QuerySql _sqlcImpl = new(_connectionString);
@@ -30,12 +30,8 @@ public class MysqlWriteBenchmark : WriteBenchmark
3030
[GlobalSetup]
3131
public async Task GlobalSetup()
3232
{
33-
if (_isInitialized) return;
34-
await _initLock.WaitAsync();
35-
try
33+
await InitializeOnceAsync(async () =>
3634
{
37-
if (_isInitialized) return;
38-
3935
await MysqlDatabaseHelper.CleanupDatabaseAsync(_connectionString);
4036
var seeder = new MysqlDatabaseSeeder(_connectionString);
4137
await seeder.SeedAsync(
@@ -54,20 +50,14 @@ await seeder.SeedAsync(
5450
Quantity: Random.Shared.Next(1, 10),
5551
UnitPrice: (decimal)(Random.Shared.NextDouble() * 100 + 5)
5652
))];
57-
58-
_isInitialized = true;
59-
}
60-
finally
61-
{
62-
_initLock.Release();
63-
}
53+
});
6454
}
6555

6656
[IterationSetup]
6757
public void IterationSetup()
6858
{
6959
MysqlDatabaseHelper.CleanupWriteTableAsync(_connectionString).GetAwaiter().GetResult();
70-
Helpers.InvokeGarbageCollection();
60+
InvokeGarbageCollection();
7161
}
7262

7363
[BenchmarkCategory("Write")]

‎benchmark/BenchmarkRunner/Benchmarks/PostgresqlReadBenchmark.cs‎

Lines changed: 6 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ namespace BenchmarkRunner.Benchmarks;
1212
[MarkdownExporterAttribute.GitHub]
1313
[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)]
1414
[CategoriesColumn]
15-
public class PostgresqlReadBenchmark : ReadBenchmark
15+
public sealed class PostgresqlReadBenchmark : BaseReadBenchmark
1616
{
1717
private static readonly string _connectionString = Config.GetPostgresConnectionString();
1818
private readonly QuerySql _sqlcImpl = new(_connectionString);
@@ -23,12 +23,8 @@ public class PostgresqlReadBenchmark : ReadBenchmark
2323
[GlobalSetup]
2424
public async Task GlobalSetup()
2525
{
26-
if (_isInitialized) return;
27-
await _initLock.WaitAsync();
28-
try
26+
await InitializeOnceAsync(async () =>
2927
{
30-
if (_isInitialized) return;
31-
3228
await PostgresqlDatabaseHelper.CleanupDatabaseAsync(_connectionString);
3329
var seeder = new PostgresqlDatabaseSeeder(_connectionString);
3430
await seeder.SeedAsync(
@@ -38,24 +34,14 @@ await seeder.SeedAsync(
3834
itemsPerOrder: 20
3935
// 20 * 1000 = 20,000 possible rows returned
4036
);
41-
42-
_isInitialized = true;
43-
}
44-
finally
45-
{
46-
_initLock.Release();
47-
}
37+
});
4838
}
4939

50-
[IterationSetup]
51-
public static void IterationSetup() => Helpers.InvokeGarbageCollection();
52-
53-
5440
[BenchmarkCategory("Read")]
5541
[Benchmark(Baseline = true, Description = "SQLC - GetCustomerOrders")]
5642
public override async Task Sqlc_GetCustomerOrders()
5743
{
58-
await Helpers.ExecuteConcurrentlyAsync(QueriesToRun, ConcurrentQueries, _ =>
44+
await ExecuteConcurrentlyAsync(QueriesToRun, ConcurrentQueries, _ =>
5945
{
6046
return _sqlcImpl.GetCustomerOrdersAsync(new QuerySql.GetCustomerOrdersArgs(
6147
CustomerId: Random.Shared.Next(1, CustomerCount),
@@ -69,7 +55,7 @@ await Helpers.ExecuteConcurrentlyAsync(QueriesToRun, ConcurrentQueries, _ =>
6955
[Benchmark(Description = "EFCore (NoTracking) - GetCustomerOrders")]
7056
public override async Task EFCore_NoTracking_GetCustomerOrders()
7157
{
72-
await Helpers.ExecuteConcurrentlyAsync(QueriesToRun, ConcurrentQueries, async _ =>
58+
await ExecuteConcurrentlyAsync(QueriesToRun, ConcurrentQueries, async _ =>
7359
{
7460
await using var dbContext = new SalesDbContext(_connectionString);
7561
var queries = new Queries(dbContext, useTracking: false);
@@ -85,7 +71,7 @@ await Helpers.ExecuteConcurrentlyAsync(QueriesToRun, ConcurrentQueries, async _
8571
[Benchmark(Description = "EFCore (WithTracking) - GetCustomerOrders")]
8672
public override async Task EFCore_WithTracking_GetCustomerOrders()
8773
{
88-
await Helpers.ExecuteConcurrentlyAsync(QueriesToRun, ConcurrentQueries, async _ =>
74+
await ExecuteConcurrentlyAsync(QueriesToRun, ConcurrentQueries, async _ =>
8975
{
9076
await using var dbContext = new SalesDbContext(_connectionString);
9177
var queries = new Queries(dbContext, useTracking: true);

‎benchmark/BenchmarkRunner/Benchmarks/PostgresqlWriteBenchmark.cs‎

Lines changed: 4 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ namespace BenchmarkRunner.Benchmarks;
1212
[MarkdownExporterAttribute.GitHub]
1313
[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)]
1414
[CategoriesColumn]
15-
public class PostgresqlWriteBenchmark : WriteBenchmark
15+
public sealed class PostgresqlWriteBenchmark : BaseWriteBenchmark
1616
{
1717
private static readonly string _connectionString = Config.GetPostgresConnectionString();
1818
private readonly QuerySql _sqlcImpl = new(_connectionString);
@@ -28,12 +28,8 @@ public class PostgresqlWriteBenchmark : WriteBenchmark
2828
[GlobalSetup]
2929
public async Task GlobalSetup()
3030
{
31-
if (_isInitialized) return;
32-
await _initLock.WaitAsync();
33-
try
31+
await InitializeOnceAsync(async () =>
3432
{
35-
if (_isInitialized) return;
36-
3733
PostgresqlDatabaseHelper.CleanupDatabaseAsync(_connectionString).GetAwaiter().GetResult();
3834
var seeder = new PostgresqlDatabaseSeeder(_connectionString);
3935
await seeder.SeedAsync(
@@ -52,20 +48,14 @@ await seeder.SeedAsync(
5248
Quantity: Random.Shared.Next(1, 10),
5349
UnitPrice: (decimal)(Random.Shared.NextDouble() * 100 + 5)
5450
))];
55-
56-
_isInitialized = true;
57-
}
58-
finally
59-
{
60-
_initLock.Release();
61-
}
51+
});
6252
}
6353

6454
[IterationSetup]
6555
public void IterationSetup()
6656
{
6757
PostgresqlDatabaseHelper.CleanupWriteTableAsync(_connectionString).GetAwaiter().GetResult();
68-
Helpers.InvokeGarbageCollection();
58+
InvokeGarbageCollection();
6959
}
7060

7161
[BenchmarkCategory("Write")]

‎benchmark/BenchmarkRunner/Benchmarks/ReadBenchmark.cs‎

Lines changed: 0 additions & 20 deletions
This file was deleted.

0 commit comments

Comments
 (0)