Skip to content

Commit d949dba

Browse files
ericavellaErica VellanowethCopilot
authored
Fix disk detection after LVM striping and update database workload profiles (#674)
* first commit * Fix disk detection after LVM striping and update database workload profiles - Add /proc/mounts fallback for raid0 path discovery in MySqlServerConfiguration and PostgreSQLServerConfiguration. After StripeDisks creates an LVM volume, lshw no longer reports the logical volume, causing 'Expected disks not found'. The fallback reads /proc/mounts to find the raid0 mount point. - Search all disks (not just filtered) for raid0 access paths in both MySQL and PostgreSQL server configuration components. - Remove unsupported CreateTables and DistributeDatabase dependency steps from PostgreSQL Sysbench TPCC and OLTP profiles. - Tune HammerDB TPCC WarehouseCount to LogicalCoreCount * 10 and TPCH ScaleFactor to 1 for faster validation runs. - Fix HammerDBExecutor async method missing await warning. - Add copilot-instructions.md for repository development guidance. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Add Logical, AccessPath disk filters and simplify DB config disk resolution Add two new DiskFilter types to DiskFilters: - Logical: matches LVM device mapper disks (/dev/dm-*, /dev/mapper/*) - AccessPath: matches disks by volume access path substring Add /dev/dm and /dev/mapper to valid Linux device path prefixes so LVM logical volumes are no longer silently dropped by the prefix filter. Simplify MySqlServerConfiguration and PostgreSQLServerConfiguration: - Default DiskFilter changed to 'Logical' (was 'osdisk:false&sizegreaterthan:256g') - Remove /proc/mounts parsing and multi-tier raid0 search logic - Clean fallback: Logical filter -> OsDisk:false&BiggestSize physical disk Update MySQL profiles to use DiskFilter: Logical on MySQLServerConfiguration steps. PostgreSQL profiles use the new default. StripeDisks steps unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Add /proc/mounts LVM fallback for disk path resolution lshw does not report LVM logical volumes (/dev/dm-*, /dev/mapper/*), so the Logical disk filter returns empty after StripeDisks creates an LVM volume. Added FindLvmMountPathAsync to both MySqlServerConfiguration and PostgreSQLServerConfiguration that reads /proc/mounts to find any mount backed by an LVM device, returning the mount path (e.g. mnt_raid0). Fallback chain is now: 1. Logical filter (works if lshw ever reports LVM devices) 2. /proc/mounts LVM device check (handles real-world lshw limitation) 3. Physical disk BiggestSize fallback (unmounted single disk scenario) Verified on Azure VMs: - E8ds_v6 (1 disk, single-disk LVM) -> mnt_raid0 found via /proc/mounts - E16ds_v6 (2 disks, striped LVM) -> mnt_raid0 found via /proc/mounts Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Remove .github/copilot-instructions.md from PR Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Signed-off-by: erica vellanoweth <37354999+ericavella@users.noreply.github.com> Co-authored-by: Erica Vellanoweth <evellanoweth@microsoft.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent b327939 commit d949dba

20 files changed

Lines changed: 509 additions & 606 deletions

File tree

‎src/VirtualClient/VirtualClient.Actions.UnitTests/Sysbench/SysbenchConfigurationTests.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -332,7 +332,7 @@ public async Task SysbenchConfigurationProperlyExecutesTPCCConfigurablePreparati
332332

333333
string[] expectedCommands =
334334
{
335-
$"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem MySQL --benchmark TPCC --threadCount 16 --tableCount 40 --warehouses 999 --password [A-Za-z0-9+/=]+ --hostIpAddress \"1.2.3.5\""
335+
$"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem MySQL --benchmark TPCC --threadCount 16 --tableCount 40 --warehouses 1000 --password [A-Za-z0-9+/=]+ --hostIpAddress \"1.2.3.5\""
336336
};
337337

338338
int commandNumber = 0;
@@ -446,7 +446,7 @@ public async Task SysbenchConfigurationProperlyExecutesPostgreSQLTPCCConfigurabl
446446

447447
string[] expectedCommands =
448448
{
449-
$"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem PostgreSQL --benchmark TPCC --threadCount 16 --tableCount 40 --warehouses 999 --password [A-Za-z0-9+/=]+ --hostIpAddress \"1.2.3.5\""
449+
$"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem PostgreSQL --benchmark TPCC --threadCount 16 --tableCount 40 --warehouses 1000 --password [A-Za-z0-9+/=]+ --hostIpAddress \"1.2.3.5\""
450450
};
451451

452452
int commandNumber = 0;

‎src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBClientExecutor.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ namespace VirtualClient.Actions
2121
/// </summary>
2222
public class HammerDBClientExecutor : HammerDBExecutor
2323
{
24+
private static string runTransactionsTclName = "runTransactions.tcl";
25+
2426
/// <summary>
2527
/// Initializes a new instance of the <see cref="HammerDBClientExecutor"/> class.
2628
/// </summary>
@@ -154,7 +156,7 @@ private Task ExecuteWorkloadAsync(EventContext telemetryContext, CancellationTok
154156

155157
using (IProcessProxy process = await this.ExecuteCommandAsync(
156158
command,
157-
$"{script} --runTransactionsTCLFilePath {this.RunTransactionsTclName}",
159+
$"{script} --runTransactionsTCLFilePath {runTransactionsTclName}",
158160
this.HammerDBPackagePath,
159161
telemetryContext,
160162
cancellationToken))

‎src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBExecutor.cs‎

Lines changed: 4 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ namespace VirtualClient.Actions
2525
[SupportedPlatforms("linux-x64")]
2626
public class HammerDBExecutor : VirtualClientComponent
2727
{
28+
private static string createDBTclName = "createDB.tcl";
2829
private readonly IStateManager stateManager;
2930
private static readonly List<int> Factors = new List<int> { 1, 10, 30, 100, 300, 1000, 3000, 10000, 30000, 100000 };
3031

@@ -57,28 +58,6 @@ public string Action
5758
}
5859
}
5960

60-
/// <summary>
61-
/// Defines the name of the createDB TCL file.
62-
/// </summary>
63-
public string CreateDBTclName
64-
{
65-
get
66-
{
67-
return "createDB.tcl";
68-
}
69-
}
70-
71-
/// <summary>
72-
/// Defines the name of the runTransactions TCL file.
73-
/// </summary>
74-
public string RunTransactionsTclName
75-
{
76-
get
77-
{
78-
return "runTransactions.tcl";
79-
}
80-
}
81-
8261
/// <summary>
8362
/// Defines the name of the PostgreSQL database to create/use for the transactions.
8463
/// </summary>
@@ -137,18 +116,6 @@ public string WarehouseCount
137116
}
138117
}
139118

140-
/// <summary>
141-
/// Disk filter specified
142-
/// </summary>
143-
public string DiskFilter
144-
{
145-
get
146-
{
147-
// and 256G
148-
return this.Parameters.GetValue<string>(nameof(this.DiskFilter), "osdisk:false&sizegreaterthan:256g");
149-
}
150-
}
151-
152119
/// <summary>
153120
/// Workload duration.
154121
/// </summary>
@@ -341,7 +308,7 @@ protected async Task InitializeExecutablesAsync(EventContext telemetryContext, C
341308
private async Task PrepareSQLDatabase(EventContext telemetryContext, CancellationToken cancellationToken)
342309
{
343310
string command = "python3";
344-
string arguments = $"{this.PlatformSpecifics.Combine(this.HammerDBPackagePath, "populate-database.py")} --createDBTCLPath {this.CreateDBTclName}";
311+
string arguments = $"{this.PlatformSpecifics.Combine(this.HammerDBPackagePath, "populate-database.py")} --createDBTCLPath {createDBTclName}";
345312

346313
using (IProcessProxy process = await this.ExecuteCommandAsync(
347314
command,
@@ -378,17 +345,11 @@ private async Task ConfigureCreateHammerDBFile(EventContext telemetryContext, Ca
378345
}
379346
}
380347

381-
private async Task GenerateCommandLineArguments(CancellationToken cancellationToken)
348+
private Task GenerateCommandLineArguments(CancellationToken cancellationToken)
382349
{
383350
string arguments = $"{this.PlatformSpecifics.Combine(this.HammerDBPackagePath, "configure-workload-generator.py")} --workload {this.Workload} --sqlServer {this.SQLServer} --port {this.Port}" +
384351
$" --virtualUsers {this.VirtualUsers} --password {this.SuperUserPassword} --dbName {this.DatabaseName} --hostIPAddress {this.ServerIpAddress}";
385352

386-
if (this.IsMultiRoleLayout() && this.GetLayoutClientInstance().Role == ClientRole.Server)
387-
{
388-
string directories = await this.GetDataDirectoriesAsync(cancellationToken);
389-
arguments = $"{arguments} --directories {directories}";
390-
}
391-
392353
if (this.Workload.Equals("tpcc", StringComparison.OrdinalIgnoreCase))
393354
{
394355
arguments = $"{arguments} --warehouseCount {this.WarehouseCount} --duration {this.Duration.TotalMinutes}";
@@ -407,50 +368,8 @@ private async Task GenerateCommandLineArguments(CancellationToken cancellationTo
407368
}
408369

409370
this.HammerDBScenarioArguments = arguments;
410-
}
411-
412-
private async Task<string> GetDataDirectoriesAsync(CancellationToken cancellationToken)
413-
{
414-
string diskPaths = string.Empty;
415-
416-
if (!cancellationToken.IsCancellationRequested)
417-
{
418-
IEnumerable<Disk> disks = await this.SystemManager.DiskManager.GetDisksAsync(cancellationToken)
419-
.ConfigureAwait(false);
420-
421-
if (disks?.Any() != true)
422-
{
423-
throw new WorkloadException(
424-
"Unexpected scenario. The disks defined for the system could not be properly enumerated.",
425-
ErrorReason.WorkloadUnexpectedAnomaly);
426-
}
427-
428-
IEnumerable<Disk> disksToTest = DiskFilters.FilterDisks(disks, this.DiskFilter, this.Platform).ToList();
429-
430-
if (disksToTest?.Any() != true)
431-
{
432-
throw new WorkloadException(
433-
"Expected disks to test not found. Given the parameters defined for the profile action/step or those passed " +
434-
"in on the command line, the requisite disks do not exist on the system or could not be identified based on the properties " +
435-
"of the existing disks.",
436-
ErrorReason.DependencyNotFound);
437-
}
438-
439-
foreach (Disk disk in disksToTest)
440-
{
441-
string path = this.Combine(disk.GetPreferredAccessPath(this.Platform), $"{this.SQLServer.ToLower()}");
442-
443-
// Create the directory if it doesn't exist
444-
if (!this.SystemManager.FileSystem.Directory.Exists(path))
445-
{
446-
this.SystemManager.FileSystem.Directory.CreateDirectory(path);
447-
}
448-
449-
diskPaths += $"{path}:";
450-
}
451-
}
452371

453-
return diskPaths;
372+
return Task.CompletedTask;
454373
}
455374

456375
private static Task OpenFirewallPortsAsync(int port, IFirewallManager firewallManager, CancellationToken cancellationToken)

‎src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,7 @@ private Task ExecuteWorkloadAsync(EventContext telemetryContext, CancellationTok
179179
{
180180
using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken))
181181
{
182-
this.sysbenchLoggingArguments = this.BuildSysbenchLoggingArguments(SysbenchMode.Run);
182+
this.sysbenchLoggingArguments = this.BuildSysbenchLoggingArguments();
183183
this.sysbenchExecutionArguments = $"{this.sysbenchLoggingArguments} --workload {this.Workload} --hostIpAddress {this.ServerIpAddress} --durationSecs {this.Duration.TotalSeconds} --password {this.SuperUserPassword}";
184184

185185
string script = $"{this.SysbenchPackagePath}/run-workload.py ";

‎src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs‎

Lines changed: 2 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -59,15 +59,12 @@ protected override async Task ExecuteAsync(EventContext telemetryContext, Cancel
5959
case ConfigurationAction.Cleanup:
6060
await this.CleanUpDatabase(telemetryContext, cancellationToken);
6161
break;
62-
case ConfigurationAction.CreateTables:
63-
await this.PrepareDatabase(telemetryContext, cancellationToken);
64-
break;
6562
case ConfigurationAction.PopulateTables:
6663
await this.PopulateDatabase(telemetryContext, cancellationToken);
6764
break;
6865
default:
6966
throw new DependencyException(
70-
$"The specified Sysbench action '{this.Action}' is not supported. Supported actions include: \"{ConfigurationAction.PopulateTables}, {ConfigurationAction.Cleanup}, {ConfigurationAction.CreateTables}\".",
67+
$"The specified Sysbench action '{this.Action}' is not supported. Supported actions include: \"{ConfigurationAction.PopulateTables}, {ConfigurationAction.Cleanup}\".",
7168
ErrorReason.NotSupported);
7269
}
7370
}
@@ -106,41 +103,6 @@ private async Task CleanUpDatabase(EventContext telemetryContext, CancellationTo
106103
await this.stateManager.SaveStateAsync<SysbenchState>(nameof(SysbenchState), state, cancellationToken);
107104
}
108105

109-
private async Task PrepareDatabase(EventContext telemetryContext, CancellationToken cancellationToken)
110-
{
111-
SysbenchState state = await this.stateManager.GetStateAsync<SysbenchState>(nameof(SysbenchState), cancellationToken)
112-
?? new SysbenchState();
113-
114-
if (!state.DatabasePopulated)
115-
{
116-
string serverIp = (this.IsMultiRoleLayout() && this.IsInRole(ClientRole.Client)) ? this.ServerIpAddress : "localhost";
117-
string sysbenchPrepareArguments = $"{this.BuildSysbenchLoggingArguments(SysbenchMode.Prepare)} --password {this.SuperUserPassword} --hostIpAddress \"{serverIp}\"";
118-
119-
string command = $"{this.SysbenchPackagePath}/populate-database.py";
120-
121-
using (IProcessProxy process = await this.ExecuteCommandAsync(
122-
SysbenchExecutor.PythonCommand,
123-
$"{command} {sysbenchPrepareArguments}",
124-
this.SysbenchPackagePath,
125-
telemetryContext,
126-
cancellationToken))
127-
{
128-
if (!cancellationToken.IsCancellationRequested)
129-
{
130-
await this.LogProcessDetailsAsync(process, telemetryContext, "Sysbench", logToFile: true);
131-
process.ThrowIfErrored<WorkloadException>(process.StandardError.ToString(), ErrorReason.WorkloadUnexpectedAnomaly);
132-
}
133-
}
134-
}
135-
else
136-
{
137-
throw new DependencyException(
138-
$"Database preparation failed. A database has already been populated on the system. Please drop the tables, or run \"{ConfigurationAction.Cleanup}\" Action" +
139-
$"before attempting to create new tables on this database.",
140-
ErrorReason.NotSupported);
141-
}
142-
}
143-
144106
private async Task PopulateDatabase(EventContext telemetryContext, CancellationToken cancellationToken)
145107
{
146108
SysbenchState state = await this.stateManager.GetStateAsync<SysbenchState>(nameof(SysbenchState), cancellationToken)
@@ -152,7 +114,7 @@ await this.Logger.LogMessageAsync($"{this.TypeName}.PopulateDatabase", telemetry
152114
{
153115
string serverIp = (this.IsMultiRoleLayout() && this.IsInRole(ClientRole.Client)) ? this.ServerIpAddress : "localhost";
154116

155-
string sysbenchLoggingArguments = this.BuildSysbenchLoggingArguments(SysbenchMode.Populate);
117+
string sysbenchLoggingArguments = this.BuildSysbenchLoggingArguments();
156118
this.sysbenchPopulationArguments = $"{sysbenchLoggingArguments} --password {this.SuperUserPassword} --hostIpAddress \"{serverIp}\"";
157119

158120
string script = $"{this.SysbenchPackagePath}/populate-database.py";
@@ -227,11 +189,6 @@ private void AddPopulationDurationMetric(string arguments, IProcessProxy process
227189
/// </summary>
228190
internal class ConfigurationAction
229191
{
230-
/// <summary>
231-
/// Initializes the tables on the database.
232-
/// </summary>
233-
public const string CreateTables = nameof(CreateTables);
234-
235192
/// <summary>
236193
/// Creates Database on MySQL server and Users on Server and any Clients.
237194
/// </summary>

‎src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs‎

Lines changed: 3 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -60,27 +60,6 @@ public SysbenchExecutor(IServiceCollection dependencies, IDictionary<string, ICo
6060
};
6161
}
6262

63-
/// <summary>
64-
/// Defines the mode in which Sysbench is operating.
65-
/// </summary>
66-
protected internal enum SysbenchMode
67-
{
68-
/// <summary>
69-
/// Creates the database schema with minimal data.
70-
/// </summary>
71-
Prepare,
72-
73-
/// <summary>
74-
/// Populates the database with the full dataset.
75-
/// </summary>
76-
Populate,
77-
78-
/// <summary>
79-
/// Runs the benchmark workload.
80-
/// </summary>
81-
Run
82-
}
83-
8463
/// <summary>
8564
/// The benchmark (e.g. OLTP, TPCC).
8665
/// </summary>
@@ -409,7 +388,7 @@ protected async Task InitializeExecutablesAsync(EventContext telemetryContext, C
409388
/// dbName, databaseSystem, benchmark and tableCount.
410389
/// </summary>
411390
/// <returns></returns>
412-
protected string BuildSysbenchLoggingArguments(SysbenchMode mode)
391+
protected string BuildSysbenchLoggingArguments()
413392
{
414393
int tableCount = GetTableCount(this.DatabaseScenario, this.TableCount, this.Workload);
415394
int threadCount = GetThreadCount(this.SystemManager, this.DatabaseScenario, this.Threads);
@@ -419,19 +398,11 @@ protected string BuildSysbenchLoggingArguments(SysbenchMode mode)
419398
switch (this.Benchmark)
420399
{
421400
case BenchmarkName.OLTP:
422-
int recordCount = mode == SysbenchMode.Prepare ? 1 : GetRecordCount(this.SystemManager, this.DatabaseScenario, this.RecordCount);
401+
int recordCount = GetRecordCount(this.SystemManager, this.DatabaseScenario, this.RecordCount);
423402
loggingArguments = $"{loggingArguments} --recordCount {recordCount}";
424403
break;
425404
case BenchmarkName.TPCC:
426-
int warehouseEstimate = GetWarehouseCount(this.SystemManager, this.DatabaseScenario, this.WarehouseCount);
427-
int warehouseCount = mode switch
428-
{
429-
SysbenchMode.Prepare => 1,
430-
SysbenchMode.Populate => Math.Max(1, warehouseEstimate - 1),
431-
SysbenchMode.Run => warehouseEstimate,
432-
_ => warehouseEstimate
433-
};
434-
405+
int warehouseCount = GetWarehouseCount(this.SystemManager, this.DatabaseScenario, this.WarehouseCount);
435406
loggingArguments = $"{loggingArguments} --warehouses {warehouseCount}";
436407
break;
437408
default:

0 commit comments

Comments
 (0)