Skip to content

Commit 00f4f5f

Browse files
RakeshwarKRakeshwar Reddy Kambaiahgari
andauthored
Enable defining specific logical processors against which to run a given process. (#617)
* Init Commit * Update tests * Update tests * commit changes --------- Co-authored-by: Rakeshwar Reddy Kambaiahgari <rkambaiahgar@microsoft.com>
1 parent cc392ea commit 00f4f5f

6 files changed

Lines changed: 728 additions & 20 deletions

File tree

src/VirtualClient/VirtualClient.Actions/Redis/RedisServerExecutor.cs

Lines changed: 41 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ namespace VirtualClient.Actions
1818
using VirtualClient.Common;
1919
using VirtualClient.Common.Contracts;
2020
using VirtualClient.Common.Extensions;
21+
using VirtualClient.Common.ProcessAffinity;
2122
using VirtualClient.Common.Telemetry;
2223
using VirtualClient.Contracts;
2324
using VirtualClient.Contracts.Metadata;
@@ -385,12 +386,10 @@ private void StartServerInstances(EventContext telemetryContext, CancellationTok
385386
{
386387
try
387388
{
388-
string command = "bash";
389+
string command = "bash -c";
389390
string workingDirectory = this.RedisPackagePath;
390-
List<string> commands = new List<string>();
391391

392392
relatedContext.AddContext("command", command);
393-
relatedContext.AddContext("commandArguments", commands);
394393
relatedContext.AddContext("workingDir", workingDirectory);
395394

396395
for (int i = 0; i < this.ServerInstances; i++)
@@ -399,33 +398,27 @@ private void StartServerInstances(EventContext telemetryContext, CancellationTok
399398
// will warm them up and then exit. We keep a reference to the server processes/tasks
400399
// so that they remain running until the class is disposed.
401400
int port = this.Port + i;
402-
string commandArguments = null;
403-
404-
if (this.BindToCores)
405-
{
406-
commandArguments = $"-c \"numactl -C {i} {this.RedisExecutablePath}";
407-
}
408-
else
409-
{
410-
commandArguments = $"-c \"{this.RedisExecutablePath}";
411-
}
401+
string redisCommand = this.RedisExecutablePath;
412402

413403
if (this.IsTLSEnabled)
414404
{
415-
commandArguments += $" --tls-port {port} --port 0 --tls-cert-file {this.PlatformSpecifics.Combine(this.RedisResourcesPath, "redis.crt")} --tls-key-file {this.PlatformSpecifics.Combine(this.RedisResourcesPath, "redis.key")} --tls-ca-cert-file {this.PlatformSpecifics.Combine(this.RedisResourcesPath, "ca.crt")}";
405+
redisCommand += $" --tls-port {port} --port 0 --tls-cert-file {this.PlatformSpecifics.Combine(this.RedisResourcesPath, "redis.crt")} --tls-key-file {this.PlatformSpecifics.Combine(this.RedisResourcesPath, "redis.key")} --tls-ca-cert-file {this.PlatformSpecifics.Combine(this.RedisResourcesPath, "ca.crt")}";
416406
}
417407
else
418408
{
419-
commandArguments += $" --port {port}";
409+
redisCommand += $" --port {port}";
420410
}
421411

422-
commandArguments += $" {this.CommandLine}\"";
412+
redisCommand += $" {this.CommandLine}";
413+
414+
// When binding to cores, CreateElevatedProcessWithAffinity wraps the command with numactl.
415+
// When not binding to cores, we need to wrap the redis command in quotes for bash -c.
416+
string commandArguments = this.BindToCores ? redisCommand : $"\"{redisCommand}\"";
423417

424418
// We cannot use a Task.Run here. The Task is queued on the threadpool but does not get running
425419
// until our counter 'i' is at the end. This will cause all server instances to use the same port
426420
// and to try to bind to the same core.
427-
commands.Add(commandArguments);
428-
this.serverProcesses.Add(this.StartServerInstanceAsync(port, command, commandArguments, workingDirectory, relatedContext, cancellationToken));
421+
this.serverProcesses.Add(this.StartServerInstanceAsync(port, i, command, commandArguments, workingDirectory, relatedContext, cancellationToken));
429422
}
430423
}
431424
catch (OperationCanceledException)
@@ -435,14 +428,42 @@ private void StartServerInstances(EventContext telemetryContext, CancellationTok
435428
});
436429
}
437430

438-
private Task StartServerInstanceAsync(int port, string command, string commandArguments, string workingDirectory, EventContext telemetryContext, CancellationToken cancellationToken)
431+
private Task StartServerInstanceAsync(int port, int coreIndex, string command, string commandArguments, string workingDirectory, EventContext telemetryContext, CancellationToken cancellationToken)
439432
{
440433
return (this.ServerRetryPolicy ?? Policy.NoOpAsync()).ExecuteAsync(async () =>
441434
{
442435
try
443436
{
444-
using (IProcessProxy process = await this.ExecuteCommandAsync(command, commandArguments, workingDirectory, telemetryContext, cancellationToken, runElevated: true))
437+
IProcessProxy process = null;
438+
// LINUX with affinity: Wrap command with numactl
439+
if (this.BindToCores && this.Platform == PlatformID.Unix)
440+
{
441+
ProcessAffinityConfiguration affinityConfig = ProcessAffinityConfiguration.Create(this.Platform, new[] { coreIndex });
442+
process = this.SystemManagement.ProcessManager.CreateElevatedProcessWithAffinity(
443+
this.Platform,
444+
command,
445+
commandArguments,
446+
workingDirectory,
447+
affinityConfig);
448+
}
449+
else
450+
{
451+
// No CPU affinity binding - standard elevated process
452+
process = this.SystemManagement.ProcessManager.CreateElevatedProcess(
453+
this.Platform,
454+
command,
455+
commandArguments,
456+
workingDirectory);
457+
}
458+
459+
using (process)
445460
{
461+
// Start the process
462+
process.Start();
463+
464+
// Wait for process to exit
465+
await process.WaitForExitAsync(cancellationToken);
466+
446467
if (!cancellationToken.IsCancellationRequested)
447468
{
448469
ConsoleLogger.Default.LogMessage($"Redis server process exited (port = {port})...", telemetryContext);
Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
namespace VirtualClient.Common.ProcessAffinity
5+
{
6+
using System;
7+
using System.Linq;
8+
using System.Text.RegularExpressions;
9+
using NUnit.Framework;
10+
11+
[TestFixture]
12+
[Category("Unit")]
13+
public class LinuxProcessAffinityConfigurationTests
14+
{
15+
[Test]
16+
public void LinuxProcessAffinityConfigurationGeneratesCorrectNumactlSpecForSingleCore()
17+
{
18+
LinuxProcessAffinityConfiguration config = new LinuxProcessAffinityConfiguration(new[] { 0 });
19+
20+
// Verify through GetCommandWithAffinity which uses GetNumactlCoreSpec internally
21+
string command = config.GetCommandWithAffinity("test", null);
22+
23+
Assert.IsTrue(command.Contains("-C 0"));
24+
}
25+
26+
[Test]
27+
public void LinuxProcessAffinityConfigurationGeneratesCorrectNumactlSpecForContiguousCores()
28+
{
29+
LinuxProcessAffinityConfiguration config = new LinuxProcessAffinityConfiguration(new[] { 0, 1, 2, 3 });
30+
31+
string command = config.GetCommandWithAffinity("test", null);
32+
33+
// Should be optimized to range notation
34+
Assert.IsTrue(command.Contains("-C 0-3"));
35+
}
36+
37+
[Test]
38+
public void LinuxProcessAffinityConfigurationGeneratesCorrectNumactlSpecForNonContiguousCores()
39+
{
40+
LinuxProcessAffinityConfiguration config = new LinuxProcessAffinityConfiguration(new[] { 0, 2, 4 });
41+
42+
string command = config.GetCommandWithAffinity("test", null);
43+
44+
Assert.IsTrue(command.Contains("-C 0,2,4"));
45+
}
46+
47+
[Test]
48+
public void LinuxProcessAffinityConfigurationGeneratesCorrectNumactlSpecForMixedCores()
49+
{
50+
LinuxProcessAffinityConfiguration config = new LinuxProcessAffinityConfiguration(new[] { 0, 1, 2, 5, 7, 8, 9 });
51+
52+
string command = config.GetCommandWithAffinity("test", null);
53+
54+
// Should optimize ranges: 0-2,5,7-9
55+
Assert.IsTrue(command.Contains("-C 0-2,5,7-9"));
56+
}
57+
58+
[Test]
59+
public void LinuxProcessAffinityConfigurationGeneratesCorrectNumactlSpecForComplexPattern()
60+
{
61+
LinuxProcessAffinityConfiguration config = new LinuxProcessAffinityConfiguration(
62+
new[] { 0, 1, 2, 5, 6, 10, 12, 13, 14, 15 });
63+
64+
string command = config.GetCommandWithAffinity("test", null);
65+
66+
// 0-2 (3 cores), 5,6 (2 cores), 10 (single), 12-15 (4 cores)
67+
Assert.IsTrue(command.Contains("-C 0-2,5,6,10,12-15"));
68+
}
69+
70+
[Test]
71+
public void LinuxProcessAffinityConfigurationGeneratesCorrectNumactlSpecForHighCoreIndices()
72+
{
73+
LinuxProcessAffinityConfiguration config = new LinuxProcessAffinityConfiguration(new[] { 100, 101, 102 });
74+
75+
string command = config.GetCommandWithAffinity("test", null);
76+
77+
Assert.IsTrue(command.Contains("-C 100-102"));
78+
}
79+
80+
[Test]
81+
public void LinuxProcessAffinityConfigurationGeneratesCorrectNumactlCommandForSingleCore()
82+
{
83+
LinuxProcessAffinityConfiguration config = new LinuxProcessAffinityConfiguration(new[] { 0 });
84+
85+
string command = config.GetCommandWithAffinity(null, "myworkload --arg1 --arg2");
86+
87+
Assert.AreEqual("\"numactl -C 0 myworkload --arg1 --arg2\"", command);
88+
}
89+
90+
[Test]
91+
public void LinuxProcessAffinityConfigurationGeneratesCorrectNumactlCommandForMultipleCores()
92+
{
93+
LinuxProcessAffinityConfiguration config = new LinuxProcessAffinityConfiguration(new[] { 0, 1, 2 });
94+
95+
string command = config.GetCommandWithAffinity(null, "myworkload --arg1 --arg2");
96+
97+
Assert.AreEqual("\"numactl -C 0-2 myworkload --arg1 --arg2\"", command);
98+
}
99+
100+
[Test]
101+
public void LinuxProcessAffinityConfigurationGeneratesCorrectNumactlCommandWithEmptyArguments()
102+
{
103+
LinuxProcessAffinityConfiguration config = new LinuxProcessAffinityConfiguration(new[] { 1, 3, 5 });
104+
105+
string command = config.GetCommandWithAffinity(null, "myworkload");
106+
107+
Assert.AreEqual("\"numactl -C 1,3,5 myworkload\"", command);
108+
}
109+
110+
[Test]
111+
public void LinuxProcessAffinityConfigurationHandlesComplexArguments()
112+
{
113+
LinuxProcessAffinityConfiguration config = new LinuxProcessAffinityConfiguration(new[] { 0, 1 });
114+
115+
string command = config.GetCommandWithAffinity(
116+
null,
117+
"myworkload --file=\"path with spaces\" --option=value");
118+
119+
// 2 cores use comma notation (0,1), not range (0-1)
120+
Assert.AreEqual(
121+
"\"numactl -C 0,1 myworkload --file=\"path with spaces\" --option=value\"",
122+
command);
123+
}
124+
125+
[Test]
126+
public void LinuxProcessAffinityConfigurationToStringIncludesNumactlSpec()
127+
{
128+
LinuxProcessAffinityConfiguration config = new LinuxProcessAffinityConfiguration(new[] { 0, 1, 2, 5 });
129+
130+
string result = config.ToString();
131+
132+
Assert.IsTrue(result.Contains("0,1,2,5"));
133+
Assert.IsTrue(result.Contains("numactl: -C 0-2,5"));
134+
}
135+
136+
[Test]
137+
public void LinuxProcessAffinityConfigurationOptimizesRanges()
138+
{
139+
// Test various range optimization scenarios by checking the command output
140+
// Note: 2 consecutive cores use comma notation (0,1), 3+ use range notation (0-2)
141+
var testCases = new[]
142+
{
143+
(new[] { 0 }, "-C 0"),
144+
(new[] { 0, 1 }, "-C 0,1"), // 2 cores: comma notation
145+
(new[] { 0, 1, 2 }, "-C 0-2"), // 3+ cores: range notation
146+
(new[] { 0, 2 }, "-C 0,2"),
147+
(new[] { 0, 1, 3 }, "-C 0,1,3"), // 2 cores then gap
148+
(new[] { 0, 1, 2, 4, 5, 6 }, "-C 0-2,4-6"), // Two 3-core ranges
149+
(new[] { 0, 2, 4, 6, 8 }, "-C 0,2,4,6,8"),
150+
(new[] { 0, 1, 2, 3, 5, 6, 7, 8, 10 }, "-C 0-3,5-8,10") // 4-core range, 4-core range, single
151+
};
152+
153+
foreach (var (cores, expectedSpec) in testCases)
154+
{
155+
LinuxProcessAffinityConfiguration config = new LinuxProcessAffinityConfiguration(cores);
156+
string command = config.GetCommandWithAffinity("test", null);
157+
Assert.IsTrue(command.Contains(expectedSpec), $"Failed for cores: {string.Join(",", cores)}. Expected '{expectedSpec}' in '{command}'");
158+
}
159+
}
160+
161+
[Test]
162+
public void LinuxProcessAffinityConfigurationHandlesUnsortedCores()
163+
{
164+
// Cores should be sorted before optimization
165+
LinuxProcessAffinityConfiguration config = new LinuxProcessAffinityConfiguration(new[] { 5, 0, 2, 1, 3 });
166+
167+
string command = config.GetCommandWithAffinity("test", null);
168+
169+
// Should sort and optimize: 0-3,5
170+
Assert.IsTrue(command.Contains("-C 0-3,5"));
171+
}
172+
}
173+
}

0 commit comments

Comments
 (0)