Skip to content

Commit 8b050c0

Browse files
nmalkapuramnmalkapuram
authored
DSB fix for ubuntu 24.04 (#593)
* DSB fix for ubuntu 24.04 * fixing unit tests * DSB support for ubuntu 24.04 * build fix * functional tests fix
1 parent 092c56d commit 8b050c0

7 files changed

Lines changed: 103 additions & 55 deletions

File tree

‎src/VirtualClient/VirtualClient.Actions.FunctionalTests/DeathStarBench/DeathStarBenchClientProfileTests.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ public void SetupFixture()
3838
DeathStarBenchExecutor.StateConfirmationPollingTimeout = TimeSpan.FromSeconds(1);
3939

4040
this.mockFixture.SetupPackage("deathstarbench");
41+
this.mockFixture.SetupPackage("libssl");
4142
this.mockFixture.SetupFile(@"/usr/local/bin/docker-compose");
4243
}
4344

‎src/VirtualClient/VirtualClient.Actions.FunctionalTests/DeathStarBench/DeathStarBenchServerProfileTests.cs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,18 +43,19 @@ public async Task DeathStarBenchWorkloadProfileInstallsTheExpectedDependenciesOf
4343
new ClientInstance(this.serverAgentId, "1.2.3.5", "Server"));
4444

4545
this.mockFixture.SystemManagement.SetupGet(sm => sm.AgentId).Returns(this.serverAgentId);
46+
this.mockFixture.SetupPackage("libssl");
4647
this.mockFixture.SetupFile(@"/usr/local/bin/docker-compose");
4748

4849
using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies))
49-
{
50+
{
5051
executor.ExecuteActions = false;
5152

5253
await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None)
5354
.ConfigureAwait(false);
54-
55-
// Workload dependency package expectations
56-
WorkloadAssert.WorkloadPackageInstalled(this.mockFixture, "deathstarbench");
5755
}
56+
57+
// Workload dependency package expectations
58+
WorkloadAssert.WorkloadPackageInstalled(this.mockFixture, "deathstarbench");
5859
}
5960

6061
[Test]
@@ -119,6 +120,8 @@ public void DeathStarBenchWorkloadProfileActionsWillNotBeExecutedIfTheClientWork
119120
new ClientInstance(this.clientAgentId, "1.2.3.4", "Client"),
120121
new ClientInstance(this.serverAgentId, "1.2.3.5", "Server"));
121122

123+
this.mockFixture.SetupPackage("libssl");
124+
122125
using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies))
123126
{
124127
DependencyException error = Assert.ThrowsAsync<DependencyException>(() => executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None));

‎src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchClientExecutorTests.cs‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -148,9 +148,11 @@ public async Task DeathStarBenchClientExecutorExecutesExpectedCommands_SocialNet
148148
// exactly the same as what is executed.
149149
$"sudo bash {this.mockPackage.Path}/linux-x64/scripts/dockerComposeScript.sh",
150150
$"sudo chmod +x \"/usr/local/bin/docker-compose\"",
151-
$"sudo python3 -m pip install -U pip",
152-
$"sudo python3 -m pip install -U setuptools",
153-
$"sudo -H python3 -m pip install aiohttp asyncio",
151+
$"sudo apt install python3-venv -y",
152+
$"sudo python3 -m venv {this.mockPackage.Path}/linux-x64/venv",
153+
$"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install -U pip",
154+
$"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install -U setuptools",
155+
$"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install aiohttp asyncio",
154156
$"sudo luarocks install luasocket",
155157
$"sudo bash {this.mockPackage.Path}/linux-x64/scripts/isSwarmNode.sh",
156158
$"sudo bash {this.mockPackage.Path}/linux-x64/scripts/isSwarmNode.sh",
@@ -198,9 +200,11 @@ public async Task DeathStarBenchClientExecutorExecutesExpectedCommands_MediaMicr
198200
$"sudo chmod +x \"{this.mockPackage.Path}/linux-x64/mediamicroservices/wrk2/deps/luajit/src/luajit\"",
199201
$"sudo bash {this.mockPackage.Path}/linux-x64/scripts/dockerComposeScript.sh",
200202
$"sudo chmod +x \"/usr/local/bin/docker-compose\"",
201-
$"sudo python3 -m pip install -U pip",
202-
$"sudo python3 -m pip install -U setuptools",
203-
$"sudo -H python3 -m pip install aiohttp asyncio",
203+
$"sudo apt install python3-venv -y",
204+
$"sudo python3 -m venv {this.mockPackage.Path}/linux-x64/venv",
205+
$"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install -U pip",
206+
$"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install -U setuptools",
207+
$"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install aiohttp asyncio",
204208
$"sudo luarocks install luasocket",
205209
$"sudo bash {this.mockPackage.Path}/linux-x64/scripts/isSwarmNode.sh",
206210
$"sudo bash {this.mockPackage.Path}/linux-x64/scripts/isSwarmNode.sh",
@@ -243,9 +247,11 @@ public async Task DeathStarBenchClientExecutorExecutesExpectedCommands_HotelRese
243247
// exactly the same as what is executed.
244248
$"sudo bash {this.mockPackage.Path}/linux-x64/scripts/dockerComposeScript.sh",
245249
$"sudo chmod +x \"/usr/local/bin/docker-compose\"",
246-
$"sudo python3 -m pip install -U pip",
247-
$"sudo python3 -m pip install -U setuptools",
248-
$"sudo -H python3 -m pip install aiohttp asyncio",
250+
$"sudo apt install python3-venv -y",
251+
$"sudo python3 -m venv {this.mockPackage.Path}/linux-x64/venv",
252+
$"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install -U pip",
253+
$"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install -U setuptools",
254+
$"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install aiohttp asyncio",
249255
$"sudo luarocks install luasocket",
250256
$"sudo bash {this.mockPackage.Path}/linux-x64/scripts/isSwarmNode.sh",
251257
$"sudo bash {this.mockPackage.Path}/linux-x64/scripts/isSwarmNode.sh",

‎src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchExecutorTests.cs‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -96,9 +96,11 @@ public async Task DeathStarBenchExecutorInstallsDepedenciesAsExpected()
9696
{
9797
@"bash /home/user/tools/VirtualClient/packages/deathstarbench/linux-x64/scripts/dockerComposeScript.sh",
9898
@"chmod +x ""/usr/local/bin/docker-compose""",
99-
@"python3 -m pip install -U pip",
100-
@"python3 -m pip install -U setuptools",
101-
@"-H python3 -m pip install aiohttp asyncio",
99+
@"apt install python3-venv -y",
100+
@"python3 -m venv /home/user/tools/VirtualClient/packages/deathstarbench/linux-x64/venv",
101+
@"/home/user/tools/VirtualClient/packages/deathstarbench/linux-x64/venv/bin/pip install -U pip",
102+
@"/home/user/tools/VirtualClient/packages/deathstarbench/linux-x64/venv/bin/pip install -U setuptools",
103+
@"/home/user/tools/VirtualClient/packages/deathstarbench/linux-x64/venv/bin/pip install aiohttp asyncio",
102104
@"luarocks install luasocket",
103105
};
104106

@@ -116,7 +118,7 @@ public async Task DeathStarBenchExecutorInstallsDepedenciesAsExpected()
116118

117119
await executor.OnInitialize(EventContext.None, CancellationToken.None);
118120

119-
Assert.AreEqual(6, executed);
121+
Assert.AreEqual(8, executed);
120122
}
121123
}
122124

‎src/VirtualClient/VirtualClient.Actions/DeathStarBench/DeathStarBenchExecutor.cs‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,11 @@ public string ServiceName
124124
/// </summary>
125125
protected string ScriptsDirectory { get; set; }
126126

127+
/// <summary>
128+
/// Path to Python executable in virtual environment.
129+
/// </summary>
130+
protected string PythonExecutablePath { get; set; }
131+
127132
/// <summary>
128133
/// An interface that can be used to communicate with the underlying system.
129134
/// </summary>
@@ -546,16 +551,29 @@ await this.ExecuteCommandAsync(dockerComposeCommand, this.PackageDirectory, canc
546551
await this.SystemManager.MakeFileExecutableAsync(dockerComposeFilePath, this.Platform, cancellationToken)
547552
.ConfigureAwait();
548553

549-
string pipUpgradeCommand = "python3 -m pip install -U pip";
554+
string installVenvCommand = "apt install python3-venv -y";
555+
await this.ExecuteCommandAsync(installVenvCommand, this.PackageDirectory, cancellationToken)
556+
.ConfigureAwait();
557+
558+
// Create Python virtual environment to isolate dependencies
559+
string venvPath = this.PlatformSpecifics.Combine(this.PackageDirectory, "venv");
560+
string createVenvCommand = $"python3 -m venv {venvPath}";
561+
await this.ExecuteCommandAsync(createVenvCommand, this.PackageDirectory, cancellationToken)
562+
.ConfigureAwait();
563+
564+
this.PythonExecutablePath = this.PlatformSpecifics.Combine(venvPath, "bin", "python3");
565+
string venvPip = this.PlatformSpecifics.Combine(venvPath, "bin", "pip");
566+
567+
string pipUpgradeCommand = $"{venvPip} install -U pip";
550568
await this.ExecuteCommandAsync(pipUpgradeCommand, this.PackageDirectory, cancellationToken)
551569
.ConfigureAwait();
552570

553-
string setupToolsUpgradeCommand = "python3 -m pip install -U setuptools";
571+
string setupToolsUpgradeCommand = $"{venvPip} install -U setuptools";
554572
await this.ExecuteCommandAsync(setupToolsUpgradeCommand, this.PackageDirectory, cancellationToken)
555573
.ConfigureAwait();
556574

557-
// python3-pip installs pip3 and not pip, better to leave it on python which version of pip to use
558-
string pipInstallPackagesCommand = "-H python3 -m pip install aiohttp asyncio";
575+
// Install required Python packages in virtual environment
576+
string pipInstallPackagesCommand = $"{venvPip} install aiohttp asyncio";
559577
await this.ExecuteCommandAsync(pipInstallPackagesCommand, this.PackageDirectory, cancellationToken)
560578
.ConfigureAwait();
561579

‎src/VirtualClient/VirtualClient.Actions/DeathStarBench/DeathStarBenchServerExecutor.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -123,12 +123,12 @@ await this.WaitAsync(DeathStarBenchExecutor.ServerWarmUpTime, cancellationToken)
123123

124124
if (string.Equals(this.ServiceName, DeathStarBenchExecutor.MediaMicroservices, StringComparison.OrdinalIgnoreCase))
125125
{
126-
await this.ExecuteCommandAsync("python3 scripts/write_movie_info.py -c datasets/tmdb/casts.json -m datasets/tmdb/movies.json", this.ServiceDirectory, cancellationToken)
126+
await this.ExecuteCommandAsync($"{this.PythonExecutablePath} scripts/write_movie_info.py -c datasets/tmdb/casts.json -m datasets/tmdb/movies.json", this.ServiceDirectory, cancellationToken)
127127
.ConfigureAwait();
128128
}
129129
else if (string.Equals(this.ServiceName, DeathStarBenchExecutor.SocialNetwork, StringComparison.OrdinalIgnoreCase))
130130
{
131-
await this.ExecuteCommandAsync(@$"bash -c ""python3 scripts/init_social_graph.py --graph={this.GraphType} --limit=1000""", this.ServiceDirectory, cancellationToken)
131+
await this.ExecuteCommandAsync(@$"bash -c ""{this.PythonExecutablePath} scripts/init_social_graph.py --graph={this.GraphType} --limit=1000""", this.ServiceDirectory, cancellationToken)
132132
.ConfigureAwait();
133133
}
134134
}

‎src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK-DEATHSTARBENCH.json‎

Lines changed: 50 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -54,36 +54,54 @@
5454
}
5555
}
5656
],
57-
"Dependencies": [
58-
{
59-
"Type": "DependencyPackageInstallation",
60-
"Parameters": {
61-
"Scenario": "InstallDeathStarBenchPackage",
62-
"BlobContainer": "packages",
63-
"BlobName": "deathstarbench.3.0.0.zip",
64-
"PackageName": "deathstarbench",
65-
"Extract": true
66-
}
67-
},
68-
{
69-
"Type": "DockerInstallation",
70-
"Parameters": {
71-
"Scenario": "InstallDocker"
72-
}
73-
},
74-
{
75-
"Type": "LinuxPackageInstallation",
76-
"Parameters": {
77-
"Scenario": "InstallLinuxPackages",
78-
"Repositories-Apt": "deb http://security.ubuntu.com/ubuntu xenial-security main",
79-
"Packages-Apt": "python3-pip,libssl-dev,libz-dev,luarocks,libssl1.0.0"
80-
}
81-
},
82-
{
83-
"Type": "ApiServer",
84-
"Parameters": {
85-
"Scenario": "StartAPIServer"
86-
}
87-
}
88-
]
57+
"Dependencies": [
58+
{
59+
"Type": "DependencyPackageInstallation",
60+
"Parameters": {
61+
"Scenario": "InstallDeathStarBenchPackage",
62+
"BlobContainer": "packages",
63+
"BlobName": "deathstarbench.3.0.0.zip",
64+
"PackageName": "deathstarbench",
65+
"Extract": true
66+
}
67+
},
68+
{
69+
"Type": "DockerInstallation",
70+
"Parameters": {
71+
"Scenario": "InstallDocker"
72+
}
73+
},
74+
{
75+
"Type": "WgetPackageInstallation",
76+
"Parameters": {
77+
"Scenario": "InstallLibsslPackage",
78+
"PackageName": "libssl",
79+
"PackageUri": "http://security.ubuntu.com/ubuntu/pool/main/o/openssl1.0/libssl1.0.0_1.0.2n-1ubuntu5.13_amd64.deb",
80+
"SupportedPlatforms": "linux-x64",
81+
"Notes": "Example path to package -> /packages/libssl"
82+
}
83+
},
84+
{
85+
"Type": "ExecuteCommand",
86+
"Parameters": {
87+
"Scenario": "InstallLibssl",
88+
"SupportedPlatforms": "linux-x64",
89+
"Command": "dpkg -i libssl1.0.0_1.0.2n-1ubuntu5.13_amd64.deb",
90+
"WorkingDirectory": "{PackagePath:libssl}"
91+
}
92+
},
93+
{
94+
"Type": "LinuxPackageInstallation",
95+
"Parameters": {
96+
"Scenario": "InstallLinuxPackages",
97+
"Packages-Apt": "python3-pip,libssl-dev,libz-dev,luarocks,libssl1.0.0,python3.12-venv"
98+
}
99+
},
100+
{
101+
"Type": "ApiServer",
102+
"Parameters": {
103+
"Scenario": "StartAPIServer"
104+
}
105+
}
106+
]
89107
}

0 commit comments

Comments
 (0)