Skip to content

Commit 3972ccb

Browse files
nmalkapuramnmalkapuram
committed
mysql version and adding sysbench updated scripts to git
1 parent 64a298a commit 3972ccb

7 files changed

Lines changed: 242 additions & 5 deletions

File tree

‎src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs‎

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,10 @@ namespace VirtualClient.Actions
66
using System;
77
using System.Collections.Generic;
88
using System.Linq;
9-
using System.Net;
9+
using System.Text.RegularExpressions;
1010
using System.Threading;
1111
using System.Threading.Tasks;
1212
using Microsoft.Extensions.DependencyInjection;
13-
using Microsoft.Identity.Client;
1413
using Polly;
1514
using VirtualClient.Common;
1615
using VirtualClient.Common.Extensions;
@@ -138,7 +137,7 @@ private void CaptureMetrics(IProcessProxy process, EventContext telemetryContext
138137
if (!string.IsNullOrEmpty(text))
139138
{
140139
try
141-
{
140+
{
142141
SysbenchMetricsParser parser = new SysbenchMetricsParser(text);
143142
IList<Metric> metrics = parser.Parse();
144143
string sysbenchVersion = null;
@@ -182,6 +181,14 @@ private Task ExecuteWorkloadAsync(EventContext telemetryContext, CancellationTok
182181
{
183182
using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken))
184183
{
184+
if (this.DatabaseSystem == "MySQL")
185+
{
186+
string mysqlVersion = await this.GetMySQLVersionAsync(telemetryContext, cancellationToken);
187+
188+
this.MetadataContract.Add("mysql_version", mysqlVersion, MetadataContractCategory.Dependencies);
189+
this.MetadataContract.Apply(telemetryContext);
190+
}
191+
185192
if (this.Benchmark == BenchmarkName.OLTP)
186193
{
187194
if (this.Action == ClientAction.TruncateDatabase)
@@ -362,5 +369,28 @@ private async Task PrepareOLTPMySQLDatabase(EventContext telemetryContext, Cance
362369
}
363370
}
364371
}
372+
373+
/// <summary>
374+
/// Returns MySQL Version.
375+
/// </summary>
376+
/// <returns></returns>
377+
/// <exception cref="Exception"></exception>
378+
private async Task<string> GetMySQLVersionAsync(EventContext telemetryContext, CancellationToken cancellationToken)
379+
{
380+
try
381+
{
382+
IProcessProxy mysqlversionprocess = await this.ExecuteCommandAsync("sudo", $"mysql -u {this.DatabaseName} -h {this.ServerIpAddress} -e \"SELECT VERSION();\"", Environment.CurrentDirectory, telemetryContext, cancellationToken);
383+
string mysqlVersion = mysqlversionprocess.StandardOutput.ToString();
384+
385+
Regex regex = new Regex(@"(\d+\.\d+\.\d+)");
386+
Match match = regex.Match(mysqlVersion);
387+
388+
return match.Success ? match.Groups[1].Value : string.Empty;
389+
}
390+
catch (Exception)
391+
{
392+
return string.Empty;
393+
}
394+
}
365395
}
366396
}

‎src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ public int? Threads
135135
}
136136

137137
/// <summary>
138-
/// Number of records per table.
138+
/// Database system used. e.g: MYSQL, PostgreSQL.
139139
/// </summary>
140140
public string DatabaseSystem
141141
{

‎src/VirtualClient/VirtualClient.Actions/Sysbench/sysbench scripts/cleanup-database.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
import subprocess, argparse, os
2+
3+
parser = argparse.ArgumentParser()
4+
5+
parser.add_argument('-n', '--dbName', type=str, help='Database Name', required=True)
6+
parser.add_argument('-y', '--databaseSystem', type=str, help='Database Type', required=False)
7+
parser.add_argument('-b', '--benchmark', type=str, help="Benchmark Name", required=True)
8+
parser.add_argument('-t', '--tableCount', type=str, help='Number of Tables', required=True)
9+
parser.add_argument('-a', '--hostIpAddress', type=str, help="Host IP Address", required=True)
10+
11+
args = parser.parse_args()
12+
dbName = args.dbName
13+
databaseSystem = args.databaseSystem
14+
benchmark = args.benchmark
15+
tableCount = args.tableCount
16+
hostIp = args.hostIpAddress
17+
18+
if databaseSystem == "MySQL":
19+
if benchmark == "OLTP":
20+
subprocess.run(f'sudo src/sysbench oltp_common --tables={tableCount} --mysql-db={dbName} --mysql-host={hostIp} cleanup',shell=True, check=True)
21+
else:
22+
parser.error("You are running on a database type that has not been onboarded to Virtual Client. Available options are: MySQL")

‎src/VirtualClient/VirtualClient.Actions/Sysbench/sysbench scripts/configure-workload-generator.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
import subprocess
2+
import argparse
3+
4+
parser = argparse.ArgumentParser()
5+
6+
parser.add_argument('-d', '--distro', type=str, help='Distribution', required=True)
7+
parser.add_argument('-y', '--databaseSystem', type=str, help='Database System', required=False)
8+
parser.add_argument('-p', '--packagePath', type=str, help='Workload Package Path', required=True)
9+
10+
args = parser.parse_args()
11+
distro = args.distro
12+
databaseSystem = args.databaseSystem
13+
path = args.packagePath
14+
15+
subprocess.run('sudo git clone https://github.com/akopytov/sysbench.git', shell=True, check=True)
16+
subprocess.run(f'sudo mv -v {path}/sysbench/* {path}', shell=True, check=True)
17+
18+
subprocess.run('sudo git clone https://github.com/Percona-Lab/sysbench-tpcc.git', shell=True, check=True)
19+
subprocess.run(f'sudo mv -v {path}/sysbench-tpcc/*.lua {path}/src/lua', shell=True, check=True)
20+
21+
if distro == "Ubuntu" or distro == "Debian":
22+
subprocess.run('sudo apt-get update -y', shell=True, check=True)
23+
subprocess.run('sudo apt-get install make automake libtool pkg-config libaio-dev libmysqlclient-dev libssl-dev libpq-dev -y --quiet', shell=True, check=True)
24+
elif distro == "CentOS8" or distro == "RHEL8" or distro == "Mariner":
25+
subprocess.run('sudo dnf update -y', shell=True, check=True)
26+
subprocess.run('sudo dnf install make automake libtool pkg-config libaio-devel, mariadb-devel, openssl-devel, postgresql-devel -y --quiet', shell=True, check=True)
27+
elif distro == "CentOS7" or distro == "RHEL7":
28+
subprocess.run('sudo yum update -y', shell=True, check=True)
29+
subprocess.run('sudo yum install make automake libtool pkg-config libaio-devel, mariadb-devel, openssl-devel, postgresql-devel -y --quiet', shell=True, check=True)
30+
elif distro == "SUSE":
31+
subprocess.run('sudo zypper update', shell=True, check=True)
32+
subprocess.run('sudo zypper --non-interactive install -y make automake libtool pkg-config libaio-dev, libmysqlclient-devel, openssl-devel, postgresql-devel', shell=True, check=True)
33+
else:
34+
parser.error("You are on a Linux distribution that has not been onboarded to Virtual Client.")
35+
36+
subprocess.run('sudo sed -i "s/CREATE TABLE/CREATE TABLE IF NOT EXISTS/g" src/lua/oltp_common.lua', shell=True, check=True)
37+
subprocess.run('sudo ./autogen.sh', shell=True, check=True)
38+
if databaseSystem is None or databaseSystem == "MySQL":
39+
subprocess.run('sudo ./configure', shell=True, check=True)
40+
else:
41+
subprocess.run('sudo ./configure --with-pgsql', shell=True, check=True)
42+
subprocess.run('sudo make -j', shell=True, check=True)
43+
subprocess.run('sudo make install', shell=True, check=True)

‎src/VirtualClient/VirtualClient.Actions/Sysbench/sysbench scripts/populate-database.py‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
import subprocess, argparse, os
2+
3+
parser = argparse.ArgumentParser()
4+
5+
parser.add_argument('-n', '--dbName', type=str, help='Database Name', required=True)
6+
parser.add_argument('-y', '--databaseSystem', type=str, help='Database Type', required=False)
7+
parser.add_argument('-b', '--benchmark', type=str, help="Benchmark Name", required=True)
8+
parser.add_argument('-t', '--tableCount', type=str, help='Number of Tables', required=True)
9+
parser.add_argument('-r', '--recordCount', type=str, help='Number of Records', required=False)
10+
parser.add_argument('-u', '--warehouses', type=str, help='Warehouse Count', required=False)
11+
parser.add_argument('-e', '--threadCount', type=str, help="Number of Threads", required=True)
12+
parser.add_argument('--password', type=str, help="PostgreSQL Password", required=False)
13+
parser.add_argument('--host', type=str, help="Database Server Host IP Address", required=False)
14+
15+
16+
args = parser.parse_args()
17+
dbName = args.dbName
18+
databaseSystem = args.databaseSystem
19+
benchmark = args.benchmark
20+
warehouses = args.warehouses
21+
tableCount = args.tableCount
22+
recordCount = args.recordCount
23+
threadCount = args.threadCount
24+
password = args.password
25+
host = args.host
26+
27+
def add_host_if_needed(command_base, host, benchmark):
28+
if host and benchmark != "TPCC":
29+
if "sysbench" not in command_base:
30+
return command_base.replace('-u', f'-h {host} -u')
31+
else:
32+
return f"{command_base} --mysql-host={host}"
33+
else:
34+
return command_base
35+
36+
if databaseSystem == "MySQL":
37+
if benchmark == "TPCC":
38+
subprocess.run(f'sudo src/sysbench tpcc --tables={tableCount} --scale={warehouses} --threads={threadCount} --mysql-db={dbName} --use_fk=0 prepare', shell=True, check=True)
39+
if int(warehouses) == 1:
40+
for i in range(1,int(tableCount)+1):
41+
table = str(i)
42+
# drop idxs
43+
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX idx_customer{i} ON customer{i};"', shell=True, check=True)
44+
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX idx_orders{i} ON orders{i};"', shell=True, check=True)
45+
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_stock_2{i} ON stock{i};"', shell=True, check=True)
46+
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_order_line_2{i} ON order_line{i};"', shell=True, check=True)
47+
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_history_1{i} ON history{i};"', shell=True, check=True)
48+
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_history_2{i} ON history{i};"', shell=True, check=True)
49+
50+
# truncate, to make distributing faster
51+
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE warehouse{i};"', shell=True, check=True)
52+
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE district{i};"', shell=True, check=True)
53+
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE customer{i};"', shell=True, check=True)
54+
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE history{i};"', shell=True, check=True)
55+
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE orders{i};"', shell=True, check=True)
56+
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE new_orders{i};"', shell=True, check=True)
57+
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE order_line{i};"', shell=True, check=True)
58+
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE stock{i};"', shell=True, check=True)
59+
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE item{i};"', shell=True, check=True)
60+
else:
61+
command_base = f'sudo src/sysbench oltp_common --tables={tableCount} --table-size={recordCount} --threads={threadCount} --mysql-db={dbName} prepare'
62+
command = add_host_if_needed(command_base, host, benchmark)
63+
subprocess.run(command, shell=True, check=True)
64+
if int(recordCount) == 1:
65+
for i in range(1, int(tableCount) + 1):
66+
drop_index_command = f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX k_{i} ON sbtest{i};"'
67+
drop_index_command = add_host_if_needed(drop_index_command, host, benchmark)
68+
subprocess.run(drop_index_command, shell=True, check=True)
69+
elif databaseSystem == "PostgreSQL":
70+
os.environ['PGPASSWORD'] = password
71+
if benchmark == "TPCC":
72+
subprocess.run(f'sudo src/sysbench tpcc --db-driver=pgsql --tables={tableCount} --scale={warehouses} --threads={threadCount} --pgsql-user=postgres --pgsql-password={password} --pgsql-db={dbName} --use_fk=0 prepare', shell=True, check=True)
73+
if int(warehouses) == 1:
74+
for i in range(1,int(tableCount)+1):
75+
table = str(i)
76+
# drop idxs
77+
subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS idx_customer{i};"', shell=True, check=True)
78+
subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS idx_orders{i};"', shell=True, check=True)
79+
subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_stock_2{i};"', shell=True, check=True)
80+
subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_order_line_2{i};"', shell=True, check=True)
81+
subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_history_1{i};"', shell=True, check=True)
82+
subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_history_2{i};"', shell=True, check=True)
83+
84+
# truncate, to make distributing faster
85+
subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE warehouse{i};"', shell=True, check=True)
86+
subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE district{i};"', shell=True, check=True)
87+
subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE customer{i};"', shell=True, check=True)
88+
subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE history{i};"', shell=True, check=True)
89+
subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE orders{i};"', shell=True, check=True)
90+
subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE new_orders{i};"', shell=True, check=True)
91+
subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE order_line{i};"', shell=True, check=True)
92+
subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE stock{i};"', shell=True, check=True)
93+
subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE item{i};"', shell=True, check=True)
94+
else:
95+
subprocess.run(f'sudo src/sysbench oltp_common --db-driver=pgsql --tables={tableCount} --table-size={recordCount} --threads={threadCount} --pgsql-user=postgres --pgsql-password={password} --pgsql-db={dbName} prepare', shell=True, check=True)
96+
if int(recordCount) == 1:
97+
for i in range(1,int(tableCount)+1):
98+
table = str(i)
99+
subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS k_{i};"', shell=True, check=True)
100+
else:
101+
parser.error("You are running on a database system type that has not been onboarded to Virtual Client. Available options are: MySQL, PostgreSQL")

‎src/VirtualClient/VirtualClient.Actions/Sysbench/sysbench scripts/run-workload.py‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
import subprocess, argparse, os
2+
3+
parser = argparse.ArgumentParser()
4+
5+
parser.add_argument('-n', '--dbName', type=str, help='Database Name', required=True)
6+
parser.add_argument('-y', '--databaseSystem', type=str, help='Database Type', required=False)
7+
parser.add_argument('-w', '--workload', type=str, help="Workload Name", required=True)
8+
parser.add_argument('-b', '--benchmark', type=str, help="Benchmark Name", required=True)
9+
parser.add_argument('-t', '--tableCount', type=str, help='Number of Tables', required=True)
10+
parser.add_argument('-r', '--recordCount', type=str, help='Number of Records', required=False)
11+
parser.add_argument('-u', '--warehouses', type=str, help='Warehouse Count', required=False)
12+
parser.add_argument('-e', '--threadCount', type=str, help="Number of Threads", required=True)
13+
parser.add_argument('-a', '--hostIpAddress', type=str, help="Host IP Address", required=True)
14+
parser.add_argument('-d', '--durationSecs', type=str, help="Duration in Seconds", required=True)
15+
parser.add_argument('--password', type=str, help="PostgreSQL Password", required=False)
16+
17+
args = parser.parse_args()
18+
dbName = args.dbName
19+
databaseSystem = args.databaseSystem
20+
workload = args.workload
21+
benchmark = args.benchmark
22+
tableCount = args.tableCount
23+
recordCount = args.recordCount
24+
warehouses = args.warehouses
25+
threadCount = args.threadCount
26+
hostIp = args.hostIpAddress
27+
durationSecs = args.durationSecs
28+
password = args.password
29+
30+
if databaseSystem == "MySQL":
31+
if benchmark == "TPCC":
32+
subprocess.run(f'sudo src/sysbench tpcc --tables={tableCount} --scale={warehouses} --threads={threadCount} --mysql-db={dbName} --mysql-host={hostIp} --time={durationSecs} run', shell=True, check=True)
33+
else:
34+
subprocess.run(f'sudo src/sysbench {workload} --tables={tableCount} --table-size={recordCount} --threads={threadCount} --mysql-db={dbName} --mysql-host={hostIp} --time={durationSecs} run', shell=True, check=True)
35+
elif databaseSystem == "PostgreSQL":
36+
if benchmark == "TPCC":
37+
subprocess.run(f'sudo src/sysbench tpcc --db-driver=pgsql --tables={tableCount} --scale={warehouses} --threads={threadCount} --pgsql-user=postgres --pgsql-password={password} --pgsql-db={dbName} --pgsql-host={hostIp} --time={durationSecs} run', shell=True, check=True)
38+
else:
39+
subprocess.run(f'sudo src/sysbench {workload} --db-driver=pgsql --tables={tableCount} --table-size={recordCount} --threads={threadCount} --pgsql-user=postgres --pgsql-password={password} --pgsql-db={dbName} --pgsql-host={hostIp} --time={durationSecs} run', shell=True, check=True)
40+
else:
41+
parser.error("You are running on a database type that has not been onboarded to Virtual Client. Available options are: MySQL, PostgreSQL")

‎src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@
181181
"Parameters": {
182182
"Scenario": "DownloadSysbenchPackage",
183183
"BlobContainer": "packages",
184-
"BlobName": "sysbench-1.0.20.rev2.zip",
184+
"BlobName": "sysbench-1.0.20.rev3.zip",
185185
"PackageName": "sysbench",
186186
"Extract": true
187187
}

0 commit comments

Comments
 (0)