Skip to content
This repository was archived by the owner on Mar 31, 2026. It is now read-only.

Commit 3c0dac2

Browse files
chandra-sirichandra-siri
committed
refactor: reorganize benchmark parameters add each parameter.py in reads , writes folder
1 parent 7dd5e00 commit 3c0dac2

9 files changed

Lines changed: 145 additions & 101 deletions

File tree

‎tests/perf/microbenchmarks/conftest.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
AsyncAppendableObjectWriter,
2727
)
2828
from google.cloud.storage._experimental.asyncio.async_grpc_client import AsyncGrpcClient
29-
from tests.perf.microbenchmarks.parameters import WriteParameters
29+
from tests.perf.microbenchmarks.writes.parameters import WriteParameters
3030

3131
_OBJECT_NAME_PREFIX = "micro-benchmark"
3232

‎tests/perf/microbenchmarks/parameters.py‎

Lines changed: 1 addition & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -15,22 +15,7 @@
1515

1616

1717
@dataclass
18-
class ReadParameters:
19-
name: str
20-
workload_name: str
21-
pattern: str
22-
bucket_name: str
23-
bucket_type: str
24-
num_coros: int
25-
num_processes: int
26-
num_files: int
27-
rounds: int
28-
chunk_size_bytes: int
29-
file_size_bytes: int
30-
31-
32-
@dataclass
33-
class WriteParameters:
18+
class IOBenchmarkParameters:
3419
name: str
3520
workload_name: str
3621
bucket_name: str

‎tests/perf/microbenchmarks/reads/__init__.py‎

Whitespace-only changes.

tests/perf/microbenchmarks/config.py renamed to tests/perf/microbenchmarks/reads/config.py

Lines changed: 3 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -11,20 +11,16 @@
1111
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
14-
# nit: TODO: rename it to config_to_params.py
1514
import itertools
1615
import os
1716
from typing import Dict, List
1817

1918
import yaml
2019

2120
try:
22-
from tests.perf.microbenchmarks.parameters import ReadParameters, WriteParameters
21+
from tests.perf.microbenchmarks.reads.parameters import ReadParameters
2322
except ModuleNotFoundError:
24-
from parameters import ReadParameters, WriteParameters
25-
26-
27-
23+
from reads.parameters import ReadParameters
2824

2925

3026
def _get_params() -> Dict[str, List[ReadParameters]]:
@@ -37,7 +33,7 @@ def _get_params() -> Dict[str, List[ReadParameters]]:
3733
you may use itertools.product
3834
"""
3935
params: Dict[str, List[ReadParameters]] = {}
40-
config_path = os.path.join(os.path.dirname(__file__), "reads/config.yaml")
36+
config_path = os.path.join(os.path.dirname(__file__), "config.yaml")
4137
with open(config_path, "r") as f:
4238
config = yaml.safe_load(f)
4339

@@ -103,79 +99,3 @@ def _get_params() -> Dict[str, List[ReadParameters]]:
10399
)
104100
)
105101
return params
106-
107-
108-
def get_write_params() -> Dict[str, List[WriteParameters]]:
109-
"""
110-
Docstring for get_write_params
111-
1. this function output a list of WriteParameters.
112-
2. to populate the values of WriteParameters, use default values from config_writes.yaml
113-
3. generate all possible params , ie
114-
no. of params should be equal to bucket_type*file_size_mib, chunk_size * process * coros
115-
you may use itertools.product
116-
"""
117-
params: Dict[str, List[WriteParameters]] = {}
118-
config_path = os.path.join(os.path.dirname(__file__), "writes/config.yaml")
119-
with open(config_path, "r") as f:
120-
config = yaml.safe_load(f)
121-
122-
common_params = config["common"]
123-
bucket_types = common_params["bucket_types"]
124-
file_sizes_mib = common_params["file_sizes_mib"]
125-
chunk_sizes_mib = common_params["chunk_sizes_mib"]
126-
rounds = common_params["rounds"]
127-
128-
bucket_map = {
129-
"zonal": config["defaults"]["DEFAULT_RAPID_ZONAL_BUCKET"],
130-
"regional": config["defaults"]["DEFAULT_STANDARD_BUCKET"],
131-
}
132-
133-
for workload in config["workload"]:
134-
workload_name = workload["name"]
135-
params[workload_name] = []
136-
processes = workload["processes"]
137-
coros = workload["coros"]
138-
139-
# Create a product of all parameter combinations
140-
product = itertools.product(
141-
bucket_types,
142-
file_sizes_mib,
143-
chunk_sizes_mib,
144-
processes,
145-
coros,
146-
)
147-
148-
for (
149-
bucket_type,
150-
file_size_mib,
151-
chunk_size_mib,
152-
num_processes,
153-
num_coros,
154-
) in product:
155-
file_size_bytes = file_size_mib * 1024 * 1024
156-
chunk_size_bytes = chunk_size_mib * 1024 * 1024
157-
bucket_name = bucket_map[bucket_type]
158-
159-
if "single_file" in workload_name:
160-
num_files = 1
161-
else:
162-
num_files = num_processes * num_coros
163-
164-
# Create a descriptive name for the parameter set
165-
name = f"{workload_name}_{bucket_type}_{num_processes}p_{num_coros}c"
166-
167-
params[workload_name].append(
168-
WriteParameters(
169-
name=name,
170-
workload_name=workload_name,
171-
bucket_name=bucket_name,
172-
bucket_type=bucket_type,
173-
num_coros=num_coros,
174-
num_processes=num_processes,
175-
num_files=num_files,
176-
rounds=rounds,
177-
chunk_size_bytes=chunk_size_bytes,
178-
file_size_bytes=file_size_bytes,
179-
)
180-
)
181-
return params

‎tests/perf/microbenchmarks/reads/parameters.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
from dataclasses import dataclass
15+
from ..parameters import IOBenchmarkParameters
16+
17+
18+
@dataclass
19+
class ReadParameters(IOBenchmarkParameters):
20+
pattern: str

‎tests/perf/microbenchmarks/reads/test_reads.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@
3939
from tests.perf.microbenchmarks.conftest import (
4040
publish_resource_metrics,
4141
)
42-
import tests.perf.microbenchmarks.config as config
42+
import tests.perf.microbenchmarks.reads.config as config
4343
from concurrent.futures import ThreadPoolExecutor
4444
import multiprocessing
4545

‎tests/perf/microbenchmarks/writes/__init__.py‎

Whitespace-only changes.

‎tests/perf/microbenchmarks/writes/config.py‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
import itertools
15+
import os
16+
from typing import Dict, List
17+
18+
import yaml
19+
20+
try:
21+
from tests.perf.microbenchmarks.writes.parameters import WriteParameters
22+
except ModuleNotFoundError:
23+
from writes.parameters import WriteParameters
24+
25+
26+
def get_write_params() -> Dict[str, List[WriteParameters]]:
27+
"""
28+
Docstring for get_write_params
29+
1. this function output a list of WriteParameters.
30+
2. to populate the values of WriteParameters, use default values from config_writes.yaml
31+
3. generate all possible params , ie
32+
no. of params should be equal to bucket_type*file_size_mib, chunk_size * process * coros
33+
you may use itertools.product
34+
"""
35+
params: Dict[str, List[WriteParameters]] = {}
36+
config_path = os.path.join(os.path.dirname(__file__), "config_writes.yaml")
37+
with open(config_path, "r") as f:
38+
config = yaml.safe_load(f)
39+
40+
common_params = config["common"]
41+
bucket_types = common_params["bucket_types"]
42+
file_sizes_mib = common_params["file_sizes_mib"]
43+
chunk_sizes_mib = common_params["chunk_sizes_mib"]
44+
rounds = common_params["rounds"]
45+
46+
bucket_map = {
47+
"zonal": config["defaults"]["DEFAULT_RAPID_ZONAL_BUCKET"],
48+
"regional": config["defaults"]["DEFAULT_STANDARD_BUCKET"],
49+
}
50+
51+
for workload in config["workload"]:
52+
workload_name = workload["name"]
53+
params[workload_name] = []
54+
processes = workload["processes"]
55+
coros = workload["coros"]
56+
57+
# Create a product of all parameter combinations
58+
product = itertools.product(
59+
bucket_types,
60+
file_sizes_mib,
61+
chunk_sizes_mib,
62+
processes,
63+
coros,
64+
)
65+
66+
for (
67+
bucket_type,
68+
file_size_mib,
69+
chunk_size_mib,
70+
num_processes,
71+
num_coros,
72+
) in product:
73+
file_size_bytes = file_size_mib * 1024 * 1024
74+
chunk_size_bytes = chunk_size_mib * 1024 * 1024
75+
bucket_name = bucket_map[bucket_type]
76+
77+
if "single_file" in workload_name:
78+
num_files = 1
79+
else:
80+
num_files = num_processes * num_coros
81+
82+
# Create a descriptive name for the parameter set
83+
name = f"{workload_name}_{bucket_type}_{num_processes}p_{num_coros}c"
84+
85+
params[workload_name].append(
86+
WriteParameters(
87+
name=name,
88+
workload_name=workload_name,
89+
bucket_name=bucket_name,
90+
bucket_type=bucket_type,
91+
num_coros=num_coros,
92+
num_processes=num_processes,
93+
num_files=num_files,
94+
rounds=rounds,
95+
chunk_size_bytes=chunk_size_bytes,
96+
file_size_bytes=file_size_bytes,
97+
)
98+
)
99+
return params

‎tests/perf/microbenchmarks/writes/parameters.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
from dataclasses import dataclass
15+
from ..parameters import IOBenchmarkParameters
16+
17+
18+
@dataclass
19+
class WriteParameters(IOBenchmarkParameters):
20+
pass

0 commit comments

Comments
 (0)