Skip to content
This repository was archived by the owner on Mar 31, 2026. It is now read-only.

Commit a20622c

Browse files
committed
refactor: enhance docstrings for clarity and detail in read benchmarks
1 parent 799bc99 commit a20622c

1 file changed

Lines changed: 40 additions & 47 deletions

File tree

tests/perf/microbenchmarks/reads/test_reads.py

Lines changed: 40 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -11,16 +11,15 @@
1111
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
14-
"""
15-
Docstring for tests.perf.microbenchmarks.test_reads
16-
17-
File for benchmarking zonal reads (i.e. downloads)
18-
19-
1. 1 object 1 coro with variable chunk_size
20-
21-
calculate latency, throughput, etc for downloads.
14+
"""Microbenchmarks for Google Cloud Storage read operations.
2215
16+
This module contains performance benchmarks for various read patterns from Google Cloud Storage.
17+
It includes three main test functions:
18+
- `test_downloads_single_proc_single_coro`: Benchmarks reads using a single process and a single coroutine.
19+
- `test_downloads_single_proc_multi_coro`: Benchmarks reads using a single process and multiple coroutines.
20+
- `test_downloads_multi_proc_multi_coro`: Benchmarks reads using multiple processes and multiple coroutines.
2321
22+
All other functions in this module are helper methods for these three tests.
2423
"""
2524

2625
import time
@@ -97,14 +96,10 @@ def test_downloads_single_proc_single_coro(
9796
benchmark, storage_client, blobs_to_delete, monitor, workload_params
9897
):
9998
"""
100-
1. create chunks based on the object size and chunk_size. [(start_byte, min(chunk_size, remaining_size))]
101-
2. Pass the list of chunks to `download_chunks_using_mrd` for zonal bucket
102-
`download_chunks_using_json` for regional bucket.
103-
above function are target methods.
104-
3. benchmark target method, using benchmark.pedantic
105-
106-
107-
99+
Benchmarks reads using a single process and a single coroutine.
100+
It creates chunks based on object size and chunk_size, then passes them to either
101+
`download_chunks_using_mrd` (for zonal buckets) or `download_chunks_using_json` (for regional buckets)
102+
for benchmarking using `benchmark.pedantic`.
108103
"""
109104
params, files_names = workload_params
110105

@@ -169,21 +164,22 @@ def target_wrapper(*args, **kwargs):
169164

170165
def download_files_using_mrd_multi_coro(loop, client, files, other_params, chunks):
171166
"""
172-
Docstring for download_files_using_mrd
173-
174-
1. for each file
175-
1. create chunks of size other_params.chunk_size_bytes
176-
2. create a coroutine/task using download_chunks_using_mrd
177-
3. execute all coroutines/task using asyncio.gather in loop.
178-
3. capture latency (output time)
179-
2. output max time.
180-
181-
:param loop: Description
182-
:param client: Description
183-
:param files: Description
184-
:param other_params: Description
185-
"""
167+
Downloads multiple files concurrently using AsyncMultiRangeDownloader (MRD) with asyncio.
168+
169+
For each file, it creates a coroutine to download its chunks using `download_chunks_using_mrd_async`.
170+
All coroutines are then executed concurrently using `asyncio.gather`.
171+
The function returns the maximum latency observed among all coroutines.
172+
173+
Args:
174+
loop: The asyncio event loop.
175+
client: The AsyncGrpcClient instance.
176+
files (list): A list of filenames to download.
177+
other_params: An object containing benchmark parameters (e.g., bucket_name, file_size_bytes).
178+
chunks (list): A list of (offset, size) tuples representing the parts of each file to download.
186179
180+
Returns:
181+
float: The maximum latency (in seconds) among all coroutines.
182+
"""
187183
async def main():
188184
if len(files) == 1:
189185
result = await download_chunks_using_mrd_async(
@@ -206,18 +202,15 @@ def download_files_using_json_multi_threaded(
206202
_, json_client, files, other_params, chunks
207203
):
208204
"""
209-
Docstring for download_files_using_json
210-
211-
1. for each file
212-
1. create chunks of size other_params.chunk_size_bytes
213-
2. using threaPoolexecutor send each file chunks to download_chunks_using_json
214-
3. capture latency (output time)
215-
2. output max time.
216-
217-
:param _: Description
218-
:param json_client: Description
219-
:param files: Description
220-
:param other_params: Description
205+
Downloads multiple files concurrently using the JSON API with a ThreadPoolExecutor.
206+
207+
For each file, it submits a task to a `ThreadPoolExecutor` to download its chunks
208+
using `download_chunks_using_json`. The number of concurrent downloads is
209+
determined by `other_params.num_coros` (which acts as `max_workers`).
210+
The function returns the maximum latency among all concurrent downloads.
211+
212+
The `chunks` parameter is a list of (offset, size) tuples representing
213+
the parts of each file to download.
221214
"""
222215
results = []
223216
# In the context of multi-coro, num_coros is the number of files to download concurrently.
@@ -372,12 +365,12 @@ def test_downloads_multi_proc_multi_coro(
372365
benchmark, storage_client, blobs_to_delete, monitor, workload_params
373366
):
374367
"""
375-
1. this should have the same patterns as `test_downloads_single_proc_multi_coro`
376-
`test_downloads_single_proc_single_coro` but
368+
Benchmarks reads using multiple processes and multiple coroutines.
377369
378-
* it should download files among m process n coroutines. i.e. input files_names
379-
list will contain m*n files. Spawn m process and each process should download n files.
380-
create processes in spwan mode. Output time (latency) for each round should be the maximum latency of all process.
370+
This test distributes `m*n` files among `m` processes, where each process
371+
downloads `n` files concurrently using `n` coroutines. The processes are spawned
372+
in "spawn" mode. The reported latency for each round is the maximum latency
373+
observed across all processes.
381374
"""
382375
params, files_names = workload_params
383376
logging.info(f"num files: {len(files_names)}")

0 commit comments

Comments
 (0)