Skip to content

Commit 5c2d73c

Browse files
partheaparthea
committed
chore: update post processing scripts
1 parent fdb7a44 commit 5c2d73c

File tree

8 files changed

+149
-85
lines changed

8 files changed

+149
-85
lines changed

‎.librarian/generator-input/client-post-processing/storage-integration.yaml‎

Lines changed: 38 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -115,8 +115,8 @@ replacements:
115115
@nox.session(python=DEFAULT_PYTHON_VERSION)
116116
@nox.session(python=DEFAULT_PYTHON_VERSION)
117117
@nox.parametrize(
118-
"protobuf_implementation",
119-
["python", "upb"],
118+
"protobuf_implementation",
119+
["python", "upb"],
120120
)
121121
def core_deps_from_source(session, protobuf_implementation):
122122
count: 1
@@ -639,12 +639,7 @@ replacements:
639639
]
640640
before: |
641641
google-api-core==2.21.0
642-
google-auth==2.35.0
643-
# TODO\(https://github.com/googleapis/gapic-generator-python/issues/2453\)
644-
# Add the minimum supported version of grpcio to constraints files
645-
proto-plus==1.22.3
646-
protobuf==4.25.8
647-
grpc-google-iam-v1==0.14.0
642+
[\s\S]*?grpc-google-iam-v1==0.14.0
648643
after: |
649644
google-auth==2.26.1
650645
google-api-core==2.27.0
@@ -655,6 +650,41 @@ replacements:
655650
protobuf==4.25.8
656651
opentelemetry-api==1.1.0
657652
count: 1
653+
- paths: [
654+
packages/google-cloud-storage/testing/constraints-3.10.txt,
655+
packages/google-cloud-storage/testing/constraints-3.11.txt,
656+
packages/google-cloud-storage/testing/constraints-3.12.txt,
657+
]
658+
before: |
659+
google-api-core
660+
[\s\S]*?grpc-google-iam-v1
661+
after: |
662+
google-auth
663+
google-api-core
664+
google-cloud-core
665+
google-resumable-media
666+
requests
667+
google-crc32c
668+
protobuf
669+
opentelemetry-api
670+
count: 3
671+
- paths: [
672+
packages/google-cloud-storage/testing/constraints-3.13.txt,
673+
packages/google-cloud-storage/testing/constraints-3.14.txt,
674+
]
675+
before: |
676+
google-api-core>=2
677+
[\s\S]*?grpc-google-iam-v1>=0
678+
after: |
679+
google-auth
680+
google-api-core
681+
google-cloud-core
682+
google-resumable-media
683+
requests
684+
google-crc32c
685+
protobuf
686+
opentelemetry-api
687+
count: 2
658688
- paths:
659689
- "packages/google-cloud-storage/docs/conf.py"
660690
before: |

‎packages/google-cloud-storage/noxfile.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -650,8 +650,8 @@ def prerelease_deps(session, protobuf_implementation):
650650
@nox.session(python=DEFAULT_PYTHON_VERSION)
651651
@nox.session(python=DEFAULT_PYTHON_VERSION)
652652
@nox.parametrize(
653-
"protobuf_implementation",
654-
["python", "upb"],
653+
"protobuf_implementation",
654+
["python", "upb"],
655655
)
656656
def core_deps_from_source(session, protobuf_implementation):
657657
"""Run all tests with core dependencies installed from source

‎packages/google-cloud-storage/testing/constraints-3.10.txt‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
# -*- coding: utf-8 -*-
22
# This constraints file is required for unit tests.
33
# List all library dependencies and extras in this file.
4-
google-api-core
54
google-auth
6-
grpcio
7-
proto-plus
5+
google-api-core
6+
google-cloud-core
7+
google-resumable-media
8+
requests
9+
google-crc32c
810
protobuf
9-
# cryptography is a direct dependency of google-auth
10-
cryptography
11-
grpc-google-iam-v1
11+
opentelemetry-api

‎packages/google-cloud-storage/testing/constraints-3.11.txt‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
# -*- coding: utf-8 -*-
22
# This constraints file is required for unit tests.
33
# List all library dependencies and extras in this file.
4-
google-api-core
54
google-auth
6-
grpcio
7-
proto-plus
5+
google-api-core
6+
google-cloud-core
7+
google-resumable-media
8+
requests
9+
google-crc32c
810
protobuf
9-
# cryptography is a direct dependency of google-auth
10-
cryptography
11-
grpc-google-iam-v1
11+
opentelemetry-api

‎packages/google-cloud-storage/testing/constraints-3.12.txt‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
# -*- coding: utf-8 -*-
22
# This constraints file is required for unit tests.
33
# List all library dependencies and extras in this file.
4-
google-api-core
54
google-auth
6-
grpcio
7-
proto-plus
5+
google-api-core
6+
google-cloud-core
7+
google-resumable-media
8+
requests
9+
google-crc32c
810
protobuf
9-
# cryptography is a direct dependency of google-auth
10-
cryptography
11-
grpc-google-iam-v1
11+
opentelemetry-api

‎packages/google-cloud-storage/testing/constraints-3.13.txt‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@
55
# Require the latest major version be installed for each dependency.
66
# e.g., if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0",
77
# Then this file should have google-cloud-foo>=1
8-
google-api-core>=2
9-
google-auth>=2
10-
grpcio>=1
11-
proto-plus>=1
12-
protobuf>=6
13-
grpc-google-iam-v1>=0
8+
google-auth
9+
google-api-core
10+
google-cloud-core
11+
google-resumable-media
12+
requests
13+
google-crc32c
14+
protobuf
15+
opentelemetry-api

‎packages/google-cloud-storage/testing/constraints-3.14.txt‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@
55
# Require the latest major version be installed for each dependency.
66
# e.g., if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0",
77
# Then this file should have google-cloud-foo>=1
8-
google-api-core>=2
9-
google-auth>=2
10-
grpcio>=1
11-
proto-plus>=1
12-
protobuf>=6
13-
grpc-google-iam-v1>=0
8+
google-auth
9+
google-api-core
10+
google-cloud-core
11+
google-resumable-media
12+
requests
13+
google-crc32c
14+
protobuf
15+
opentelemetry-api

‎packages/google-cloud-storage/tests/unit/test_transfer_manager.py‎

Lines changed: 75 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -110,9 +110,10 @@ def test_upload_many_passes_concurrency_options():
110110
]
111111
MAX_WORKERS = 7
112112
DEADLINE = 10
113-
with mock.patch("concurrent.futures.ThreadPoolExecutor") as pool_patch, mock.patch(
114-
"concurrent.futures.wait"
115-
) as wait_patch:
113+
with (
114+
mock.patch("concurrent.futures.ThreadPoolExecutor") as pool_patch,
115+
mock.patch("concurrent.futures.wait") as wait_patch,
116+
):
116117
transfer_manager.upload_many(
117118
FILE_BLOB_PAIRS,
118119
deadline=DEADLINE,
@@ -130,9 +131,10 @@ def test_threads_deprecation_with_upload():
130131
]
131132
MAX_WORKERS = 7
132133
DEADLINE = 10
133-
with mock.patch("concurrent.futures.ThreadPoolExecutor") as pool_patch, mock.patch(
134-
"concurrent.futures.wait"
135-
) as wait_patch:
134+
with (
135+
mock.patch("concurrent.futures.ThreadPoolExecutor") as pool_patch,
136+
mock.patch("concurrent.futures.wait") as wait_patch,
137+
):
136138
with pytest.warns():
137139
transfer_manager.upload_many(
138140
FILE_BLOB_PAIRS, deadline=DEADLINE, threads=MAX_WORKERS
@@ -321,9 +323,10 @@ def test_download_many_passes_concurrency_options():
321323
]
322324
MAX_WORKERS = 7
323325
DEADLINE = 10
324-
with mock.patch("concurrent.futures.ThreadPoolExecutor") as pool_patch, mock.patch(
325-
"concurrent.futures.wait"
326-
) as wait_patch:
326+
with (
327+
mock.patch("concurrent.futures.ThreadPoolExecutor") as pool_patch,
328+
mock.patch("concurrent.futures.wait") as wait_patch,
329+
):
327330
transfer_manager.download_many(
328331
BLOB_FILE_PAIRS,
329332
deadline=DEADLINE,
@@ -965,10 +968,10 @@ def test_download_chunks_concurrently_passes_concurrency_options():
965968
MULTIPLE = 4
966969
blob_mock.size = CHUNK_SIZE * MULTIPLE
967970

968-
with mock.patch("concurrent.futures.ThreadPoolExecutor") as pool_patch, mock.patch(
969-
"concurrent.futures.wait"
970-
) as wait_patch, mock.patch(
971-
"google.cloud.storage.transfer_manager.open", mock.mock_open()
971+
with (
972+
mock.patch("concurrent.futures.ThreadPoolExecutor") as pool_patch,
973+
mock.patch("concurrent.futures.wait") as wait_patch,
974+
mock.patch("google.cloud.storage.transfer_manager.open", mock.mock_open()),
972975
):
973976
transfer_manager.download_chunks_concurrently(
974977
blob_mock,
@@ -1002,11 +1005,15 @@ def test_upload_chunks_concurrently():
10021005
ETAG = "efgh"
10031006
part_mock.etag = ETAG
10041007

1005-
with mock.patch("os.path.getsize", return_value=SIZE), mock.patch(
1006-
"google.cloud.storage.transfer_manager.XMLMPUContainer",
1007-
return_value=container_mock,
1008-
), mock.patch(
1009-
"google.cloud.storage.transfer_manager.XMLMPUPart", return_value=part_mock
1008+
with (
1009+
mock.patch("os.path.getsize", return_value=SIZE),
1010+
mock.patch(
1011+
"google.cloud.storage.transfer_manager.XMLMPUContainer",
1012+
return_value=container_mock,
1013+
),
1014+
mock.patch(
1015+
"google.cloud.storage.transfer_manager.XMLMPUPart", return_value=part_mock
1016+
),
10101017
):
10111018
transfer_manager.upload_chunks_concurrently(
10121019
FILENAME,
@@ -1045,10 +1052,15 @@ def test_upload_chunks_concurrently_quotes_urls():
10451052
part_mock.etag = ETAG
10461053
container_cls_mock = mock.Mock(return_value=container_mock)
10471054

1048-
with mock.patch("os.path.getsize", return_value=SIZE), mock.patch(
1049-
"google.cloud.storage.transfer_manager.XMLMPUContainer", new=container_cls_mock
1050-
), mock.patch(
1051-
"google.cloud.storage.transfer_manager.XMLMPUPart", return_value=part_mock
1055+
with (
1056+
mock.patch("os.path.getsize", return_value=SIZE),
1057+
mock.patch(
1058+
"google.cloud.storage.transfer_manager.XMLMPUContainer",
1059+
new=container_cls_mock,
1060+
),
1061+
mock.patch(
1062+
"google.cloud.storage.transfer_manager.XMLMPUPart", return_value=part_mock
1063+
),
10521064
):
10531065
transfer_manager.upload_chunks_concurrently(
10541066
FILENAME,
@@ -1089,12 +1101,15 @@ def test_upload_chunks_concurrently_passes_concurrency_options():
10891101
MAX_WORKERS = 7
10901102
DEADLINE = 10
10911103

1092-
with mock.patch("os.path.getsize", return_value=SIZE), mock.patch(
1093-
"google.cloud.storage.transfer_manager.XMLMPUContainer",
1094-
return_value=container_mock,
1095-
), mock.patch("concurrent.futures.ThreadPoolExecutor") as pool_patch, mock.patch(
1096-
"concurrent.futures.wait"
1097-
) as wait_patch:
1104+
with (
1105+
mock.patch("os.path.getsize", return_value=SIZE),
1106+
mock.patch(
1107+
"google.cloud.storage.transfer_manager.XMLMPUContainer",
1108+
return_value=container_mock,
1109+
),
1110+
mock.patch("concurrent.futures.ThreadPoolExecutor") as pool_patch,
1111+
mock.patch("concurrent.futures.wait") as wait_patch,
1112+
):
10981113
try:
10991114
transfer_manager.upload_chunks_concurrently(
11001115
FILENAME,
@@ -1170,13 +1185,19 @@ def test_upload_chunks_concurrently_with_metadata_and_encryption():
11701185

11711186
invocation_id = "b9f8cbb0-6456-420c-819d-3f4ee3c0c455"
11721187

1173-
with mock.patch("os.path.getsize", return_value=SIZE), mock.patch(
1174-
"google.cloud.storage.transfer_manager.XMLMPUContainer", new=container_cls_mock
1175-
), mock.patch(
1176-
"google.cloud.storage.transfer_manager.XMLMPUPart", return_value=part_mock
1177-
), mock.patch(
1178-
"google.cloud.storage._helpers._get_invocation_id",
1179-
return_value="gccl-invocation-id/" + invocation_id,
1188+
with (
1189+
mock.patch("os.path.getsize", return_value=SIZE),
1190+
mock.patch(
1191+
"google.cloud.storage.transfer_manager.XMLMPUContainer",
1192+
new=container_cls_mock,
1193+
),
1194+
mock.patch(
1195+
"google.cloud.storage.transfer_manager.XMLMPUPart", return_value=part_mock
1196+
),
1197+
mock.patch(
1198+
"google.cloud.storage._helpers._get_invocation_id",
1199+
return_value="gccl-invocation-id/" + invocation_id,
1200+
),
11801201
):
11811202
transfer_manager.upload_chunks_concurrently(
11821203
FILENAME,
@@ -1280,10 +1301,13 @@ def test_download_chunks_concurrently_with_processes():
12801301
)
12811302
FILENAME = "file_a.txt"
12821303

1283-
with mock.patch(
1284-
"google.cloud.storage.transfer_manager._download_and_write_chunk_in_place",
1285-
new=_validate_blob_token_in_subprocess_for_chunk,
1286-
), mock.patch("google.cloud.storage.transfer_manager.open", mock.mock_open()):
1304+
with (
1305+
mock.patch(
1306+
"google.cloud.storage.transfer_manager._download_and_write_chunk_in_place",
1307+
new=_validate_blob_token_in_subprocess_for_chunk,
1308+
),
1309+
mock.patch("google.cloud.storage.transfer_manager.open", mock.mock_open()),
1310+
):
12871311
result = transfer_manager.download_chunks_concurrently(
12881312
blob,
12891313
FILENAME,
@@ -1298,9 +1322,12 @@ def test_download_chunks_concurrently_with_processes():
12981322
def test__LazyClient():
12991323
fake_cache = {}
13001324
MOCK_ID = 9999
1301-
with mock.patch(
1302-
"google.cloud.storage.transfer_manager._cached_clients", new=fake_cache
1303-
), mock.patch("google.cloud.storage.transfer_manager.Client"):
1325+
with (
1326+
mock.patch(
1327+
"google.cloud.storage.transfer_manager._cached_clients", new=fake_cache
1328+
),
1329+
mock.patch("google.cloud.storage.transfer_manager.Client"),
1330+
):
13041331
lazyclient = transfer_manager._LazyClient(MOCK_ID)
13051332
lazyclient_cached = transfer_manager._LazyClient(MOCK_ID)
13061333
assert lazyclient is lazyclient_cached
@@ -1370,9 +1397,12 @@ def test__reduce_client():
13701397
}
13711398
client._extra_headers = custom_headers
13721399

1373-
with mock.patch(
1374-
"google.cloud.storage.transfer_manager._cached_clients", new=fake_cache
1375-
), mock.patch("google.cloud.storage.transfer_manager.Client"):
1400+
with (
1401+
mock.patch(
1402+
"google.cloud.storage.transfer_manager._cached_clients", new=fake_cache
1403+
),
1404+
mock.patch("google.cloud.storage.transfer_manager.Client"),
1405+
):
13761406
replicated_client, kwargs = transfer_manager._reduce_client(client)
13771407
assert replicated_client is not None
13781408
assert custom_headers in kwargs

0 commit comments

Comments
 (0)