Skip to content

Commit b87ab82

Browse files
author
Sharon Yu
committed
merge replica resize and checkpoint patch
1 parent 3d4c538 commit b87ab82

1 file changed

Lines changed: 20 additions & 0 deletions

File tree

pathwaysutils/persistence/orbax_handler.py

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,27 @@ async def serialize(
143143
)
144144

145145
self._wait_for_directory_creation_signals()
146+
146147
locations, names = extract_parent_dir_and_name(infos)
148+
149+
path_counts = collections.Counter()
150+
for location, name in zip(locations, names):
151+
path_counts[(location, name)] += 1
152+
for (location, name), count in path_counts.items():
153+
if count > 1:
154+
logger.warning(
155+
"CloudPathwaysArrayHandler: Found duplicate write request for"
156+
" location=%s, name=%s, count=%d",
157+
location,
158+
name,
159+
count,
160+
)
161+
else:
162+
logging.info(
163+
"CloudPathwaysArrayHandler: Writing location=%s, name=%s",
164+
location,
165+
name,
166+
)
147167
f = functools.partial(helper.write_one_array, timeout=self.timeout)
148168
futures_results = list(map(f, locations, names, arrays))
149169

0 commit comments

Comments
 (0)