Skip to content

Commit 53b276b

Browse files
Make GCP region configurable for managed ML diagnostics runs in MaxText
to support multi-region deployments. `managed_mldiagnostics_region` defaults to empty string. When empty, the SDK will auto-detect the region from GCP metadata server. - Add strictly typed `managed_mldiagnostics_region`: str = "" to `ManagedMLDiagnostics` Pydantic class in `types.py`. - Add `managed_mldiagnostics_region`: "" to `base.yml`. - Pass configured region directly to `machinelearning_run` in `managed_mldiagnostics.py`, falling back to `None` if empty to trigger SDK auto-detection. PiperOrigin-RevId: 932279476
1 parent 5f3dc2b commit 53b276b

4 files changed

Lines changed: 91 additions & 9 deletions

File tree

src/maxtext/common/managed_mldiagnostics.py

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
# limitations under the License.
1414

1515
"""Create the managed mldiagnostics run."""
16+
1617
import json
1718
from typing import Any
1819

@@ -24,16 +25,16 @@
2425

2526

2627
class ManagedMLDiagnostics:
27-
"""
28-
ML Diagnostics Run, implemented with the Singleton pattern.
28+
"""ML Diagnostics Run, implemented with the Singleton pattern.
29+
2930
Ensures that only one instance of the class can exist.
3031
"""
3132

3233
_instance = None # Class attribute to hold the single instance
3334

3435
def __new__(cls, *args: Any, **kwargs: Any):
35-
"""
36-
Overrides the instance creation method.
36+
"""Overrides the instance creation method.
37+
3738
If an instance already exists, it is returned instead of creating a new one.
3839
"""
3940
if cls._instance is None:
@@ -42,9 +43,7 @@ def __new__(cls, *args: Any, **kwargs: Any):
4243
return cls._instance
4344

4445
def __init__(self, config):
45-
"""
46-
Initializes the ManagedMLDiagnostics, ensuring this method runs only once.
47-
"""
46+
"""Initializes the ManagedMLDiagnostics, ensuring this method runs only once."""
4847
# We need a flag to ensure __init__ only runs once,
4948
# as the object is returned multiple times by __new__.
5049
if hasattr(self, "_initialized"):
@@ -67,11 +66,11 @@ def should_log_key(key, value):
6766
config_dict = {key: value for key, value in config.get_keys().items() if should_log_key(key, value)}
6867

6968
# Create a run for the managed mldiagnostics, and upload the configuration.
69+
region = config.managed_mldiagnostics_region if config.managed_mldiagnostics_region else None
7070
mldiag.machinelearning_run(
7171
name=f"{config.run_name}",
7272
run_group=config.managed_mldiagnostics_run_group,
7373
configs=config_dict,
7474
gcs_path=config.managed_mldiagnostics_dir,
75-
# TODO: b/455623960 - Remove the following once multi-region and prod support are enabled.
76-
region="us-central1",
75+
region=region,
7776
)

src/maxtext/configs/base.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -839,6 +839,7 @@ tpu_num_sparse_cores_to_trace: 2
839839
# - upload training metrics, at the defined log_period interval.
840840
managed_mldiagnostics: false # Whether to enable the managed diagnostics
841841
managed_mldiagnostics_run_group: "" # Optional. Used to group multiple runs.
842+
managed_mldiagnostics_region: "" # Optional. GCP region for managed mldiagnostics. If empty, it will be auto-detected by the SDK.
842843

843844
# Dump HLO and jaxpr options
844845
dump_hlo: false

src/maxtext/configs/types.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1786,6 +1786,7 @@ class ManagedMLDiagnostics(BaseModel):
17861786

17871787
managed_mldiagnostics: bool = Field(False, description="Enable managed mldiagnostics.")
17881788
managed_mldiagnostics_run_group: str = Field("", description="Name used to group multiple runs.")
1789+
managed_mldiagnostics_region: str = Field("", description="GCP region for managed mldiagnostics.")
17891790

17901791

17911792
class Goodput(BaseModel):
Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Unit tests for ManagedMLDiagnostics."""
16+
17+
import unittest
18+
from unittest import mock
19+
20+
from maxtext.common.managed_mldiagnostics import ManagedMLDiagnostics
21+
import pytest
22+
23+
24+
@pytest.mark.cpu_only
25+
class ManagedMLDiagnosticsTest(unittest.TestCase):
26+
# pylint: disable=protected-access
27+
28+
def setUp(self):
29+
super().setUp()
30+
# Reset singleton instance between tests
31+
ManagedMLDiagnostics._instance = None
32+
33+
def test_not_enabled_noop(self):
34+
mock_config = mock.MagicMock()
35+
mock_config.managed_mldiagnostics = False
36+
37+
with mock.patch.object(ManagedMLDiagnostics.mldiag, "machinelearning_run") as mock_run:
38+
ManagedMLDiagnostics(mock_config)
39+
mock_run.assert_not_called()
40+
41+
def test_enabled_empty_region_passes_none(self):
42+
mock_config = mock.MagicMock()
43+
mock_config.managed_mldiagnostics = True
44+
mock_config.managed_mldiagnostics_region = ""
45+
mock_config.run_name = "test_run"
46+
mock_config.managed_mldiagnostics_run_group = "test_group"
47+
mock_config.managed_mldiagnostics_dir = "gs://test_dir"
48+
mock_config.get_keys.return_value = {"key1": "val1"}
49+
50+
with mock.patch.object(ManagedMLDiagnostics.mldiag, "machinelearning_run") as mock_run:
51+
ManagedMLDiagnostics(mock_config)
52+
mock_run.assert_called_once_with(
53+
name="test_run",
54+
run_group="test_group",
55+
configs={"key1": "val1"},
56+
gcs_path="gs://test_dir",
57+
region=None,
58+
)
59+
60+
def test_enabled_populated_region_passes_region(self):
61+
mock_config = mock.MagicMock()
62+
mock_config.managed_mldiagnostics = True
63+
mock_config.managed_mldiagnostics_region = "us-east1"
64+
mock_config.run_name = "test_run"
65+
mock_config.managed_mldiagnostics_run_group = "test_group"
66+
mock_config.managed_mldiagnostics_dir = "gs://test_dir"
67+
mock_config.get_keys.return_value = {"key1": "val1"}
68+
69+
with mock.patch.object(ManagedMLDiagnostics.mldiag, "machinelearning_run") as mock_run:
70+
ManagedMLDiagnostics(mock_config)
71+
mock_run.assert_called_once_with(
72+
name="test_run",
73+
run_group="test_group",
74+
configs={"key1": "val1"},
75+
gcs_path="gs://test_dir",
76+
region="us-east1",
77+
)
78+
79+
80+
if __name__ == "__main__":
81+
unittest.main()

0 commit comments

Comments
 (0)