Skip to content

Commit c2d1190

Browse files
Google-ML-Automationcj401-amd
authored andcommitted
Add support for On-The-Fly Dynamic SafeTensors loading.
Reverts cccfc67 PiperOrigin-RevId: 940173584
1 parent f0314c0 commit c2d1190

9 files changed

Lines changed: 21 additions & 1061 deletions

File tree

src/maxtext/checkpoint_conversion/to_maxtext.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,7 @@
6767
from maxtext.common.common_types import MODEL_MODE_TRAIN
6868
from maxtext.checkpoint_conversion.utils.hf_model_configs import HF_MODEL_CONFIGS
6969
from maxtext.checkpoint_conversion.utils.param_mapping import HOOK_FNS, PARAM_MAPPING
70-
from maxtext.checkpoint_conversion.utils.tensor_handling import apply_hook_fns
71-
from maxtext.checkpoint_conversion.utils.utils import MemoryMonitorTqdm, load_hf_dict_from_transformers, load_hf_dict_from_safetensors, param_key_parts_from_path, print_peak_memory, print_ram_usage, save_weights_to_checkpoint, validate_and_filter_param_map_keys
70+
from maxtext.checkpoint_conversion.utils.utils import MemoryMonitorTqdm, apply_hook_fns, load_hf_dict_from_transformers, load_hf_dict_from_safetensors, param_key_parts_from_path, print_peak_memory, print_ram_usage, save_weights_to_checkpoint, validate_and_filter_param_map_keys
7271
from maxtext.inference.inference_utils import str2bool
7372
from maxtext.layers import quantizations
7473
from maxtext.models import models

src/maxtext/checkpoint_conversion/utils/load_dynamic.py

Lines changed: 0 additions & 371 deletions
This file was deleted.

0 commit comments

Comments
 (0)