-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy path__init__.py
More file actions
90 lines (80 loc) · 4.33 KB
/
Copy path__init__.py
File metadata and controls
90 lines (80 loc) · 4.33 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
"""Data readers and batch collation helpers.
The package also contains portable corpus/indexer modules used on Linux hosts
where Apple's MLX runtime is unavailable. Public MLX-backed exports are loaded
on first attribute access so importing a portable submodule does not eagerly
import the entire training runtime.
"""
from __future__ import annotations
from importlib import import_module
from typing import Any
_EXPORT_MODULES = {
"LMTokenBatch": "cppmega_mlx.data.batch",
"ensure_lm_batch": "cppmega_mlx.data.batch",
"synthetic_token_batch": "cppmega_mlx.data.batch",
"LocalTokenBatchDataset": "cppmega_mlx.data.dataloader_bridge",
"TorchDataLoaderBridgeConfig": "cppmega_mlx.data.dataloader_bridge",
"TorchDataLoaderBridgeError": "cppmega_mlx.data.dataloader_bridge",
"build_spawn_dataloader": "cppmega_mlx.data.dataloader_bridge",
"is_torch_dataloader_available": "cppmega_mlx.data.dataloader_bridge",
"iter_mlx_batches": "cppmega_mlx.data.dataloader_bridge",
"EOT_ID": "cppmega_mlx.data.fim",
"FIMSpecialTokenIds": "cppmega_mlx.data.fim",
"FIM_INSTRUCTION_ID": "cppmega_mlx.data.fim",
"FIMMode": "cppmega_mlx.data.fim",
"FIM_MIDDLE_ID": "cppmega_mlx.data.fim",
"FIM_PREFIX_ID": "cppmega_mlx.data.fim",
"FIM_SPECIAL_TOKEN_IDS": "cppmega_mlx.data.fim",
"FIM_SUFFIX_ID": "cppmega_mlx.data.fim",
"apply_fim_permutation": "cppmega_mlx.data.fim",
"apply_fim_transform": "cppmega_mlx.data.fim",
"apply_ifim_permutation": "cppmega_mlx.data.fim",
"apply_ifim_transform": "cppmega_mlx.data.fim",
"extract_ifim_instruction_text": "cppmega_mlx.data.fim",
"sample_middle_span": "cppmega_mlx.data.fim",
"MegatronIndexedDataset": "cppmega_mlx.data.megatron_indexed",
"MegatronIndexedMetadata": "cppmega_mlx.data.megatron_indexed",
"MegatronIndexedMultiShardDataset": "cppmega_mlx.data.megatron_indexed",
"MegatronIndexedMultiShardMetadata": "cppmega_mlx.data.megatron_indexed",
"megatron_indexed_side_channel_schema": "cppmega_mlx.data.megatron_indexed",
"open_megatron_indexed_dataset": "cppmega_mlx.data.megatron_indexed",
"OversizedSamplePolicy": "cppmega_mlx.data.packing",
"PackedSequences": "cppmega_mlx.data.packing",
"PackingStrategy": "cppmega_mlx.data.packing",
"cumulative_doc_ids_from_eos": "cppmega_mlx.data.packing",
"document_boundary_mask": "cppmega_mlx.data.packing",
"mlx_cumulative_doc_ids_from_eos": "cppmega_mlx.data.packing",
"mlx_document_boundary_mask": "cppmega_mlx.data.packing",
"mlx_sequence_packing_attention_mask": "cppmega_mlx.data.packing",
"pack_bos_aligned_best_fit": "cppmega_mlx.data.packing",
"pack_documents_with_eos": "cppmega_mlx.data.packing",
"MultiShardTokenParquetDataset": "cppmega_mlx.data.parquet_dataset",
"ParquetColumns": "cppmega_mlx.data.parquet_dataset",
"TokenParquetDataset": "cppmega_mlx.data.parquet_dataset",
"MAX_PLATFORM_IDS": "cppmega_mlx.data.platform_context",
"PLATFORM_VOCAB": "cppmega_mlx.data.platform_context",
"PLATFORM_VOCAB_SIZE": "cppmega_mlx.data.platform_context",
"PlatformContext": "cppmega_mlx.data.platform_context",
"encode_platform_context": "cppmega_mlx.data.platform_context",
"parse_platform_context": "cppmega_mlx.data.platform_context",
"platform_ids_array": "cppmega_mlx.data.platform_context",
"render_platform_context": "cppmega_mlx.data.platform_context",
"REQUIRED_SPECIAL_TOKEN_IDS": "cppmega_mlx.data.tokenizer_contract",
"SpecialTokenMapping": "cppmega_mlx.data.tokenizer_contract",
"TOOL_USE_SPECIAL_TOKEN_IDS": "cppmega_mlx.data.tokenizer_contract",
"validate_required_special_token_ids": "cppmega_mlx.data.tokenizer_contract",
"BatchCursor": "cppmega_mlx.data.token_dataset",
"TokenDatasetMetadata": "cppmega_mlx.data.token_dataset",
"TokenNpzDataset": "cppmega_mlx.data.token_dataset",
"iterate_token_batches": "cppmega_mlx.data.token_dataset",
"open_token_dataset": "cppmega_mlx.data.token_dataset",
}
__all__ = list(_EXPORT_MODULES)
def __getattr__(name: str) -> Any:
module_name = _EXPORT_MODULES.get(name)
if module_name is None:
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
value = getattr(import_module(module_name), name)
globals()[name] = value
return value
def __dir__() -> list[str]:
return sorted(set(globals()) | set(__all__))