|
| 1 | +import os |
| 2 | +from dataclasses import dataclass |
| 3 | + |
| 4 | +from aiu_fms_testing_utils.utils.aiu_setup import dprint |
| 5 | + |
| 6 | + |
| 7 | +@dataclass |
| 8 | +class DPPRunnerConfig: |
| 9 | + """Class to configure parameters that may vary with model architecture""" |
| 10 | + |
| 11 | + # populated during setup |
| 12 | + num_blocks: int | None = None |
| 13 | + tkv_limit: int | None = None |
| 14 | + |
| 15 | + def _get_int_env(self, key: str, default: int, context: str) -> int: |
| 16 | + """ |
| 17 | + Read an integer environment variable or use a default. |
| 18 | + Always emits a debug message explaining the choice. |
| 19 | + """ |
| 20 | + value = os.environ.get(key) |
| 21 | + if value is None: |
| 22 | + dprint(f"{context}. Using default {key}={default}") |
| 23 | + return default |
| 24 | + |
| 25 | + try: |
| 26 | + parsed = int(value) |
| 27 | + except ValueError as e: |
| 28 | + raise ValueError( |
| 29 | + f"{context}. Invalid value for environment variable {key}: " |
| 30 | + f"expected an integer, got '{value}'" |
| 31 | + ) from e |
| 32 | + |
| 33 | + dprint(f"{context}. Using {key} from environment: {parsed}") |
| 34 | + return parsed |
| 35 | + |
| 36 | + def _configure_granite_3_8b(self, use_distributed, world_size, prefill_chunk_size): |
| 37 | + """Configure environment for granite 3 8b architecture \ |
| 38 | + We are setting defaults for env variables not provided. \ |
| 39 | + Config class is set in wrapper setup_config function.""" |
| 40 | + |
| 41 | + if use_distributed and world_size == 4: |
| 42 | + ##Only set defaults for TP=4 |
| 43 | + context = ( |
| 44 | + "Model granite-3.3-8b (or compatible) " |
| 45 | + "with tensor parallel size 4 detected" |
| 46 | + ) |
| 47 | + self.tkv_limit = self._get_int_env( |
| 48 | + key="VLLM_DT_MAX_BATCH_TKV_LIMIT", |
| 49 | + default=524288, |
| 50 | + context=context, |
| 51 | + ) |
| 52 | + |
| 53 | + # these values are to be consistent with vllm for granite 3.3 8b instruct |
| 54 | + blocks_override = 8192 if prefill_chunk_size > 0 else 2080 |
| 55 | + |
| 56 | + self.num_blocks = self._get_int_env( |
| 57 | + key="AFTU_PAGED_KVCACHE_NUM_BLOCKS_HINT", |
| 58 | + default=blocks_override, |
| 59 | + context=context, |
| 60 | + ) |
| 61 | + |
| 62 | + def setup_config( |
| 63 | + self, model_variant, use_distributed, world_size, prefill_chunk_size |
| 64 | + ): |
| 65 | + """Set up environment variables and default values if not specified""" |
| 66 | + |
| 67 | + ## configure per model architecture |
| 68 | + if ( |
| 69 | + "granite-3.3-8b-instruct" in model_variant |
| 70 | + or "granite-4.0-8b" in model_variant |
| 71 | + ): |
| 72 | + self._configure_granite_3_8b( |
| 73 | + use_distributed, world_size, prefill_chunk_size |
| 74 | + ) |
| 75 | + |
| 76 | + ## global defaults (fallback) |
| 77 | + ## TODO: IN future we may remove defaults for unknown configurations \ |
| 78 | + ## and require users to set the environment variables |
| 79 | + ## num_blocks is set in generate if not set here |
| 80 | + if self.tkv_limit is None: |
| 81 | + self.tkv_limit = self._get_int_env( |
| 82 | + key="VLLM_DT_MAX_BATCH_TKV_LIMIT", |
| 83 | + default=524288, |
| 84 | + context="Unknown model configuration", |
| 85 | + ) |
| 86 | + |
| 87 | + def env_updates(self) -> dict[str, str]: |
| 88 | + """Returns a key/value of environment variables needed for model compile""" |
| 89 | + if self.tkv_limit is None: |
| 90 | + raise RuntimeError( |
| 91 | + "ModelConfig.env_updates() called before setup_config(). " |
| 92 | + "Call setup_config(...) first." |
| 93 | + ) |
| 94 | + |
| 95 | + return {"VLLM_DT_MAX_BATCH_TKV_LIMIT": str(self.tkv_limit)} |
0 commit comments