Merge branch 'main' into release-please--branches--main--changes--next

Author: blainekasten

src/together/lib/cli/api/fine_tuning/create.py

@@ -91,6 +91,10 @@ async def create(
         int | Literal["max"],
         Parameter(converter=int_or_max_converter, alias="-b", help="Train batch size"),
     ] = "max",
+    gradient_accumulation_steps: Annotated[
+        Optional[int],
+        Parameter(help="Number of gradient accumulation steps (increases effective batch size without more memory)"),
+    ] = None,
     learning_rate: Annotated[float, Parameter(alias="--lr", help="Learning rate")] = DEFAULT_LEARNING_RATE,
     lr_scheduler_type: Annotated[
         Literal["linear", "cosine"], Parameter(help="Learning rate scheduler type")
@@ -174,6 +178,7 @@ async def create(
         max_seq_length=max_seq_length,
         n_checkpoints=n_checkpoints,
         batch_size=batch_size,
+        gradient_accumulation_steps=gradient_accumulation_steps,
         learning_rate=learning_rate,
         lr_scheduler_type=lr_scheduler_type,
         min_lr_ratio=min_lr_ratio,

src/together/lib/resources/fine_tuning.py

@@ -42,6 +42,7 @@ def create_finetune_request(
     n_evals: int | None = 0,
     n_checkpoints: int | None = 1,
     batch_size: int | Literal["max"] = "max",
+    gradient_accumulation_steps: int | None = None,
     learning_rate: float | None = 0.00001,
     lr_scheduler_type: Literal["linear", "cosine"] = "cosine",
     min_lr_ratio: float | None = 0.0,
@@ -255,6 +256,7 @@ def create_finetune_request(
         n_evals=n_evals,
         n_checkpoints=n_checkpoints,
         batch_size=batch_size,
+        gradient_accumulation_steps=gradient_accumulation_steps,
         learning_rate=learning_rate or 0.00001,
         lr_scheduler=lr_scheduler,
         warmup_ratio=warmup_ratio,

src/together/lib/types/fine_tuning.py

@@ -481,6 +481,8 @@ class FinetuneRequest(BaseModel):
     max_seq_length: Union[int, None] = None
     # training batch size
     batch_size: Union[int, Literal["max"], None] = None
+    # gradient accumulation steps
+    gradient_accumulation_steps: Union[int, None] = None
     # up to 40 character suffix for output model name
     suffix: Union[str, None] = None
     # weights & biases api key

src/together/resources/fine_tuning.py

@@ -88,6 +88,7 @@ def create(
         n_evals: int | None = 0,
         n_checkpoints: int | None = 1,
         batch_size: int | Literal["max"] = "max",
+        gradient_accumulation_steps: int | None = None,
         learning_rate: float | None = 0.00001,
         lr_scheduler_type: Literal["linear", "cosine"] = "cosine",
         min_lr_ratio: float = 0.0,
@@ -220,6 +221,7 @@ def create(
             n_evals=n_evals,
             n_checkpoints=n_checkpoints,
             batch_size=batch_size,
+            gradient_accumulation_steps=gradient_accumulation_steps,
             learning_rate=learning_rate,
             lr_scheduler_type=lr_scheduler_type,
             min_lr_ratio=min_lr_ratio,
@@ -693,6 +695,7 @@ async def create(
         n_evals: int | None = 0,
         n_checkpoints: int | None = 1,
         batch_size: int | Literal["max"] = "max",
+        gradient_accumulation_steps: int | None = None,
         learning_rate: float | None = 0.00001,
         lr_scheduler_type: Literal["linear", "cosine"] = "cosine",
         min_lr_ratio: float = 0.0,
@@ -824,6 +827,7 @@ async def create(
             n_evals=n_evals,
             n_checkpoints=n_checkpoints,
             batch_size=batch_size,
+            gradient_accumulation_steps=gradient_accumulation_steps,
             learning_rate=learning_rate,
             lr_scheduler_type=lr_scheduler_type,
             min_lr_ratio=min_lr_ratio,