feat(cli): add fine-tuning preview command (#466)

* feat(cli): add fine-tuning preview command

Co-authored-by: Blaine Kasten <blainekasten@gmail.com>

* fix(cli): polish fine-tuning preview command

Co-authored-by: Blaine Kasten <blainekasten@gmail.com>

* fix(cli): type preview token formatting

Co-authored-by: Blaine Kasten <blainekasten@gmail.com>

---------

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: Blaine Kasten <blainekasten@gmail.com>

Author: 3 people

src/together/lib/cli/__init__.py

@@ -67,6 +67,7 @@
     JIG_VOLUMES_CREATE_HELP_EXAMPLES,
     JIG_VOLUMES_UPDATE_HELP_EXAMPLES,
     BETA_MODELS_CONFIGS_HELP_EXAMPLES,
+    FINE_TUNING_PREVIEW_HELP_EXAMPLES,
     BETA_CLUSTERS_CREATE_HELP_EXAMPLES,
     BETA_CLUSTERS_UPDATE_HELP_EXAMPLES,
     BETA_MODELS_DOWNLOAD_HELP_EXAMPLES,
@@ -466,6 +467,11 @@ async def run_command() -> None:
     help="Retrieve training metrics for a fine-tuning job",
     help_epilogue=FINE_TUNING_LIST_METRICS_HELP_EXAMPLES,
 )
+fine_tuning_app.command(
+    (f"{_CLI}.fine_tuning.preview:preview"),
+    help="Preview how a fine-tuning training file will be tokenized",
+    help_epilogue=FINE_TUNING_PREVIEW_HELP_EXAMPLES,
+)
 
 ## Models API commands
 models_app = app.command(App(name="models", help="List and upload models", help_epilogue=MODELS_HELP_EXAMPLES))

src/together/lib/cli/api/fine_tuning/preview.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+from typing import Literal, Optional, Annotated
+
+from cyclopts import Parameter
+from rich.markup import escape as escape_rich_markup
+
+from together import omit
+from together._utils._json import openapi_dumps
+from together.lib.cli.utils.config import CLIConfigParameter
+from together.lib.cli.utils._console import console
+from together.lib.cli.components.list import ListTable
+from together.lib.cli.components.loader import show_loading_status
+from together.types.fine_tune_preview_row import FineTunePreviewRow
+
+_TOKEN_PREVIEW_LIMIT = 32
+
+
+def _format_token(token: str) -> str:
+    return escape_rich_markup(token.encode("unicode_escape").decode("ascii"))
+
+
+def _format_tokens(row: FineTunePreviewRow) -> str:
+    tokens = row.tokens[:_TOKEN_PREVIEW_LIMIT]
+    labels = row.labels[:_TOKEN_PREVIEW_LIMIT]
+    formatted: list[str] = []
+    for token, label in zip(tokens, labels):
+        token_text = _format_token(token)
+        formatted.append(f"[dim]{token_text}[/dim]" if label == -100 else token_text)
+
+    if len(row.tokens) > _TOKEN_PREVIEW_LIMIT:
+        formatted.append("[dim]...[/dim]")
+
+    return " ".join(formatted)
+
+
+def _format_spans(row: FineTunePreviewRow) -> str:
+    if not row.trained_spans:
+        return "-"
+    return ", ".join(f"{start}-{end}" for start, end in row.trained_spans)
+
+
+async def preview(
+    training_file: Annotated[
+        str,
+        Parameter(
+            alias="-t",
+            help="Training file ID from the Files API to sample for preview",
+        ),
+    ],
+    model: Annotated[
+        str,
+        Parameter(
+            alias="-M",
+            help="Name of the base model whose tokenizer and chat template will be used",
+        ),
+    ],
+    top_k: Annotated[
+        Optional[int],
+        Parameter(help="Maximum number of rows from the start of the training file to tokenize"),
+    ] = None,
+    train_on_inputs: Annotated[
+        Optional[bool],
+        Parameter(help="Whether prompt or user-message tokens should contribute to training loss"),
+    ] = None,
+    training_method: Annotated[
+        Literal["sft"],
+        Parameter(help="Fine-tuning method to preview; only supervised fine-tuning is currently supported"),
+    ] = "sft",
+    *,
+    config: CLIConfigParameter,
+) -> None:
+    """Preview how a fine-tuning training file will be tokenized."""
+    response = await show_loading_status(
+        "Loading fine-tuning preview...",
+        config.client.fine_tuning.preview(
+            model=model,
+            training_file=training_file,
+            top_k=top_k if top_k is not None else omit,
+            train_on_inputs=train_on_inputs if train_on_inputs is not None else omit,
+            training_method=training_method,
+        ),
+    )
+
+    if config.json:
+        console.print_json(openapi_dumps(response).decode("utf-8"))
+        return
+
+    console.print(f"[dim][primary]Model:[/primary][/dim]\t\t[bold]{escape_rich_markup(response.model)}[/bold]")
+    console.print(f"[dim][primary]Dataset format:[/primary][/dim]\t{response.dataset_format}")
+    console.print(f"[dim][primary]Max sequence:[/primary][/dim]\t{response.max_seq_length}")
+    console.print(f"[dim][primary]Train inputs:[/primary][/dim]\t{response.train_on_inputs}")
+
+    table = ListTable("Preview Rows", empty_message="No preview rows returned")
+    table.add_primary_column("Row")
+    table.add_column("Tokens", justify="right")
+    table.add_column("Trained", justify="right")
+    table.add_column("Truncated")
+    table.add_column("Trained Spans")
+    table.add_column("Token Preview", ratio=4)
+
+    for index, row in enumerate(response.rows, start=1):
+        table.add_row(
+            str(index),
+            str(row.num_tokens),
+            str(row.num_trained_tokens),
+            "yes" if row.truncated else "no",
+            _format_spans(row),
+            _format_tokens(row),
+        )
+
+    console.print(table)

src/together/lib/cli/utils/_help_examples.py

@@ -91,6 +91,9 @@
 
 [dim]-[/dim] Cancel a fine-tuning job:
   [primary]tg ft cancel <ft-job-id>[/primary]
+
+[dim]-[/dim] Preview how a training file will be tokenized:
+  [primary]tg ft preview --model Qwen/Qwen2-1.5B --training-file <file-id>[/primary]
 """
 
 FINE_TUNING_CREATE_HELP_EXAMPLES = """[dim]Examples:[/dim]
@@ -127,6 +130,17 @@
   [primary]tg ft list-metrics <ft-job-id> > plots.txt[/primary]
 """
 
+FINE_TUNING_PREVIEW_HELP_EXAMPLES = """[dim]Examples:[/dim]
+[dim]-[/dim] Preview tokenization for the first rows in a training file:
+  [primary]tg ft preview --model Qwen/Qwen2-1.5B --training-file <file-id>[/primary]
+
+[dim]-[/dim] Preview more rows and include prompt tokens in training loss:
+  [primary]tg ft preview -M Qwen/Qwen2-1.5B -t <file-id> --top-k 10 --train-on-inputs[/primary]
+
+[dim]-[/dim] Save the raw preview response:
+  [primary]tg ft preview -M Qwen/Qwen2-1.5B -t <file-id> --json > preview.json[/primary]
+"""
+
 FINE_TUNING_DOWNLOAD_HELP_EXAMPLES = """[dim]Examples:[/dim]
 [dim]-[/dim] Download a fine-tuned model's weights:
   [primary]tg ft download <ft-job-id> --output-dir ./my-model[/primary]

tests/cli/test_fine_tuning.py

@@ -85,6 +85,24 @@
     ]
 }
 
+_FT_PREVIEW_BODY = {
+    "dataset_format": "conversation",
+    "max_seq_length": 4096,
+    "model": "meta-llama/Llama-3-8b",
+    "train_on_inputs": False,
+    "rows": [
+        {
+            "input_ids": [1, 2, 3],
+            "labels": [-100, 2, 3],
+            "num_tokens": 3,
+            "num_trained_tokens": 2,
+            "tokens": ["<s>", " hello", " world"],
+            "trained_spans": [[1, 3]],
+            "truncated": False,
+        }
+    ],
+}
+
 _MODEL_LIMITS_BODY = {
     "max_num_epochs": 10,
     "max_learning_rate": 1,
@@ -397,6 +415,74 @@ def test_list_metrics_json_includes_zero_step_filters(self, respx_mock: MockRout
         assert json.loads(result.output) == _FT_METRICS_BODY["metrics"]
 
 
+class TestFineTuningPreview:
+    @pytest.mark.respx(base_url=base_url)
+    @pytest.mark.parametrize(
+        ("train_on_inputs_flag", "expected_train_on_inputs"),
+        [
+            ("--train-on-inputs", True),
+            ("--no-train-on-inputs", False),
+        ],
+    )
+    def test_preview_json_sends_params(
+        self,
+        respx_mock: MockRouter,
+        cli_runner: CliRunner,
+        train_on_inputs_flag: str,
+        expected_train_on_inputs: bool,
+    ) -> None:
+        route = respx_mock.post("/fine-tunes/preview").mock(return_value=httpx.Response(200, json=_FT_PREVIEW_BODY))
+
+        result = cli_runner.invoke(
+            [
+                "fine-tuning",
+                "preview",
+                "--training-file",
+                "file-train",
+                "--model",
+                "meta-llama/Llama-3-8b",
+                "--top-k",
+                "5",
+                train_on_inputs_flag,
+                "--training-method",
+                "sft",
+                "--json",
+            ]
+        )
+
+        assert result.exit_code == 0
+        assert json.loads(result.output) == _FT_PREVIEW_BODY
+        request_body = json.loads(cast(Call, route.calls[0]).request.content)
+        assert request_body == {
+            "model": "meta-llama/Llama-3-8b",
+            "training_file": "file-train",
+            "top_k": 5,
+            "train_on_inputs": expected_train_on_inputs,
+            "training_method": "sft",
+        }
+
+    @pytest.mark.respx(base_url=base_url)
+    def test_preview_table(self, respx_mock: MockRouter, cli_runner: CliRunner) -> None:
+        respx_mock.post("/fine-tunes/preview").mock(return_value=httpx.Response(200, json=_FT_PREVIEW_BODY))
+
+        result = cli_runner.invoke(
+            [
+                "fine-tuning",
+                "preview",
+                "--training-file",
+                "file-train",
+                "--model",
+                "meta-llama/Llama-3-8b",
+            ]
+        )
+
+        assert result.exit_code == 0
+        assert "conversation" in result.output
+        assert "Preview Rows" in result.output
+        assert "1-3" in result.output
+        assert "hello" in result.output
+
+
 class TestFineTuningDownload:
     @pytest.mark.respx(base_url=base_url)
     def test_download_invokes_download_manager(

tests/cli/test_json_mode_pipeable_to_jq.py

@@ -126,6 +126,7 @@ def test_fine_tuning_json_mode(self) -> None:
         fine_tuning.run_and_assert("delete ft-123 --force")
         fine_tuning.run_and_assert("list-events ft-123")
         fine_tuning.run_and_assert("list-checkpoints ft-123")
+        fine_tuning.run_and_assert("preview --training-file file-123 --model deepseek-ai/DeepSeek-R1")
         fine_tuning.run_and_assert("retrieve-checkpoint ft-123/checkpoint-123")
         fine_tuning.run_and_assert("retrieve-checkpoint ft-123/checkpoint-123")