diff --git a/src/together/lib/cli/__init__.py b/src/together/lib/cli/__init__.py index abbb8e2e..6148af15 100644 --- a/src/together/lib/cli/__init__.py +++ b/src/together/lib/cli/__init__.py @@ -466,6 +466,10 @@ async def run_command() -> None: help="Retrieve training metrics for a fine-tuning job", help_epilogue=FINE_TUNING_LIST_METRICS_HELP_EXAMPLES, ) +fine_tuning_app.command( + (f"{_CLI}.fine_tuning.preview:preview"), + help="Preview how a fine-tuning training file will be tokenized", +) ## Models API commands models_app = app.command(App(name="models", help="List and upload models", help_epilogue=MODELS_HELP_EXAMPLES)) diff --git a/src/together/lib/cli/api/fine_tuning/preview.py b/src/together/lib/cli/api/fine_tuning/preview.py new file mode 100644 index 00000000..7327ec5b --- /dev/null +++ b/src/together/lib/cli/api/fine_tuning/preview.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +from typing import Literal, Optional, Annotated + +from cyclopts import Parameter +from rich.markup import escape as escape_rich_markup + +from together import omit +from together._utils._json import openapi_dumps +from together.types.fine_tune_preview_row import FineTunePreviewRow +from together.lib.cli.utils.config import CLIConfigParameter +from together.lib.cli.utils._console import console +from together.lib.cli.components.list import ListTable +from together.lib.cli.components.loader import show_loading_status + +_TOKEN_PREVIEW_LIMIT = 32 + + +def _format_token(token: str) -> str: + return escape_rich_markup(token.encode("unicode_escape").decode("ascii")) + + +def _format_tokens(row: FineTunePreviewRow) -> str: + tokens = row.tokens[:_TOKEN_PREVIEW_LIMIT] + labels = row.labels[:_TOKEN_PREVIEW_LIMIT] + formatted = [] + for token, label in zip(tokens, labels): + token_text = _format_token(token) + formatted.append(f"[dim]{token_text}[/dim]" if label == -100 else token_text) + + if len(row.tokens) > _TOKEN_PREVIEW_LIMIT: + formatted.append("[dim]...[/dim]") + + return " ".join(formatted) + + +def _format_spans(row: FineTunePreviewRow) -> str: + if not row.trained_spans: + return "-" + return ", ".join(f"{start}-{end}" for start, end in row.trained_spans) + + +async def preview( + training_file: Annotated[ + str, + Parameter( + alias="-t", + help="Training file ID from the Files API to sample for preview", + ), + ], + model: Annotated[ + str, + Parameter( + alias="-M", + help="Name of the base model whose tokenizer and chat template will be used", + ), + ], + top_k: Annotated[ + Optional[int], + Parameter(help="Maximum number of rows from the start of the training file to tokenize"), + ] = None, + train_on_inputs: Annotated[ + Optional[bool], + Parameter(help="Whether prompt or user-message tokens should contribute to training loss"), + ] = None, + training_method: Annotated[ + Literal["sft"], + Parameter(help="Fine-tuning method to preview; only supervised fine-tuning is currently supported"), + ] = "sft", + *, + config: CLIConfigParameter, +) -> None: + """Preview how a fine-tuning training file will be tokenized.""" + response = await show_loading_status( + "Loading fine-tuning preview...", + config.client.fine_tuning.preview( + model=model, + training_file=training_file, + top_k=top_k if top_k is not None else omit, + train_on_inputs=train_on_inputs if train_on_inputs is not None else omit, + training_method=training_method, + ), + ) + + if config.json: + console.print_json(openapi_dumps(response).decode("utf-8")) + return + + console.print(f"[dim][primary]Model:[/primary][/dim]\t\t[bold]{escape_rich_markup(response.model)}[/bold]") + console.print(f"[dim][primary]Dataset format:[/primary][/dim]\t{response.dataset_format}") + console.print(f"[dim][primary]Max sequence:[/primary][/dim]\t{response.max_seq_length}") + console.print(f"[dim][primary]Train inputs:[/primary][/dim]\t{response.train_on_inputs}") + + table = ListTable("Preview Rows", empty_message="No preview rows returned") + table.add_primary_column("Row") + table.add_column("Tokens", justify="right") + table.add_column("Trained", justify="right") + table.add_column("Truncated") + table.add_column("Trained Spans") + table.add_column("Token Preview", ratio=4) + + for index, row in enumerate(response.rows, start=1): + table.add_row( + str(index), + str(row.num_tokens), + str(row.num_trained_tokens), + "yes" if row.truncated else "no", + _format_spans(row), + _format_tokens(row), + ) + + console.print(table) diff --git a/tests/cli/test_fine_tuning.py b/tests/cli/test_fine_tuning.py index 5a17a6fc..27b4be41 100644 --- a/tests/cli/test_fine_tuning.py +++ b/tests/cli/test_fine_tuning.py @@ -82,6 +82,24 @@ ] } +_FT_PREVIEW_BODY = { + "dataset_format": "conversation", + "max_seq_length": 4096, + "model": "meta-llama/Llama-3-8b", + "train_on_inputs": False, + "rows": [ + { + "input_ids": [1, 2, 3], + "labels": [-100, 2, 3], + "num_tokens": 3, + "num_trained_tokens": 2, + "tokens": ["", " hello", " world"], + "trained_spans": [[1, 3]], + "truncated": False, + } + ], +} + _MODEL_LIMITS_BODY = { "max_num_epochs": 10, "max_learning_rate": 1, @@ -392,6 +410,63 @@ def test_list_metrics_json_includes_zero_step_filters(self, respx_mock: MockRout assert json.loads(result.output) == _FT_METRICS_BODY["metrics"] +class TestFineTuningPreview: + @pytest.mark.respx(base_url=base_url) + def test_preview_json_sends_params(self, respx_mock: MockRouter, cli_runner: CliRunner) -> None: + route = respx_mock.post("/fine-tunes/preview").mock( + return_value=httpx.Response(200, json=_FT_PREVIEW_BODY) + ) + + result = cli_runner.invoke( + [ + "fine-tuning", + "preview", + "--training-file", + "file-train", + "--model", + "meta-llama/Llama-3-8b", + "--top-k", + "5", + "--no-train-on-inputs", + "--training-method", + "sft", + "--json", + ] + ) + + assert result.exit_code == 0 + assert json.loads(result.output) == _FT_PREVIEW_BODY + request_body = json.loads(cast(Call, route.calls[0]).request.content) + assert request_body == { + "model": "meta-llama/Llama-3-8b", + "training_file": "file-train", + "top_k": 5, + "train_on_inputs": False, + "training_method": "sft", + } + + @pytest.mark.respx(base_url=base_url) + def test_preview_table(self, respx_mock: MockRouter, cli_runner: CliRunner) -> None: + respx_mock.post("/fine-tunes/preview").mock(return_value=httpx.Response(200, json=_FT_PREVIEW_BODY)) + + result = cli_runner.invoke( + [ + "fine-tuning", + "preview", + "--training-file", + "file-train", + "--model", + "meta-llama/Llama-3-8b", + ] + ) + + assert result.exit_code == 0 + assert "conversation" in result.output + assert "Preview Rows" in result.output + assert "1-3" in result.output + assert "hello" in result.output + + class TestFineTuningDownload: @pytest.mark.respx(base_url=base_url) def test_download_invokes_download_manager( diff --git a/tests/cli/test_json_mode_pipeable_to_jq.py b/tests/cli/test_json_mode_pipeable_to_jq.py index 1b2ac2f8..e88481d6 100644 --- a/tests/cli/test_json_mode_pipeable_to_jq.py +++ b/tests/cli/test_json_mode_pipeable_to_jq.py @@ -126,6 +126,7 @@ def test_fine_tuning_json_mode(self) -> None: fine_tuning.run_and_assert("delete ft-123 --force") fine_tuning.run_and_assert("list-events ft-123") fine_tuning.run_and_assert("list-checkpoints ft-123") + fine_tuning.run_and_assert("preview --training-file file-123 --model deepseek-ai/DeepSeek-R1") fine_tuning.run_and_assert("retrieve-checkpoint ft-123/checkpoint-123") fine_tuning.run_and_assert("retrieve-checkpoint ft-123/checkpoint-123")