From 89081ee916a2f4500d6c6d5864c2bada41644f54 Mon Sep 17 00:00:00 2001 From: Luke Schaefer Date: Tue, 28 Jul 2026 09:53:32 -0500 Subject: [PATCH 1/3] feat: make create_benchmark asynchronous (poll build job) The server now creates a benchmark in a 'building' state and streams its members in via a background job, responding 202 with {benchmark_id, job_id} (this removes the previous item-count ceiling on slice/dataset-sourced benchmarks). Update create_benchmark to match: - POST, then by default poll the build job to completion (reusing AsyncJob) and return the ready benchmark. Return type is unchanged, so existing blocking callers are unaffected; a failed build raises JobError. - Add wait_for_completion (default True) to return the 'building' benchmark immediately for callers that want to poll themselves. - Benchmark now exposes `status` ('building' | 'ready' | 'failed'). Bumps to 0.20.0. Stacked on #467. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 5 ++++ nucleus/__init__.py | 34 +++++++++++++++++++++++----- nucleus/benchmark.py | 4 ++++ pyproject.toml | 2 +- tests/test_benchmarks.py | 49 ++++++++++++++++++++++++++++++++++++---- 5 files changed, 82 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15998781..01c049a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to the [Nucleus Python Client](https://github.com/scaleapi/n The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.20.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.20.0) - 2026-07-27 + +### Changed +- **`create_benchmark()` is now asynchronous.** The server creates the benchmark in a `"building"` state and streams its members in via a background job (removing the previous item-count ceiling on slice/dataset-sourced benchmarks). `create_benchmark()` blocks on that job by default and returns the completed `"ready"` benchmark — the return type is unchanged, so existing blocking callers are unaffected. Pass `wait_for_completion=False` to return the `"building"` benchmark immediately and poll it yourself via `Benchmark.refresh()` (checking the new `Benchmark.status` field). A failed build job raises `JobError`. `Benchmark` now exposes `status` (`"building"` / `"ready"` / `"failed"`). + ## [0.19.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.19.0) - 2026-07-10 ### Added diff --git a/nucleus/__init__.py b/nucleus/__init__.py index 055439cc..0f6e2484 100644 --- a/nucleus/__init__.py +++ b/nucleus/__init__.py @@ -1096,15 +1096,22 @@ def create_benchmark( items: Optional[List[Dict[str, str]]] = None, slice_id: Optional[str] = None, dataset_id: Optional[str] = None, + wait_for_completion: bool = True, + verbose: bool = True, ) -> Benchmark: """Create a benchmark from ground-truth items. Provide the members through exactly one source: explicit ``item_ids``, ``(dataset_id, ref_id)`` pairs via ``items``, all items in a slice via ``slice_id``, or all items in a dataset via ``dataset_id``. Items - without ground truth are skipped (reported on the returned benchmark - as ``skipped_items_without_ground_truth``). Membership is frozen at - creation. + without ground truth are skipped. Membership is frozen at creation. + + Creation is **asynchronous**: the server creates the benchmark in a + ``"building"`` state and streams its members in via a background job. + By default this method blocks until that job finishes and returns the + completed (``"ready"``) benchmark. Pass ``wait_for_completion=False`` + to return immediately with a ``"building"`` benchmark you can poll via + :meth:`Benchmark.refresh` (checking ``benchmark.status``). Parameters: name: Benchmark display name. @@ -1114,9 +1121,14 @@ def create_benchmark( items: ``{"dataset_id": ..., "ref_id": ...}`` pairs. slice_id: Slice id (``slc_*``) whose items become members. dataset_id: Dataset id (``ds_*``) whose items become members. + wait_for_completion: Block until the build job finishes and return + the ready benchmark (default). If ``False``, return the + ``"building"`` benchmark immediately. + verbose: Log build-job polling progress while waiting. Returns: - :class:`Benchmark`: The created benchmark. + :class:`Benchmark`: The created benchmark — ``"ready"`` when + ``wait_for_completion`` is ``True``, otherwise ``"building"``. """ sources = [ source @@ -1141,8 +1153,18 @@ def create_benchmark( payload["slice_id"] = slice_id if dataset_id is not None: payload["dataset_id"] = dataset_id - data = self.post(payload, "benchmarks") - return Benchmark.from_json(data, self) + + # Async: the server responds 202 with {benchmark_id, job_id}. The + # benchmark row already exists (in 'building' state); the build job + # streams members in and flips it to 'ready' (or 'failed'). + response = self.post(payload, "benchmarks") + benchmark_id = response["benchmark_id"] + job_id = response.get("job_id") + + if wait_for_completion and job_id is not None: + self.get_job(job_id).sleep_until_complete(verbose_std_out=verbose) + + return self.get_benchmark(benchmark_id) def list_benchmarks(self) -> List[Benchmark]: """List benchmarks visible to the current user. diff --git a/nucleus/benchmark.py b/nucleus/benchmark.py index f7c92802..25bcccd3 100644 --- a/nucleus/benchmark.py +++ b/nucleus/benchmark.py @@ -46,6 +46,9 @@ class Benchmark: item_count: Optional[int] = None dataset_count: Optional[int] = None skipped_items_without_ground_truth: Optional[int] = None + #: Async build lifecycle: ``"building"`` until the server's build job finishes + #: streaming members in, then ``"ready"`` (or ``"failed"``). + status: Optional[str] = None _client: Optional["NucleusClient"] = field(repr=False, default=None) @classmethod @@ -66,6 +69,7 @@ def from_json( skipped_items_without_ground_truth=payload.get( "skipped_items_without_ground_truth" ), + status=payload.get("status"), _client=client, ) diff --git a/pyproject.toml b/pyproject.toml index e12a2c2f..6f6de6a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ ignore = ["E501", "E741", "E731", "F401"] # Easy ignore for getting it running [tool.poetry] name = "scale-nucleus" -version = "0.19.0" +version = "0.20.0" description = "The official Python client library for Nucleus, the Data Platform for AI" license = "MIT" authors = ["Scale AI Nucleus Team "] diff --git a/tests/test_benchmarks.py b/tests/test_benchmarks.py index 49c5510e..9333f916 100644 --- a/tests/test_benchmarks.py +++ b/tests/test_benchmarks.py @@ -48,9 +48,31 @@ def test_benchmark_from_json_maps_benchmark_id(): assert b.skipped_items_without_ground_truth == 3 -def test_create_benchmark_from_slice(): - client = NucleusClient(api_key="test") - client.connection.post = MagicMock(return_value=dict(_BENCHMARK_ROW)) +def test_benchmark_from_json_parses_status(): + assert Benchmark.from_json(_BENCHMARK_ROW).status is None + assert ( + Benchmark.from_json({**_BENCHMARK_ROW, "status": "building"}).status + == "building" + ) + + +def _mock_async_create(client, *, benchmark_row=None): + """Wire up the async create_benchmark flow: 202 {benchmark_id, job_id} -> + poll the build job -> re-fetch the ready benchmark.""" + client.connection.post = MagicMock( + return_value={"benchmark_id": "bm_1", "job_id": "job_1"} + ) + client.get_job = MagicMock() # .sleep_until_complete() is a no-op MagicMock + client.get_benchmark = MagicMock( + return_value=Benchmark.from_json( + benchmark_row or {**_BENCHMARK_ROW, "status": "ready"}, client + ) + ) + return client + + +def test_create_benchmark_from_slice_polls_then_returns_ready(): + client = _mock_async_create(NucleusClient(api_key="test")) benchmark = client.create_benchmark( "city-streets", description="desc", slice_id="slc_1" ) @@ -61,12 +83,16 @@ def test_create_benchmark_from_slice(): "description": "desc", "slice_id": "slc_1", } + # Blocks on the build job by default, then fetches the ready benchmark. + client.get_job.assert_called_once_with("job_1") + client.get_job.return_value.sleep_until_complete.assert_called_once() + client.get_benchmark.assert_called_once_with("bm_1") assert benchmark.id == "bm_1" + assert benchmark.status == "ready" def test_create_benchmark_from_item_ids_and_metadata(): - client = NucleusClient(api_key="test") - client.connection.post = MagicMock(return_value=dict(_BENCHMARK_ROW)) + client = _mock_async_create(NucleusClient(api_key="test")) client.create_benchmark( "city-streets", metadata={"team": "av"}, @@ -77,6 +103,19 @@ def test_create_benchmark_from_item_ids_and_metadata(): assert payload["metadata"] == {"team": "av"} +def test_create_benchmark_no_wait_returns_building_without_polling(): + client = _mock_async_create( + NucleusClient(api_key="test"), + benchmark_row={**_BENCHMARK_ROW, "status": "building", "item_count": 0}, + ) + benchmark = client.create_benchmark( + "city-streets", slice_id="slc_1", wait_for_completion=False + ) + client.get_job.assert_not_called() + client.get_benchmark.assert_called_once_with("bm_1") + assert benchmark.status == "building" + + def test_create_benchmark_requires_exactly_one_member_source(): client = NucleusClient(api_key="test") with pytest.raises(ValueError, match="exactly one"): From c871c8b73f604eef87c997b03b6812baa2e8c38e Mon Sep 17 00:00:00 2001 From: Luke Schaefer Date: Tue, 28 Jul 2026 09:57:07 -0500 Subject: [PATCH 2/3] feat: multi-source create_benchmark (slice_ids/dataset_ids, combinable) Mirror the server's multi-source benchmark creation: create_benchmark now accepts plural slice_ids and dataset_ids alongside the singular slice_id/ dataset_id/item_ids/items, and members from all provided sources are unioned and de-duplicated server-side. Requirement relaxed from "exactly one source" to "at least one source". Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 3 +++ nucleus/__init__.py | 38 +++++++++++++++++++++++++++----------- tests/test_benchmarks.py | 22 ++++++++++++++++++---- 3 files changed, 48 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01c049a5..4dd7a2fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.20.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.20.0) - 2026-07-27 +### Added +- **Multi-source `create_benchmark()`.** Members can now come from any combination of `item_ids`, `(dataset_id, ref_id)` `items`, one or more slices (`slice_id` / `slice_ids`), and one or more datasets (`dataset_id` / `dataset_ids`) — unioned and de-duplicated server-side. At least one source is required (previously exactly one). + ### Changed - **`create_benchmark()` is now asynchronous.** The server creates the benchmark in a `"building"` state and streams its members in via a background job (removing the previous item-count ceiling on slice/dataset-sourced benchmarks). `create_benchmark()` blocks on that job by default and returns the completed `"ready"` benchmark — the return type is unchanged, so existing blocking callers are unaffected. Pass `wait_for_completion=False` to return the `"building"` benchmark immediately and poll it yourself via `Benchmark.refresh()` (checking the new `Benchmark.status` field). A failed build job raises `JobError`. `Benchmark` now exposes `status` (`"building"` / `"ready"` / `"failed"`). diff --git a/nucleus/__init__.py b/nucleus/__init__.py index 0f6e2484..46801138 100644 --- a/nucleus/__init__.py +++ b/nucleus/__init__.py @@ -1096,15 +1096,19 @@ def create_benchmark( items: Optional[List[Dict[str, str]]] = None, slice_id: Optional[str] = None, dataset_id: Optional[str] = None, + slice_ids: Optional[List[str]] = None, + dataset_ids: Optional[List[str]] = None, wait_for_completion: bool = True, verbose: bool = True, ) -> Benchmark: """Create a benchmark from ground-truth items. - Provide the members through exactly one source: explicit ``item_ids``, - ``(dataset_id, ref_id)`` pairs via ``items``, all items in a slice via - ``slice_id``, or all items in a dataset via ``dataset_id``. Items - without ground truth are skipped. Membership is frozen at creation. + Provide members through any combination of sources: explicit + ``item_ids``, ``(dataset_id, ref_id)`` pairs via ``items``, one or more + slices via ``slice_id`` / ``slice_ids``, and one or more datasets via + ``dataset_id`` / ``dataset_ids``. Members are unioned and de-duplicated + across all sources; at least one source is required. Items without + ground truth are skipped. Membership is frozen at creation. Creation is **asynchronous**: the server creates the benchmark in a ``"building"`` state and streams its members in via a background job. @@ -1121,6 +1125,8 @@ def create_benchmark( items: ``{"dataset_id": ..., "ref_id": ...}`` pairs. slice_id: Slice id (``slc_*``) whose items become members. dataset_id: Dataset id (``ds_*``) whose items become members. + slice_ids: Multiple slice ids whose items become members. + dataset_ids: Multiple dataset ids whose items become members. wait_for_completion: Block until the build job finishes and return the ready benchmark (default). If ``False``, return the ``"building"`` benchmark immediately. @@ -1130,15 +1136,21 @@ def create_benchmark( :class:`Benchmark`: The created benchmark — ``"ready"`` when ``wait_for_completion`` is ``True``, otherwise ``"building"``. """ - sources = [ + has_source = any( source - for source in (item_ids, items, slice_id, dataset_id) - if source is not None - ] - if len(sources) != 1: + for source in ( + item_ids, + items, + slice_id, + dataset_id, + slice_ids, + dataset_ids, + ) + ) + if not has_source: raise ValueError( - "Provide exactly one of item_ids, items, slice_id, or " - "dataset_id to define benchmark membership" + "Provide at least one of item_ids, items, slice_id(s), or " + "dataset_id(s) to define benchmark membership" ) payload: Dict[str, Any] = {"name": name} if description is not None: @@ -1153,6 +1165,10 @@ def create_benchmark( payload["slice_id"] = slice_id if dataset_id is not None: payload["dataset_id"] = dataset_id + if slice_ids is not None: + payload["slice_ids"] = slice_ids + if dataset_ids is not None: + payload["dataset_ids"] = dataset_ids # Async: the server responds 202 with {benchmark_id, job_id}. The # benchmark row already exists (in 'building' state); the build job diff --git a/tests/test_benchmarks.py b/tests/test_benchmarks.py index 9333f916..96e809fc 100644 --- a/tests/test_benchmarks.py +++ b/tests/test_benchmarks.py @@ -116,12 +116,26 @@ def test_create_benchmark_no_wait_returns_building_without_polling(): assert benchmark.status == "building" -def test_create_benchmark_requires_exactly_one_member_source(): +def test_create_benchmark_requires_at_least_one_member_source(): client = NucleusClient(api_key="test") - with pytest.raises(ValueError, match="exactly one"): + with pytest.raises(ValueError, match="at least one"): client.create_benchmark("b") - with pytest.raises(ValueError, match="exactly one"): - client.create_benchmark("b", slice_id="slc_1", dataset_id="ds_1") + with pytest.raises(ValueError, match="at least one"): + client.create_benchmark("b", slice_ids=[], dataset_ids=[]) + + +def test_create_benchmark_combines_multiple_sources(): + client = _mock_async_create(NucleusClient(api_key="test")) + client.create_benchmark( + "multi", + item_ids=["di_3"], + slice_ids=["slc_1", "slc_2"], + dataset_ids=["ds_1", "ds_2"], + ) + payload = client.connection.post.call_args[0][0] + assert payload["item_ids"] == ["di_3"] + assert payload["slice_ids"] == ["slc_1", "slc_2"] + assert payload["dataset_ids"] == ["ds_1", "ds_2"] def test_list_benchmarks(): From 9f5ffd4fb02c6739feb1aa96fff99848c5ed399b Mon Sep 17 00:00:00 2001 From: Luke Schaefer Date: Tue, 28 Jul 2026 10:01:00 -0500 Subject: [PATCH 3/3] Update nucleus/__init__.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- nucleus/__init__.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nucleus/__init__.py b/nucleus/__init__.py index 46801138..f9e8c0b1 100644 --- a/nucleus/__init__.py +++ b/nucleus/__init__.py @@ -1175,9 +1175,13 @@ def create_benchmark( # streams members in and flips it to 'ready' (or 'failed'). response = self.post(payload, "benchmarks") benchmark_id = response["benchmark_id"] - job_id = response.get("job_id") - - if wait_for_completion and job_id is not None: + if wait_for_completion: + if job_id is None: + raise ValueError( + "Server did not return a job_id in the create-benchmark " + "response; cannot poll for completion. Pass " + "wait_for_completion=False to suppress this error." + ) self.get_job(job_id).sleep_until_complete(verbose_std_out=verbose) return self.get_benchmark(benchmark_id)