diff --git a/langfuse/_client/client.py b/langfuse/_client/client.py index c8782c638..4214bc44b 100644 --- a/langfuse/_client/client.py +++ b/langfuse/_client/client.py @@ -2921,17 +2921,31 @@ async def _process_experiment_item( try: # Use sync API to avoid event loop issues when run_async_safely # creates multiple event loops across different threads + + # Create request, only include datasetVersion if not None + if dataset_version is not None: + dataset_run_item_request = CreateDatasetRunItemRequest( + run_name=experiment_run_name, + run_description=experiment_description, + metadata=experiment_metadata, + dataset_item_id=item.id, # type: ignore + trace_id=trace_id, + observation_id=span.id, + dataset_version=dataset_version, + ) + else: + dataset_run_item_request = CreateDatasetRunItemRequest( + run_name=experiment_run_name, + run_description=experiment_description, + metadata=experiment_metadata, + dataset_item_id=item.id, # type: ignore + trace_id=trace_id, + observation_id=span.id, + ) + dataset_run_item = await asyncio.to_thread( self.api.dataset_run_items.create, - request=CreateDatasetRunItemRequest( - runName=experiment_run_name, - runDescription=experiment_description, - metadata=experiment_metadata, - datasetItemId=item.id, # type: ignore - traceId=trace_id, - observationId=span.id, - datasetVersion=dataset_version, - ), + request=dataset_run_item_request, ) dataset_run_id = dataset_run_item.dataset_run_id diff --git a/tests/test_datasets.py b/tests/test_datasets.py index 1a0cda63a..0c6e03d61 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -640,3 +640,75 @@ def simple_task(*, item, **kwargs): assert result.name == "Versioned Dataset Test" assert len(result.item_results) == 1 # Only one item in versioned dataset assert result.item_results[0].output == 4 + + +def test_run_experiment_without_dataset_version(monkeypatch): + langfuse = Langfuse(debug=False) + + # Create dataset + name = create_uuid() + langfuse.create_dataset(name=name) + + # Create item + langfuse.create_dataset_item( + dataset_name=name, input={"question": "What is 2+2?"}, expected_output=4 + ) + langfuse.flush() + time.sleep(3) + + # Get dataset without version timestamp - this should have version=None + dataset = langfuse.get_dataset(name) + assert dataset.version is None, "Dataset should have no version" + + # Capture HTTP requests to dataset-run-items endpoint to inspect the request body + captured_requests = [] + original_request = langfuse.api._client_wrapper.httpx_client.request + + def capture_request(method, url, **kwargs): + # Capture requests to the dataset-run-items endpoint + if "dataset-run-items" in str(url) and method.upper() == "POST": + # Capture the JSON body being sent + captured_requests.append(kwargs.get("json")) + return original_request(method, url, **kwargs) + + monkeypatch.setattr( + langfuse.api._client_wrapper.httpx_client, "request", capture_request + ) + + # Run a simple experiment on the dataset + def simple_task(*, item, **kwargs): + # Just return a static answer + return item.expected_output + + result = dataset.run_experiment( + name="Dataset without version Test", + description="Testing experiment with dataset without version", + task=simple_task, + ) + + # Verify experiment ran successfully + assert result.name == "Dataset without version Test" + assert len(result.item_results) == 1 # Only one item in dataset + assert result.item_results[0].output == 4 + assert result.dataset_run_id is not None, "Should have created a dataset run" + + # Verify that HTTP requests were captured + assert len(captured_requests) > 0, ( + "Should have captured dataset run item creation requests" + ) + + # Verify that datasetVersion was NOT included in any request body + for request_body in captured_requests: + assert request_body is not None, "Request body should not be None" + # Check if request_body is a dict or has a dict method + if hasattr(request_body, "dict"): + body_dict = request_body.dict() + elif isinstance(request_body, dict): + body_dict = request_body + else: + body_dict = json.loads(json.dumps(request_body)) + + assert "datasetVersion" not in body_dict, ( + f"datasetVersion should not be in request body when dataset has no version. " + f"Found in body: {body_dict}" + )