feat(datasets): support multimodal dataset items in the SDK

Mirror langfuse/langfuse-python#1710 to add media support to datasets:

- LangfuseMediaReference (core): a signed-URL media handle returned when
  resolving dataset media, with urlIsExpired / fetchBytes / fetchBase64 /
  fetchDataUri helpers for feeding media to LLM providers
- uploadMedia (core): reusable upload routine that works without a trace
  context; MediaService now delegates to it instead of duplicating the
  upload + backoff logic
- DatasetManager.createItem: uploads any LangfuseMedia found in input,
  expectedOutput, or metadata (deduped) and replaces it with a reference
  string before creating the item; createDatasetItem now routes here
- DatasetManager.get(resolveMediaReferences): requests includeMediaReferences
  and hydrates reference strings into LangfuseMediaReference objects using
  jsonpath-plus (eval-free, so it is safe under strict CSP / edge runtimes)
- re-export LangfuseMedia and LangfuseMediaReference from @langfuse/client

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: wochinge

packages/client/package.json

@@ -35,6 +35,7 @@
   "dependencies": {
     "@langfuse/core": "workspace:^",
     "@langfuse/tracing": "workspace:^",
+    "jsonpath-plus": "^10.3.0",
     "mustache": "^4.2.0"
   },
   "peerDependencies": {

packages/client/src/LangfuseClient.ts

@@ -219,9 +219,9 @@ export class LangfuseClient {
    */
   public getDatasetItem: typeof LangfuseAPIClient.prototype.datasetItems.get;
   /**
-   * @deprecated Use api.datasetItems.create instead
+   * @deprecated Use dataset.createItem instead
    */
-  public createDatasetItem: typeof LangfuseAPIClient.prototype.datasetItems.create;
+  public createDatasetItem: typeof DatasetManager.prototype.createItem;
   /**
    * @deprecated Use api.media.get instead
    */
@@ -312,7 +312,7 @@ export class LangfuseClient {
     this.getDatasetRuns = this.api.datasets.getRuns;
     this.createDataset = this.api.datasets.create;
     this.getDatasetItem = this.api.datasetItems.get;
-    this.createDatasetItem = this.api.datasetItems.create;
+    this.createDatasetItem = this.dataset.createItem.bind(this.dataset);
     this.fetchMedia = this.api.media.get;
     this.resolveMediaReferences = this.media.resolveReferences;
   }

packages/client/src/dataset/dataset.test.ts

@@ -0,0 +1,225 @@
+import { LangfuseMedia, LangfuseMediaReference } from "@langfuse/core";
+import { describe, expect, it, vi } from "vitest";
+
+import { DatasetManager } from "./index.js";
+
+function makeMedia(bytes: number[] = [1, 2, 3, 4]): LangfuseMedia {
+  return new LangfuseMedia({
+    source: "bytes",
+    contentBytes: new Uint8Array(bytes),
+    contentType: "image/png",
+  });
+}
+
+function makeItem(overrides: Record<string, unknown> = {}) {
+  return {
+    id: "item-1",
+    status: "ACTIVE",
+    input: null,
+    expectedOutput: null,
+    metadata: null,
+    sourceTraceId: null,
+    sourceObservationId: null,
+    datasetId: "ds-id",
+    datasetName: "ds",
+    createdAt: "2026-06-16T12:00:00.000Z",
+    updatedAt: "2026-06-16T12:00:00.000Z",
+    ...overrides,
+  };
+}
+
+describe("DatasetManager.createItem media processing", () => {
+  it("uploads media and replaces it with a reference string", async () => {
+    const media = makeMedia();
+    const referenceString = await media.getTag();
+
+    const create = vi.fn().mockResolvedValue({ id: "created" });
+    const uploadMedia = vi.fn().mockResolvedValue(undefined);
+    const manager = new DatasetManager({
+      langfuseClient: {
+        api: { datasetItems: { create } },
+        media: { uploadMedia },
+      } as never,
+    });
+
+    const input = { image: media, question: "q" };
+    const result = await manager.createItem({ datasetName: "ds", input });
+
+    expect(result).toEqual({ id: "created" });
+    expect(uploadMedia).toHaveBeenCalledTimes(1);
+    expect(uploadMedia).toHaveBeenCalledWith(media);
+    expect(create).toHaveBeenCalledWith({
+      datasetName: "ds",
+      input: { image: referenceString, question: "q" },
+      expectedOutput: undefined,
+      metadata: undefined,
+    });
+    // original input is not mutated
+    expect(input.image).toBe(media);
+  });
+
+  it("processes media in expectedOutput and metadata, deduping uploads", async () => {
+    const media = makeMedia();
+    const referenceString = await media.getTag();
+
+    const create = vi.fn().mockResolvedValue({ id: "created" });
+    const uploadMedia = vi.fn().mockResolvedValue(undefined);
+    const manager = new DatasetManager({
+      langfuseClient: {
+        api: { datasetItems: { create } },
+        media: { uploadMedia },
+      } as never,
+    });
+
+    await manager.createItem({
+      datasetName: "ds",
+      input: { a: media },
+      expectedOutput: media,
+      metadata: { nested: [media] },
+    });
+
+    // same media id -> uploaded once
+    expect(uploadMedia).toHaveBeenCalledTimes(1);
+    expect(create).toHaveBeenCalledWith({
+      datasetName: "ds",
+      input: { a: referenceString },
+      expectedOutput: referenceString,
+      metadata: { nested: [referenceString] },
+    });
+  });
+
+  it("throws when media is invalid", async () => {
+    const invalid = new LangfuseMedia({
+      source: "base64_data_uri",
+      base64DataUri: "not-a-data-uri",
+    });
+
+    const manager = new DatasetManager({
+      langfuseClient: {
+        api: { datasetItems: { create: vi.fn() } },
+        media: { uploadMedia: vi.fn() },
+      } as never,
+    });
+
+    await expect(
+      manager.createItem({ datasetName: "ds", input: { image: invalid } }),
+    ).rejects.toThrow(/invalid LangfuseMedia/);
+  });
+});
+
+describe("DatasetManager.get resolveMediaReferences", () => {
+  function managerReturning(item: Record<string, unknown>) {
+    const list = vi
+      .fn()
+      .mockResolvedValue({ data: [item], meta: { totalPages: 1 } });
+    const datasetsGet = vi.fn().mockResolvedValue({ id: "ds-id", name: "ds" });
+    const manager = new DatasetManager({
+      langfuseClient: {
+        api: {
+          datasets: { get: datasetsGet },
+          datasetItems: { list },
+        },
+        experiment: { run: vi.fn() },
+      } as never,
+    });
+    return { manager, list };
+  }
+
+  const mediaPayload = {
+    mediaId: "med-1",
+    contentType: "image/png",
+    contentLength: 4,
+    url: "https://example.com/med.png",
+    urlExpiry: "2026-06-16T13:00:00.000Z",
+  };
+
+  it("hydrates a bracket-notation json path into a LangfuseMediaReference", async () => {
+    const item = makeItem({
+      input: { image: "@@@langfuseMedia:...@@@", question: "q" },
+      mediaReferences: [
+        {
+          field: "input",
+          referenceString: "@@@langfuseMedia:...@@@",
+          jsonPath: "$['image']",
+          media: mediaPayload,
+        },
+      ],
+    });
+    const { manager, list } = managerReturning(item);
+
+    const dataset = await manager.get("ds", { resolveMediaReferences: true });
+
+    expect(list).toHaveBeenCalledWith(
+      expect.objectContaining({ includeMediaReferences: true }),
+    );
+    const input = dataset.items[0].input as Record<string, unknown>;
+    expect(input.image).toBeInstanceOf(LangfuseMediaReference);
+    expect((input.image as LangfuseMediaReference).mediaId).toBe("med-1");
+    expect(input.question).toBe("q");
+  });
+
+  it("hydrates expected_output field and nested json paths", async () => {
+    const item = makeItem({
+      expectedOutput: { a: { b: "@@@langfuseMedia:...@@@" } },
+      mediaReferences: [
+        {
+          field: "expected_output",
+          referenceString: "@@@langfuseMedia:...@@@",
+          jsonPath: "$['a']['b']",
+          media: mediaPayload,
+        },
+      ],
+    });
+    const { manager } = managerReturning(item);
+
+    const dataset = await manager.get("ds", { resolveMediaReferences: true });
+
+    const expectedOutput = dataset.items[0].expectedOutput as {
+      a: { b: unknown };
+    };
+    expect(expectedOutput.a.b).toBeInstanceOf(LangfuseMediaReference);
+  });
+
+  it("does not request or hydrate references when option is off", async () => {
+    const item = makeItem({
+      input: { image: "@@@langfuseMedia:...@@@" },
+      mediaReferences: [
+        {
+          field: "input",
+          referenceString: "@@@langfuseMedia:...@@@",
+          jsonPath: "$['image']",
+          media: mediaPayload,
+        },
+      ],
+    });
+    const { manager, list } = managerReturning(item);
+
+    const dataset = await manager.get("ds");
+
+    expect(list).toHaveBeenCalledWith(
+      expect.not.objectContaining({ includeMediaReferences: true }),
+    );
+    const input = dataset.items[0].input as Record<string, unknown>;
+    expect(input.image).toBe("@@@langfuseMedia:...@@@");
+  });
+
+  it("leaves the field unchanged when media is null", async () => {
+    const item = makeItem({
+      input: { image: "@@@langfuseMedia:...@@@" },
+      mediaReferences: [
+        {
+          field: "input",
+          referenceString: "@@@langfuseMedia:...@@@",
+          jsonPath: "$['image']",
+          media: null,
+        },
+      ],
+    });
+    const { manager } = managerReturning(item);
+
+    const dataset = await manager.get("ds", { resolveMediaReferences: true });
+
+    const input = dataset.items[0].input as Record<string, unknown>;
+    expect(input.image).toBe("@@@langfuseMedia:...@@@");
+  });
+});