[codex] Fix local experiment ID stability (#770)

Author: hassiebp

packages/client/src/experiment/ExperimentManager.ts

@@ -119,6 +119,7 @@ export class ExperimentManager {
    * @param config.maxConcurrency - Maximum number of concurrent task executions (default: 50)
    *
    * @returns Promise that resolves to experiment results including:
+   *   - experimentId: Stable identifier for the experiment execution
    *   - runName: The experiment run name (either provided or generated)
    *   - itemResults: Results for each processed data item
    *   - runEvaluations: Results from run-level evaluators
@@ -193,6 +194,7 @@ export class ExperimentManager {
       name,
       runName: providedRunName,
     });
+    const fallbackExperimentId = await createExperimentId();
 
     if (!this.isOtelRegistered()) {
       this.logger.warn(
@@ -217,6 +219,7 @@ export class ExperimentManager {
           experimentRunName: runName,
           experimentDescription: description,
           experimentMetadata: metadata,
+          fallbackExperimentId,
           datasetVersion: config.datasetVersion,
         });
       });
@@ -244,8 +247,10 @@ export class ExperimentManager {
     }
 
     // Get dataset run URL
-    const datasetRunId =
-      itemResults.length > 0 ? itemResults[0].datasetRunId : undefined;
+    const datasetRunId = itemResults.find(
+      (item) => item.datasetRunId,
+    )?.datasetRunId;
+    const experimentId = datasetRunId || fallbackExperimentId;
 
     let datasetRunUrl = undefined;
     if (datasetRunId && data.length > 0 && "datasetId" in data[0]) {
@@ -294,6 +299,7 @@ export class ExperimentManager {
     await this.langfuseClient.score.flush();
 
     return {
+      experimentId,
       runName,
       itemResults,
       datasetRunId,
@@ -355,6 +361,7 @@ export class ExperimentManager {
       ExpectedOutput,
       Metadata
     >["metadata"];
+    fallbackExperimentId: string;
     item: ExperimentParams<Input, ExpectedOutput, Metadata>["data"][0];
     task: ExperimentTask<Input, ExpectedOutput, Metadata>;
     evaluators?: Evaluator<Input, ExpectedOutput, Metadata>[];
@@ -405,7 +412,7 @@ export class ExperimentManager {
         // Generate IDs
         const experimentItemId =
           datasetItemId || (await createExperimentItemId(input));
-        const experimentId = datasetRunId || (await createExperimentId());
+        const experimentId = datasetRunId || params.fallbackExperimentId;
 
         // Set non-propagated experiment attributes directly on root span
         const rootSpanAttributes: Record<string, string> = {

packages/client/src/experiment/types.ts

@@ -306,6 +306,8 @@ export type ExperimentItemResult<
  * ```typescript
  * const result = await langfuse.experiment.run(config);
  *
+ * console.log(`Experiment ID: ${result.experimentId}`);
+ *
  * // Access individual results
  * console.log(`Processed ${result.itemResults.length} items`);
  *
@@ -332,6 +334,15 @@ export type ExperimentResult<
   ExpectedOutput = any,
   Metadata extends Record<string, any> = Record<string, any>,
 > = {
+  /**
+   * Stable identifier for this experiment execution.
+   *
+   * For Langfuse datasets, this is the dataset run ID when available.
+   * For local data, this is a generated fallback ID shared across all items
+   * in the run.
+   */
+  experimentId: string;
+
   /**
    * The experiment run name.
    *

tests/e2e/experiments.e2e.test.ts

@@ -155,6 +155,7 @@ describe("Langfuse Datasets E2E", () => {
     expect(result.runName).toMatch(
       /^Euro capitals - \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/,
     );
+    expect(result.experimentId).toMatch(/^[0-9a-f]{16}$/);
     // No datasetRunId for local datasets
     expect(result.datasetRunId).toBeUndefined();
 
@@ -233,6 +234,7 @@ describe("Langfuse Datasets E2E", () => {
       /^Euro capitals on LF dataset - \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/,
     );
     expect(result.datasetRunId).toBeDefined();
+    expect(result.experimentId).toBe(result.datasetRunId);
 
     // Validate item results structure
     result.itemResults.forEach((itemResult, index) => {
@@ -333,6 +335,7 @@ describe("Langfuse Datasets E2E", () => {
     // Should use the custom run name exactly
     expect(result.runName).toBe(customRunName);
     expect(result.datasetRunId).toBeDefined();
+    expect(result.experimentId).toBe(result.datasetRunId);
 
     // Fetch dataset run and verify it has the custom name
     const datasetRun = await langfuse.api.datasets.getRun(
@@ -365,6 +368,7 @@ describe("Langfuse Datasets E2E", () => {
 
     // Should use the custom run name exactly
     expect(result.runName).toBe(customRunName);
+    expect(result.experimentId).toMatch(/^[0-9a-f]{16}$/);
     expect(result.itemResults).toHaveLength(2);
     // No dataset run for local datasets
     expect(result.datasetRunId).toBeUndefined();
@@ -483,6 +487,7 @@ describe("Langfuse Datasets E2E", () => {
       await waitForServerIngestion(500);
 
       expect(result.itemResults).toHaveLength(0);
+      expect(result.experimentId).toMatch(/^[0-9a-f]{16}$/);
       expect(result.runEvaluations).toHaveLength(1); // Run evaluators will still execute with empty data
       expect(await result.format()).toContain("No experiment results");
     });

tests/integration/experiment-propagation.integration.test.ts

@@ -251,7 +251,7 @@ describe("Experiment Attribute Propagation", () => {
   });
 
   describe("Multiple Experiment Items", () => {
-    it("should not leak attributes between experiment items", async () => {
+    it("should share experiment ID across local items while keeping item IDs distinct", async () => {
       const items = [
         { input: "item1", metadata: { index: "1" } },
         { input: "item2", metadata: { index: "2" } },
@@ -260,7 +260,7 @@ describe("Experiment Attribute Propagation", () => {
       const experimentIds: string[] = [];
       const itemIds: string[] = [];
 
-      await langfuse.experiment.run({
+      const result = await langfuse.experiment.run({
         name: "no-leakage-test",
         data: items,
         task: async (item) => {
@@ -285,8 +285,9 @@ describe("Experiment Attribute Propagation", () => {
       // Each item should have different item IDs
       expect(itemIds[0]).not.toBe(itemIds[1]);
 
-      // Each item should have different experiment IDs (randomly generated)
-      expect(experimentIds[0]).not.toBe(experimentIds[1]);
+      // All local items should share the same experiment ID
+      expect(experimentIds[0]).toBe(experimentIds[1]);
+      expect(result.experimentId).toBe(experimentIds[0]);
     });
   });