DataDog
diff --git a/‎index.d.ts‎
Lines changed: 93 additions & 0 deletions b/‎index.d.ts‎
Lines changed: 93 additions & 0 deletions
diff --git a/‎index.d.v5.ts‎
Lines changed: 93 additions & 0 deletions b/‎index.d.v5.ts‎
Lines changed: 93 additions & 0 deletions
diff --git a/‎packages/dd-trace/src/llmobs/experiments/client.js‎
Lines changed: 113 additions & 0 deletions b/‎packages/dd-trace/src/llmobs/experiments/client.js‎
Lines changed: 113 additions & 0 deletions
@@ -3568,6 +3568,12 @@ declare namespace tracer {
        */
       enabled: boolean,
 
+      /**
+       * Datasets & Experiments API. Requires LLM Observability to be enabled and
+       * `DD_API_KEY` / `DD_APP_KEY` to be set.
+       */
+      experiments: Experiments,
+
       /**
        * Enable LLM Observability tracing.
        *
@@ -3721,6 +3727,93 @@ declare namespace tracer {
       flush (): void
     }
 
+    /**
+     * A task run over each dataset record during an experiment.
+     */
+    type ExperimentTask = (input: any, config: Record<string, any>) => any | Promise<any>
+
+    /**
+     * Scores a single task output. The return type selects the metric:
+     * `boolean` -> boolean, `number` -> score, anything else -> categorical.
+     */
+    type ExperimentEvaluator = (input: any, output: any, expectedOutput: any) => any | Promise<any>
+
+    interface ExperimentOptions {
+      name: string
+      dataset: Dataset
+      task: ExperimentTask
+      /** Evaluators keyed by metric label. */
+      evaluators?: Record<string, ExperimentEvaluator>
+      description?: string
+      config?: Record<string, any>
+      tags?: Record<string, string>
+    }
+
+    interface PullDatasetOptions {
+      /** Wait until at least this many records are readable (absorbs write lag). */
+      expectedRecordCount?: number
+      /** Maximum total time to wait, in ms. Default 30000. */
+      maxWaitMs?: number
+    }
+
+    interface ExperimentResultRow {
+      index: number
+      spanId: string
+      traceId: string
+      startNs: number
+      durationNs: number
+      input: any
+      output: any
+      expectedOutput: any
+      readonly isError: boolean
+      errorType: string | null
+      errorMessage: string | null
+      evaluations: Record<string, any>
+      evaluationErrors: Record<string, string>
+    }
+
+    interface ExperimentResult {
+      experimentId: string
+      rows: ExperimentResultRow[]
+      /** Dashboard URL for the experiment. */
+      url: string
+    }
+
+    interface DatasetPushResult {
+      /** Number of records from this push that were confirmed with a record id. */
+      pushedCount: number
+      /** Number of records attempted in this push. */
+      totalCount: number
+    }
+
+    interface Dataset {
+      addRecord (input: any, expectedOutput?: any, metadata?: Record<string, any>): Dataset
+      /** Creates the dataset remotely if needed and pushes any unpushed records. */
+      push (): Promise<DatasetPushResult>
+      name (): string
+      id (): string | null
+      projectId (): string | null
+      records (): Array<{ input: any, expectedOutput: any, metadata: Record<string, any> }>
+      /** Dashboard URL for the dataset, or null until pushed. */
+      url (): string | null
+    }
+
+    interface Experiment {
+      name (): string
+      experimentId (): string | null
+      url (): string | null
+      run (): Promise<ExperimentResult>
+    }
+
+    interface Experiments {
+      /** Create a local dataset buffer; pushed on the first experiment run. */
+      createDataset (name: string, description?: string): Dataset
+      /** Pull an existing dataset (with records) by name. */
+      pullDataset (name: string, options?: PullDatasetOptions): Promise<Dataset>
+      /** Build an experiment to run over a dataset. */
+      experiment (options: ExperimentOptions): Experiment
+    }
+
     interface LLMObservabilitySpan {
       /**
        * The span kind
 
@@ -3774,6 +3774,12 @@ declare namespace tracer {
        */
       enabled: boolean,
 
+      /**
+       * Datasets & Experiments API. Requires LLM Observability to be enabled and
+       * `DD_API_KEY` / `DD_APP_KEY` to be set.
+       */
+      experiments: Experiments,
+
       /**
        * Enable LLM Observability tracing.
        *
@@ -3918,6 +3924,93 @@ declare namespace tracer {
       flush (): void
     }
 
+    /**
+     * A task run over each dataset record during an experiment.
+     */
+    type ExperimentTask = (input: any, config: Record<string, any>) => any | Promise<any>
+
+    /**
+     * Scores a single task output. The return type selects the metric:
+     * `boolean` -> boolean, `number` -> score, anything else -> categorical.
+     */
+    type ExperimentEvaluator = (input: any, output: any, expectedOutput: any) => any | Promise<any>
+
+    interface ExperimentOptions {
+      name: string
+      dataset: Dataset
+      task: ExperimentTask
+      /** Evaluators keyed by metric label. */
+      evaluators?: Record<string, ExperimentEvaluator>
+      description?: string
+      config?: Record<string, any>
+      tags?: Record<string, string>
+    }
+
+    interface PullDatasetOptions {
+      /** Wait until at least this many records are readable (absorbs write lag). */
+      expectedRecordCount?: number
+      /** Maximum total time to wait, in ms. Default 30000. */
+      maxWaitMs?: number
+    }
+
+    interface ExperimentResultRow {
+      index: number
+      spanId: string
+      traceId: string
+      startNs: number
+      durationNs: number
+      input: any
+      output: any
+      expectedOutput: any
+      readonly isError: boolean
+      errorType: string | null
+      errorMessage: string | null
+      evaluations: Record<string, any>
+      evaluationErrors: Record<string, string>
+    }
+
+    interface ExperimentResult {
+      experimentId: string
+      rows: ExperimentResultRow[]
+      /** Dashboard URL for the experiment. */
+      url: string
+    }
+
+    interface DatasetPushResult {
+      /** Number of records from this push that were confirmed with a record id. */
+      pushedCount: number
+      /** Number of records attempted in this push. */
+      totalCount: number
+    }
+
+    interface Dataset {
+      addRecord (input: any, expectedOutput?: any, metadata?: Record<string, any>): Dataset
+      /** Creates the dataset remotely if needed and pushes any unpushed records. */
+      push (): Promise<DatasetPushResult>
+      name (): string
+      id (): string | null
+      projectId (): string | null
+      records (): Array<{ input: any, expectedOutput: any, metadata: Record<string, any> }>
+      /** Dashboard URL for the dataset, or null until pushed. */
+      url (): string | null
+    }
+
+    interface Experiment {
+      name (): string
+      experimentId (): string | null
+      url (): string | null
+      run (): Promise<ExperimentResult>
+    }
+
+    interface Experiments {
+      /** Create a local dataset buffer; pushed on the first experiment run. */
+      createDataset (name: string, description?: string): Dataset
+      /** Pull an existing dataset (with records) by name. */
+      pullDataset (name: string, options?: PullDatasetOptions): Promise<Dataset>
+      /** Build an experiment to run over a dataset. */
+      experiment (options: ExperimentOptions): Experiment
+    }
+
     interface LLMObservabilitySpan {
       /**
        * The span kind
 
@@ -0,0 +1,113 @@
+'use strict'
+
+// Control-plane HTTP client for LLM Obs Experiments. Uses the global `fetch`,
+// so this module adds no new dependency; credentials and site come from config.
+
+const API_BASE_PATH = '/api/v2/llm-obs/v1'
+
+// Control-plane host for a Datadog site, e.g.
+//   datadoghq.com        -> api.datadoghq.com
+//   us3.datadoghq.com    -> api.us3.datadoghq.com
+//   datad0g.com (staging)-> api.datad0g.com
+function apiHost (site) {
+  return `api.${site}`
+}
+
+// Web-app host for dashboard URLs. Single-level sites (datadoghq.com,
+// ddog-gov.com) are served from the `app.` subdomain; regional sites
+// (us3.datadoghq.com, ap1.datadoghq.com) are used as-is.
+function appHost (site) {
+  return site.split('.').length === 2 ? `app.${site}` : site
+}
+
+class ExperimentsClient {
+  #apiKey
+  #appKey
+  #site
+  #projectName
+  #timeout
+  #cachedProjectId
+
+  constructor ({ apiKey, appKey, site, projectName, timeout = 30_000 } = {}) {
+    this.#apiKey = apiKey
+    this.#appKey = appKey
+    this.#site = site
+    this.#projectName = projectName
+    this.#timeout = timeout
+    this.#cachedProjectId = null
+  }
+
+  // Whether the client has everything it needs to talk to the control plane.
+  get configured () {
+    return Boolean(this.#apiKey && this.#appKey && this.#site)
+  }
+
+  get site () {
+    return this.#site
+  }
+
+  // Dashboard URL base for the configured site, e.g. https://app.datadoghq.com
+  get appBase () {
+    return `https://${appHost(this.#site)}`
+  }
+
+  // Resolve the configured project's id (get-or-create), cached.
+  ensureProjectId () {
+    return this.getOrCreateProject(this.#projectName)
+  }
+
+  // Low-level request. Builds https://api.<site><path>, attaches both keys, and
+  // returns the parsed JSON body. Throws with status + body on a non-2xx.
+  async request (method, path, body) {
+    const url = `https://${apiHost(this.#site)}${path}`
+    const headers = {
+      'DD-API-KEY': this.#apiKey,
+      'DD-APPLICATION-KEY': this.#appKey,
+    }
+
+    let payload
+    if (body !== undefined) {
+      payload = JSON.stringify(body)
+      headers['Content-Type'] = 'application/json'
+    }
+
+    let response
+    try {
+      response = await fetch(url, {
+        method,
+        headers,
+        body: payload,
+        signal: AbortSignal.timeout(this.#timeout),
+      })
+    } catch (err) {
+      throw new Error(`${method} ${path} failed: ${err.message}`)
+    }
+
+    const text = await response.text()
+    if (!response.ok) {
+      throw new Error(`${method} ${path} failed: HTTP ${response.status} ${text}`)
+    }
+    return text ? JSON.parse(text) : {}
+  }
+
+  // Resolve the project id for `name`, creating it if absent. The create
+  // endpoint is get-or-create on name, so repeated calls return the same id.
+  // Cached after the first resolution.
+  async getOrCreateProject (name) {
+    if (this.#cachedProjectId) return this.#cachedProjectId
+
+    let response
+    try {
+      response = await this.request('POST', `${API_BASE_PATH}/projects`, {
+        data: { type: 'projects', attributes: { name } },
+      })
+    } catch (err) {
+      throw new Error(`Failed to create or get project '${name}': ${err.message}`)
+    }
+
+    this.#cachedProjectId = response?.data?.id ?? null
+    return this.#cachedProjectId
+  }
+}
+
+module.exports = { ExperimentsClient, apiHost, appHost, API_BASE_PATH }