diff --git a/bun.lock b/bun.lock index 4e4aa7a707d5..c25752ba8622 100644 --- a/bun.lock +++ b/bun.lock @@ -280,6 +280,7 @@ "@opentelemetry/api": "1.9.0", "@opentelemetry/context-async-hooks": "2.6.1", "@opentelemetry/exporter-trace-otlp-http": "0.214.0", + "@opentelemetry/exporter-trace-otlp-proto": "0.214.0", "@opentelemetry/sdk-trace-base": "2.6.1", "@parcel/watcher": "2.5.1", "@silvia-odwyer/photon-node": "0.3.4", @@ -542,6 +543,7 @@ "@opentelemetry/api": "1.9.0", "@opentelemetry/context-async-hooks": "2.6.1", "@opentelemetry/exporter-trace-otlp-http": "0.214.0", + "@opentelemetry/exporter-trace-otlp-proto": "0.214.0", "@opentelemetry/sdk-trace-base": "2.6.1", "@opentelemetry/sdk-trace-node": "2.6.1", "@opentui/core": "catalog:", @@ -1836,6 +1838,8 @@ "@opentelemetry/exporter-trace-otlp-http": ["@opentelemetry/exporter-trace-otlp-http@0.214.0", "", { "dependencies": { "@opentelemetry/core": "2.6.1", "@opentelemetry/otlp-exporter-base": "0.214.0", "@opentelemetry/otlp-transformer": "0.214.0", "@opentelemetry/resources": "2.6.1", "@opentelemetry/sdk-trace-base": "2.6.1" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-kIN8nTBMgV2hXzV/a20BCFilPZdAIMYYJGSgfMMRm/Xa+07y5hRDS2Vm12A/z8Cdu3Sq++ZvJfElokX2rkgGgw=="], + "@opentelemetry/exporter-trace-otlp-proto": ["@opentelemetry/exporter-trace-otlp-proto@0.214.0", "", { "dependencies": { "@opentelemetry/core": "2.6.1", "@opentelemetry/otlp-exporter-base": "0.214.0", "@opentelemetry/otlp-transformer": "0.214.0", "@opentelemetry/resources": "2.6.1", "@opentelemetry/sdk-trace-base": "2.6.1" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-ON0spYWb2yAdQ9b+ItNyK0c6qdtcs+0eVR4YFJkhJL7agfT8sHFg0e5EesauSRiTHPZHiDobI92k77q0lwAmqg=="], + "@opentelemetry/otlp-exporter-base": ["@opentelemetry/otlp-exporter-base@0.214.0", "", { "dependencies": { "@opentelemetry/core": "2.6.1", "@opentelemetry/otlp-transformer": "0.214.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-u1Gdv0/E9wP+apqWf7Wv2npXmgJtxsW2XL0TEv9FZloTZRuMBKmu8cYVXwS4Hm3q/f/3FuCnPTgiwYvIqRSpRg=="], "@opentelemetry/otlp-transformer": ["@opentelemetry/otlp-transformer@0.214.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.214.0", "@opentelemetry/core": "2.6.1", "@opentelemetry/resources": "2.6.1", "@opentelemetry/sdk-logs": "0.214.0", "@opentelemetry/sdk-metrics": "2.6.1", "@opentelemetry/sdk-trace-base": "2.6.1", "protobufjs": "^7.0.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-DSaYcuBRh6uozfsWN3R8HsN0yDhCuWP7tOFdkUOVaWD1KVJg8m4qiLUsg/tNhTLS9HUYUcwNpwL2eroLtsZZ/w=="], diff --git a/packages/core/package.json b/packages/core/package.json index 26449b5165b4..f6885e6124bd 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -94,6 +94,7 @@ "@opentelemetry/api": "1.9.0", "@opentelemetry/context-async-hooks": "2.6.1", "@opentelemetry/exporter-trace-otlp-http": "0.214.0", + "@opentelemetry/exporter-trace-otlp-proto": "0.214.0", "@opentelemetry/sdk-trace-base": "2.6.1", "@parcel/watcher": "2.5.1", "@silvia-odwyer/photon-node": "0.3.4", diff --git a/packages/core/src/flag/flag.ts b/packages/core/src/flag/flag.ts index a0eb78a13e2a..76b2980d0998 100644 --- a/packages/core/src/flag/flag.ts +++ b/packages/core/src/flag/flag.ts @@ -16,6 +16,18 @@ export const Flag = { OTEL_EXPORTER_OTLP_ENDPOINT: process.env["OTEL_EXPORTER_OTLP_ENDPOINT"], OTEL_EXPORTER_OTLP_HEADERS: process.env["OTEL_EXPORTER_OTLP_HEADERS"], + // Evaluated at access time because tests and embedding runtimes can set OTLP + // protocol env vars after this module has loaded. + get OTEL_EXPORTER_OTLP_PROTOCOL() { + return process.env["OTEL_EXPORTER_OTLP_PROTOCOL"] + }, + get OTEL_EXPORTER_OTLP_TRACES_PROTOCOL() { + return process.env["OTEL_EXPORTER_OTLP_TRACES_PROTOCOL"] + }, + get OTEL_EXPORTER_OTLP_LOGS_PROTOCOL() { + return process.env["OTEL_EXPORTER_OTLP_LOGS_PROTOCOL"] + }, + OPENCODE_AUTO_HEAP_SNAPSHOT: truthy("OPENCODE_AUTO_HEAP_SNAPSHOT"), OPENCODE_GIT_BASH_PATH: process.env["OPENCODE_GIT_BASH_PATH"], OPENCODE_CONFIG: process.env["OPENCODE_CONFIG"], diff --git a/packages/core/src/observability.ts b/packages/core/src/observability.ts index faffb2733333..f1949513d5dd 100644 --- a/packages/core/src/observability.ts +++ b/packages/core/src/observability.ts @@ -3,7 +3,6 @@ export * as Observability from "./observability" import { NodeFileSystem } from "@effect/platform-node" import { Effect, Layer, Logger, References } from "effect" import { FetchHttpClient } from "effect/unstable/http" -import { OtlpSerialization } from "effect/unstable/observability" import { Logging } from "./observability/logging" import { Otlp } from "./observability/otlp" @@ -11,7 +10,7 @@ export const layer = Layer.unwrap( Effect.gen(function* () { const logs = Logger.layer([...Logging.loggers(), ...Otlp.loggers()], { mergeWithExisting: false }).pipe( Layer.provide(NodeFileSystem.layer), - Layer.provide(OtlpSerialization.layerJson), + Layer.provide(Otlp.serializationLayer()), Layer.provide(FetchHttpClient.layer), Layer.orDie, Layer.merge(Layer.succeed(References.MinimumLogLevel, Logging.minimumLogLevel())), diff --git a/packages/core/src/observability/otlp.ts b/packages/core/src/observability/otlp.ts index dd99ebc1436b..0401f764e898 100644 --- a/packages/core/src/observability/otlp.ts +++ b/packages/core/src/observability/otlp.ts @@ -1,11 +1,16 @@ import { Layer } from "effect" -import { OtlpLogger } from "effect/unstable/observability" +import { OtlpLogger, OtlpSerialization } from "effect/unstable/observability" import { Flag } from "../flag/flag" import { InstallationChannel, InstallationVersion } from "../installation/version" import { runID } from "./shared" const endpoint = Flag.OTEL_EXPORTER_OTLP_ENDPOINT +export type OtlpProtocol = "http/json" | "http/protobuf" +export type OtlpSignal = "logs" | "traces" + +const defaultProtocol: OtlpProtocol = "http/json" + const headers = Flag.OTEL_EXPORTER_OTLP_HEADERS ? Flag.OTEL_EXPORTER_OTLP_HEADERS.split(",").reduce( (acc, entry) => { @@ -17,6 +22,24 @@ const headers = Flag.OTEL_EXPORTER_OTLP_HEADERS ) : undefined +function protocolValue(signal: OtlpSignal) { + return ( + (signal === "logs" ? Flag.OTEL_EXPORTER_OTLP_LOGS_PROTOCOL : Flag.OTEL_EXPORTER_OTLP_TRACES_PROTOCOL) ?? + Flag.OTEL_EXPORTER_OTLP_PROTOCOL + ) +} + +export function protocol(signal: OtlpSignal): OtlpProtocol { + const value = protocolValue(signal) + if (value === "http/json") return value + if (value === "http/protobuf") return value + return defaultProtocol +} + +export function serializationLayer() { + return protocol("logs") === "http/protobuf" ? OtlpSerialization.layerProtobuf : OtlpSerialization.layerJson +} + function resourceAttributes() { const value = process.env.OTEL_RESOURCE_ATTRIBUTES if (!value) return {} @@ -55,7 +78,10 @@ export function loggers() { export async function tracingLayer() { if (!endpoint) return Layer.empty const NodeSdk = await import("@effect/opentelemetry/NodeSdk") - const OTLP = await import("@opentelemetry/exporter-trace-otlp-http") + const OTLP = + protocol("traces") === "http/protobuf" + ? await import("@opentelemetry/exporter-trace-otlp-proto") + : await import("@opentelemetry/exporter-trace-otlp-http") const SdkBase = await import("@opentelemetry/sdk-trace-base") const { AsyncLocalStorageContextManager } = await import("@opentelemetry/context-async-hooks") const { context } = await import("@opentelemetry/api") diff --git a/packages/core/test/effect/observability.test.ts b/packages/core/test/effect/observability.test.ts index 4758563f287b..3e5efb579f01 100644 --- a/packages/core/test/effect/observability.test.ts +++ b/packages/core/test/effect/observability.test.ts @@ -1,23 +1,73 @@ import { afterEach, describe, expect, test } from "bun:test" import { NodeFileSystem } from "@effect/platform-node" import { Effect, Layer, Logger } from "effect" +import { OtlpSerialization } from "effect/unstable/observability" import fs from "fs/promises" import os from "os" import path from "path" import { fileLogger } from "../../src/observability/logging" -import { resource } from "../../src/observability/otlp" +import { protocol, resource, serializationLayer } from "../../src/observability/otlp" const otelResourceAttributes = process.env.OTEL_RESOURCE_ATTRIBUTES +const otelProtocol = process.env.OTEL_EXPORTER_OTLP_PROTOCOL +const otelTracesProtocol = process.env.OTEL_EXPORTER_OTLP_TRACES_PROTOCOL +const otelLogsProtocol = process.env.OTEL_EXPORTER_OTLP_LOGS_PROTOCOL const opencodeClient = process.env.OPENCODE_CLIENT afterEach(() => { if (otelResourceAttributes === undefined) delete process.env.OTEL_RESOURCE_ATTRIBUTES else process.env.OTEL_RESOURCE_ATTRIBUTES = otelResourceAttributes + if (otelProtocol === undefined) delete process.env.OTEL_EXPORTER_OTLP_PROTOCOL + else process.env.OTEL_EXPORTER_OTLP_PROTOCOL = otelProtocol + if (otelTracesProtocol === undefined) delete process.env.OTEL_EXPORTER_OTLP_TRACES_PROTOCOL + else process.env.OTEL_EXPORTER_OTLP_TRACES_PROTOCOL = otelTracesProtocol + if (otelLogsProtocol === undefined) delete process.env.OTEL_EXPORTER_OTLP_LOGS_PROTOCOL + else process.env.OTEL_EXPORTER_OTLP_LOGS_PROTOCOL = otelLogsProtocol if (opencodeClient === undefined) delete process.env.OPENCODE_CLIENT else process.env.OPENCODE_CLIENT = opencodeClient }) +describe("protocol", () => { + test("defaults to json", () => { + delete process.env.OTEL_EXPORTER_OTLP_PROTOCOL + delete process.env.OTEL_EXPORTER_OTLP_TRACES_PROTOCOL + delete process.env.OTEL_EXPORTER_OTLP_LOGS_PROTOCOL + + expect(protocol("logs")).toBe("http/json") + expect(protocol("traces")).toBe("http/json") + expect(serializationLayer()).toBe(OtlpSerialization.layerJson) + }) + + test("uses general protobuf protocol for logs and traces", () => { + process.env.OTEL_EXPORTER_OTLP_PROTOCOL = "http/protobuf" + delete process.env.OTEL_EXPORTER_OTLP_TRACES_PROTOCOL + delete process.env.OTEL_EXPORTER_OTLP_LOGS_PROTOCOL + + expect(protocol("logs")).toBe("http/protobuf") + expect(protocol("traces")).toBe("http/protobuf") + expect(serializationLayer()).toBe(OtlpSerialization.layerProtobuf) + }) + + test("signal-specific protocol overrides the general protocol", () => { + process.env.OTEL_EXPORTER_OTLP_PROTOCOL = "http/protobuf" + process.env.OTEL_EXPORTER_OTLP_TRACES_PROTOCOL = "http/json" + process.env.OTEL_EXPORTER_OTLP_LOGS_PROTOCOL = "http/protobuf" + + expect(protocol("logs")).toBe("http/protobuf") + expect(protocol("traces")).toBe("http/json") + expect(serializationLayer()).toBe(OtlpSerialization.layerProtobuf) + }) + + test("unsupported protocol falls back to json", () => { + process.env.OTEL_EXPORTER_OTLP_PROTOCOL = "grpc" + delete process.env.OTEL_EXPORTER_OTLP_TRACES_PROTOCOL + delete process.env.OTEL_EXPORTER_OTLP_LOGS_PROTOCOL + + expect(protocol("logs")).toBe("http/json") + }) +}) + describe("resource", () => { test("parses and decodes OTEL resource attributes", () => { process.env.OTEL_RESOURCE_ATTRIBUTES = diff --git a/packages/opencode/package.json b/packages/opencode/package.json index fe6e8a8ff1b8..0c1b8c7cf7ad 100644 --- a/packages/opencode/package.json +++ b/packages/opencode/package.json @@ -94,6 +94,7 @@ "@opentelemetry/api": "1.9.0", "@opentelemetry/context-async-hooks": "2.6.1", "@opentelemetry/exporter-trace-otlp-http": "0.214.0", + "@opentelemetry/exporter-trace-otlp-proto": "0.214.0", "@opentelemetry/sdk-trace-base": "2.6.1", "@opentelemetry/sdk-trace-node": "2.6.1", "@opentui/core": "catalog:", diff --git a/packages/opencode/src/control-plane/workspace.ts b/packages/opencode/src/control-plane/workspace.ts index 76aab6ef61ab..095723c3c5b6 100644 --- a/packages/opencode/src/control-plane/workspace.ts +++ b/packages/opencode/src/control-plane/workspace.ts @@ -548,6 +548,9 @@ export const layer = Layer.effect( OPENCODE_EXPERIMENTAL_WORKSPACES: "true", OTEL_EXPORTER_OTLP_HEADERS: process.env.OTEL_EXPORTER_OTLP_HEADERS, OTEL_EXPORTER_OTLP_ENDPOINT: process.env.OTEL_EXPORTER_OTLP_ENDPOINT, + OTEL_EXPORTER_OTLP_PROTOCOL: process.env.OTEL_EXPORTER_OTLP_PROTOCOL, + OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: process.env.OTEL_EXPORTER_OTLP_TRACES_PROTOCOL, + OTEL_EXPORTER_OTLP_LOGS_PROTOCOL: process.env.OTEL_EXPORTER_OTLP_LOGS_PROTOCOL, OTEL_RESOURCE_ATTRIBUTES: process.env.OTEL_RESOURCE_ATTRIBUTES, } diff --git a/packages/opencode/test/control-plane/workspace.test.ts b/packages/opencode/test/control-plane/workspace.test.ts index 0b680bc91ad6..cf1f4e012289 100644 --- a/packages/opencode/test/control-plane/workspace.test.ts +++ b/packages/opencode/test/control-plane/workspace.test.ts @@ -40,6 +40,9 @@ const originalEnv = { OPENCODE_EXPERIMENTAL_WORKSPACES: process.env.OPENCODE_EXPERIMENTAL_WORKSPACES, OTEL_EXPORTER_OTLP_HEADERS: process.env.OTEL_EXPORTER_OTLP_HEADERS, OTEL_EXPORTER_OTLP_ENDPOINT: process.env.OTEL_EXPORTER_OTLP_ENDPOINT, + OTEL_EXPORTER_OTLP_PROTOCOL: process.env.OTEL_EXPORTER_OTLP_PROTOCOL, + OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: process.env.OTEL_EXPORTER_OTLP_TRACES_PROTOCOL, + OTEL_EXPORTER_OTLP_LOGS_PROTOCOL: process.env.OTEL_EXPORTER_OTLP_LOGS_PROTOCOL, OTEL_RESOURCE_ATTRIBUTES: process.env.OTEL_RESOURCE_ATTRIBUTES, } @@ -428,6 +431,9 @@ describe("workspace CRUD", () => { process.env.OPENCODE_AUTH_CONTENT = JSON.stringify({ test: { type: "api", key: "secret" } }) process.env.OTEL_EXPORTER_OTLP_HEADERS = "authorization=otel" process.env.OTEL_EXPORTER_OTLP_ENDPOINT = "https://otel.test" + process.env.OTEL_EXPORTER_OTLP_PROTOCOL = "http/protobuf" + process.env.OTEL_EXPORTER_OTLP_TRACES_PROTOCOL = "http/json" + process.env.OTEL_EXPORTER_OTLP_LOGS_PROTOCOL = "http/protobuf" process.env.OTEL_RESOURCE_ATTRIBUTES = "service.name=opencode-test" const workspaceID = WorkspaceV2.ID.ascending("wrk_create_local") @@ -491,6 +497,9 @@ describe("workspace CRUD", () => { expect(recorded.calls.create[0].env.OPENCODE_EXPERIMENTAL_WORKSPACES).toBe("true") expect(recorded.calls.create[0].env.OTEL_EXPORTER_OTLP_HEADERS).toBe("authorization=otel") expect(recorded.calls.create[0].env.OTEL_EXPORTER_OTLP_ENDPOINT).toBe("https://otel.test") + expect(recorded.calls.create[0].env.OTEL_EXPORTER_OTLP_PROTOCOL).toBe("http/protobuf") + expect(recorded.calls.create[0].env.OTEL_EXPORTER_OTLP_TRACES_PROTOCOL).toBe("http/json") + expect(recorded.calls.create[0].env.OTEL_EXPORTER_OTLP_LOGS_PROTOCOL).toBe("http/protobuf") expect(recorded.calls.create[0].env.OTEL_RESOURCE_ATTRIBUTES).toBe("service.name=opencode-test") expect((yield* workspace.status()).find((item) => item.workspaceID === workspaceID)?.status).toBe("connected")