Skip to content

Commit 9c50106

Browse files
committed
feat(analytics): record provider flows as duration spans
- collapse provider turn, tool call, and pr/mr create events into span-based analytics - export duration metadata and span events through otlp - tighten codex turn-completed notifications around active turn ids
1 parent 7ce89ca commit 9c50106

12 files changed

Lines changed: 484 additions & 1556 deletions

File tree

apps/server/src/git/Layers/GitManager.ts

Lines changed: 38 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -502,8 +502,11 @@ export const makeGitManager = Effect.fn("makeGitManager")(function* () {
502502
flush: Effect.void,
503503
}));
504504

505-
const recordGitAnalytics = (event: string, properties: Record<string, unknown>) =>
506-
analytics.record(event, properties);
505+
const recordGitAnalytics = (
506+
event: string,
507+
properties: Record<string, unknown>,
508+
options?: Parameters<typeof analytics.record>[2],
509+
) => analytics.record(event, properties, options);
507510

508511
const resolveJiraTickets = (
509512
threadId: ThreadId | undefined,
@@ -1310,12 +1313,41 @@ export const makeGitManager = Effect.fn("makeGitManager")(function* () {
13101313
label: `Creating ${prOrMr}...`,
13111314
});
13121315
const originRepo = headContext.originRepositoryNameWithOwner;
1313-
yield* recordGitAnalytics("marcode.git.pr_mr.create_requested", {
1316+
const createStartedAtMs = Date.now();
1317+
const baseAnalyticsProperties = {
13141318
"git.host.provider": detectedHostProvider ?? "unknown",
13151319
"git.change_request.kind": changeRequestKind,
13161320
"git.branch": headContext.headBranch,
13171321
...(repositoryName ? { "repository.name": repositoryName } : {}),
1318-
});
1322+
};
1323+
const recordCreateResult = (outcome: "created" | "error", hasUrl: boolean) => {
1324+
const completedAtMs = Date.now();
1325+
const durationMs = Math.max(1, completedAtMs - createStartedAtMs);
1326+
return recordGitAnalytics(
1327+
"marcode.git.pr_mr.create",
1328+
{
1329+
...baseAnalyticsProperties,
1330+
outcome,
1331+
has_url: hasUrl,
1332+
"duration.ms": durationMs,
1333+
},
1334+
{
1335+
durationMs,
1336+
startedAt: createStartedAtMs,
1337+
spanEvents: [
1338+
{
1339+
name: "marcode.git.pr_mr.create.requested",
1340+
at: createStartedAtMs,
1341+
},
1342+
{
1343+
name: "marcode.git.pr_mr.create.completed",
1344+
at: completedAtMs,
1345+
attributes: { outcome, has_url: hasUrl },
1346+
},
1347+
],
1348+
},
1349+
);
1350+
};
13191351
yield* gitHostCli
13201352
.createPullRequest({
13211353
cwd,
@@ -1331,27 +1363,11 @@ export const makeGitManager = Effect.fn("makeGitManager")(function* () {
13311363
schedule: Schedule.exponential(PR_CREATE_RETRY_BASE_DELAY, 2),
13321364
while: isBranchNotReadyError,
13331365
}),
1334-
Effect.tapError(() =>
1335-
recordGitAnalytics("marcode.git.pr_mr.create_completed", {
1336-
"git.host.provider": detectedHostProvider ?? "unknown",
1337-
"git.change_request.kind": changeRequestKind,
1338-
"git.branch": headContext.headBranch,
1339-
outcome: "error",
1340-
has_url: false,
1341-
...(repositoryName ? { "repository.name": repositoryName } : {}),
1342-
}),
1343-
),
1366+
Effect.tapError(() => recordCreateResult("error", false)),
13441367
);
13451368

13461369
const created = yield* findOpenPr(cwd, headContext);
1347-
yield* recordGitAnalytics("marcode.git.pr_mr.create_completed", {
1348-
"git.host.provider": detectedHostProvider ?? "unknown",
1349-
"git.change_request.kind": changeRequestKind,
1350-
"git.branch": headContext.headBranch,
1351-
outcome: "created",
1352-
has_url: created?.url ? true : false,
1353-
...(repositoryName ? { "repository.name": repositoryName } : {}),
1354-
});
1370+
yield* recordCreateResult("created", Boolean(created?.url));
13551371
if (!created) {
13561372
return {
13571373
status: "created" as const,

apps/server/src/provider/Layers/ProviderService.test.ts

Lines changed: 88 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,10 @@ import {
4343
} from "../../persistence/Layers/Sqlite.ts";
4444
import { ServerSettingsService } from "../../serverSettings.ts";
4545
import { AnalyticsServiceNoopLive } from "../../telemetry/Layers/AnalyticsService.ts";
46+
import {
47+
AnalyticsService,
48+
type AnalyticsServiceShape,
49+
} from "../../telemetry/Services/AnalyticsService.ts";
4650

4751
const defaultServerSettingsLayer = ServerSettingsService.layerTest();
4852

@@ -242,6 +246,12 @@ const hasMetricSnapshot = (
242246
Object.entries(attributes).every(([key, value]) => snapshot.attributes?.[key] === value),
243247
);
244248

249+
interface AnalyticsRecord {
250+
readonly event: string;
251+
readonly properties?: Record<string, unknown> | undefined;
252+
readonly options?: Parameters<AnalyticsServiceShape["record"]>[2] | undefined;
253+
}
254+
245255
function makeProviderServiceLayer() {
246256
const codex = makeFakeCodexAdapter();
247257
const claude = makeFakeCodexAdapter("claudeAgent");
@@ -382,7 +392,7 @@ it.effect("ProviderServiceLive writes canonical events to the emitting thread se
382392
state: "completed",
383393
},
384394
});
385-
yield* sleep(20);
395+
yield* sleep(100);
386396
}).pipe(Effect.provide(providerLayer));
387397

388398
assert.equal(canonicalEvents.length, 1);
@@ -391,6 +401,83 @@ it.effect("ProviderServiceLive writes canonical events to the emitting thread se
391401
}).pipe(Effect.provide(NodeServices.layer)),
392402
);
393403

404+
it.effect("ProviderServiceLive records provider turns as one duration product span", () =>
405+
Effect.gen(function* () {
406+
const codex = makeFakeCodexAdapter();
407+
const analyticsRecords: AnalyticsRecord[] = [];
408+
const registry: typeof ProviderAdapterRegistry.Service = {
409+
getByProvider: (provider) =>
410+
provider === "codex"
411+
? Effect.succeed(codex.adapter)
412+
: Effect.fail(new ProviderUnsupportedError({ provider })),
413+
listProviders: () => Effect.succeed(["codex"]),
414+
};
415+
const runtimeRepositoryLayer = ProviderSessionRuntimeRepositoryLive.pipe(
416+
Layer.provide(SqlitePersistenceMemory),
417+
);
418+
const directoryLayer = ProviderSessionDirectoryLive.pipe(Layer.provide(runtimeRepositoryLayer));
419+
const analyticsLayer = Layer.succeed(AnalyticsService, {
420+
record: (event, properties, options) =>
421+
Effect.sync(() => {
422+
analyticsRecords.push({ event, properties, options });
423+
}),
424+
flush: Effect.void,
425+
});
426+
const providerLayer = makeProviderServiceLive().pipe(
427+
Layer.provide(Layer.succeed(ProviderAdapterRegistry, registry)),
428+
Layer.provide(directoryLayer),
429+
Layer.provide(defaultServerSettingsLayer),
430+
Layer.provide(analyticsLayer),
431+
);
432+
433+
yield* Effect.gen(function* () {
434+
const provider = yield* ProviderService;
435+
yield* sleep(10);
436+
const threadId = asThreadId("thread-turn-analytics");
437+
yield* provider.startSession(threadId, {
438+
provider: "codex",
439+
threadId,
440+
runtimeMode: "full-access",
441+
});
442+
const turn = yield* provider.sendTurn({
443+
threadId,
444+
input: "hello",
445+
interactionMode: "default",
446+
});
447+
codex.emit({
448+
eventId: asEventId("evt-turn-analytics-completed"),
449+
provider: "codex",
450+
threadId,
451+
turnId: turn.turnId,
452+
createdAt: new Date(Date.now() + 250).toISOString(),
453+
type: "turn.completed",
454+
payload: {
455+
state: "completed",
456+
},
457+
});
458+
yield* sleep(100);
459+
}).pipe(Effect.provide(providerLayer));
460+
461+
const turnRecord = analyticsRecords.find((record) => record.event === "marcode.provider.turn");
462+
463+
assert.isDefined(turnRecord);
464+
assert.isUndefined(
465+
analyticsRecords.find((record) => record.event === "marcode.provider.turn.sent"),
466+
);
467+
assert.isUndefined(
468+
analyticsRecords.find((record) => record.event === "marcode.provider.turn.completed"),
469+
);
470+
assert.equal(turnRecord?.properties?.provider, "codex");
471+
assert.equal(turnRecord?.properties?.outcome, "success");
472+
assert.equal(turnRecord?.properties?.interaction_mode, "default");
473+
assert.isAtLeast(Number(turnRecord?.options?.durationMs ?? 0), 1);
474+
assert.deepEqual(
475+
turnRecord?.options?.spanEvents?.map((event) => event.name),
476+
["marcode.provider.turn.sent", "marcode.provider.turn.completed"],
477+
);
478+
}).pipe(Effect.provide(NodeServices.layer)),
479+
);
480+
394481
it.effect("ProviderServiceLive keeps persisted resumable sessions on startup", () =>
395482
Effect.gen(function* () {
396483
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "marcode-provider-service-"));

0 commit comments

Comments
 (0)