Skip to content

Commit 9eebd54

Browse files
pamelachiajgoux
pamelachia
and
jgoux
authored
chore(cli): track output_format and fix is_agent CI parity (#5513)
## Summary Adds an `output_format` property to the `cli_command_executed` telemetry event and fixes a Go/TS divergence in how `is_agent` is computed. Together these make the JSON-vs-human output choice measurable and ensure both CLI shells agree on what counts as an agent. This is the measurement-and-parity groundwork for auto-switching agents to JSON output (next subissue under GROWTH-806). ## Changes - **Record the resolved output format on every `cli_command_executed`.** Added `output_format` to the Go catalog (`events.go`) and its TS mirror (`event-catalog.ts`), and emit it from the Go capture and both TS instrumentation paths. `db query` resolves its format into a command-local flag, so it now mirrors the resolved value onto the global that telemetry reads, keeping the value accurate for the highest-volume agent command. The property does not exist today, so current JSON usage is unmeasurable without it. - **Fix `is_agent` Go/TS parity.** The TS legacy analytics layer folded CI into `is_agent` (`is_agent = aiTool || isCi`), tagging CI environments as agents while the Go binary does not. `is_agent` now reflects coding-agent detection only, matching Go on both shells. `is_ci` continues to be reported as its own separate property. ## Notes for reviewers - `output_format` values are shell-native (Go: `pretty|json|yaml|toml|env`; TS: `text|json|stream-json`), disambiguated by `$lib`; normalize in analysis. - Deliberately out of scope: aligning the CI env-var lists across the two shells. `runtime.isCi` gates behavior in `legacy-platform-api.layer.ts`, so changing those lists would alter non-interactive behavior for some CI environments. That belongs in a separate change, not this telemetry-parity fix. - The three instrumentation/platform-api unit tests now provide a mock `Output` because the instrumentation reads `output.format`. ## Linear - fixes GROWTH-912 - parent GROWTH-806 --------- Co-authored-by: Julien Goux <hi@jgoux.dev>
1 parent ff3482b commit 9eebd54

10 files changed

Lines changed: 133 additions & 10 deletions

File tree

‎apps/cli-go/cmd/db.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -308,6 +308,9 @@ without the envelope.`,
308308
outputFormat = "table"
309309
}
310310
}
311+
// db query resolves --output into a command-local flag, so mirror the
312+
// resolved value onto the global that telemetry's output_format reads.
313+
utils.OutputFormat.Value = outputFormat
311314
if flag := cmd.Flags().Lookup("linked"); flag != nil && flag.Changed {
312315
return query.RunLinked(cmd.Context(), sql, flags.ProjectRef, outputFormat, agentMode, os.Stdout)
313316
}

‎apps/cli-go/cmd/root.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -175,8 +175,9 @@ func Execute() {
175175
if service := telemetry.FromContext(executedCmd.Context()); service != nil {
176176
ensureProjectGroupsCached(executedCmd.Context(), service)
177177
_ = service.Capture(executedCmd.Context(), telemetry.EventCommandExecuted, map[string]any{
178-
telemetry.PropExitCode: exitCode(err),
179-
telemetry.PropDurationMs: time.Since(startedAt).Milliseconds(),
178+
telemetry.PropExitCode: exitCode(err),
179+
telemetry.PropDurationMs: time.Since(startedAt).Milliseconds(),
180+
telemetry.PropOutputFormat: utils.OutputFormat.Value,
180181
}, nil)
181182
_ = service.Close()
182183
}

‎apps/cli-go/internal/telemetry/events.go‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,10 @@ import "context"
1111
const (
1212
// - EventCommandExecuted: sent after a CLI command finishes, whether it
1313
// succeeds or fails. This helps measure command usage, failure rates, and
14-
// runtime. Event-specific properties are PropExitCode (process exit code)
15-
// and PropDurationMs (command runtime in milliseconds). Related groups:
16-
// none added directly by this event.
14+
// runtime. Event-specific properties are PropExitCode (process exit code),
15+
// PropDurationMs (command runtime in milliseconds), and PropOutputFormat
16+
// (the resolved output format the command used). Related groups: none
17+
// added directly by this event.
1718
EventCommandExecuted = "cli_command_executed"
1819
// - EventProjectLinked: sent after the local CLI directory is linked to a
1920
// Supabase project. This helps measure project-linking adoption and connect
@@ -109,6 +110,10 @@ const (
109110
PropExitCode = "exit_code"
110111
// PropDurationMs is the command runtime in milliseconds.
111112
PropDurationMs = "duration_ms"
113+
// PropOutputFormat is the resolved output format the command used (for
114+
// example "pretty", "json", "yaml"), after agent auto-detection and any
115+
// command-local override are applied.
116+
PropOutputFormat = "output_format"
112117
)
113118

114119
// Group identifiers associate events with higher-level entities in PostHog.

‎apps/cli/src/legacy/telemetry/legacy-analytics.layer.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,7 @@ export const legacyAnalyticsLayer = Layer.effect(
161161

162162
const loadLinkedProject = makeLoadLinkedProject(fs, path);
163163

164-
const isAgent = Option.isSome(aiTool.name) || runtime.isCi;
164+
const isAgent = Option.isSome(aiTool.name);
165165
const envSignals = collectEnvSignals();
166166

167167
const baseProperties = stripUndefined({

‎apps/cli/src/legacy/telemetry/legacy-command-instrumentation.ts‎

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,14 @@ import {
44
getCommandRuntimeCommand,
55
getCommandRuntimeSpanName,
66
} from "../../shared/runtime/command-runtime.service.ts";
7+
import { Output } from "../../shared/output/output.service.ts";
78
import { withAnalyticsContext } from "../../shared/telemetry/analytics-context.ts";
89
import { Analytics } from "../../shared/telemetry/analytics.service.ts";
910
import {
1011
EventCommandExecuted,
1112
PropDurationMs,
1213
PropExitCode,
14+
PropOutputFormat,
1315
} from "../../shared/telemetry/event-catalog.ts";
1416

1517
interface LegacyCommandInstrumentationOptions<Flags extends Record<string, unknown> = never> {
@@ -22,11 +24,48 @@ interface LegacyCommandInstrumentationOptions<Flags extends Record<string, unkno
2224
}
2325

2426
const REDACTED_VALUE = "<redacted>";
27+
const LEGACY_GO_MACHINE_OUTPUT_FORMATS = new Set(["env", "json", "toml", "yaml"]);
28+
const LEGACY_GO_OUTPUT_FORMATS = new Set([...LEGACY_GO_MACHINE_OUTPUT_FORMATS, "pretty"]);
2529

2630
function toCliFlagName(key: string): string {
2731
return key.replace(/[A-Z]/g, (char) => `-${char.toLowerCase()}`);
2832
}
2933

34+
function extractLegacyGoOutputFormat(args: ReadonlyArray<string>): string | undefined {
35+
let format: string | undefined;
36+
37+
for (let index = 0; index < args.length; index++) {
38+
const arg = args[index];
39+
if (arg === undefined) continue;
40+
41+
if (arg === "--output" || arg === "-o") {
42+
const value = args[index + 1];
43+
if (value !== undefined && LEGACY_GO_OUTPUT_FORMATS.has(value)) {
44+
format = value;
45+
}
46+
index++;
47+
continue;
48+
}
49+
50+
if (arg.startsWith("--output=") || arg.startsWith("-o=")) {
51+
const value = arg.slice(arg.indexOf("=") + 1);
52+
if (LEGACY_GO_OUTPUT_FORMATS.has(value)) {
53+
format = value;
54+
}
55+
}
56+
}
57+
58+
return format;
59+
}
60+
61+
function resolveOutputFormatForTelemetry(args: ReadonlyArray<string>, outputFormat: string) {
62+
const goOutputFormat = extractLegacyGoOutputFormat(args);
63+
if (goOutputFormat !== undefined && LEGACY_GO_MACHINE_OUTPUT_FORMATS.has(goOutputFormat)) {
64+
return goOutputFormat;
65+
}
66+
return outputFormat;
67+
}
68+
3069
function extractChangedFlagNames(args: ReadonlyArray<string>): ReadonlyArray<string> {
3170
const used = new Set<string>();
3271

@@ -114,6 +153,7 @@ function withLegacyCommandAnalyticsImplementation<Flags extends Record<string, u
114153
});
115154

116155
const analytics = yield* Analytics;
156+
const output = yield* Output;
117157
const stdio = yield* Stdio.Stdio;
118158
const args = yield* stdio.args;
119159
const startedAt = yield* Clock.currentTimeMillis;
@@ -132,6 +172,7 @@ function withLegacyCommandAnalyticsImplementation<Flags extends Record<string, u
132172
.capture(EventCommandExecuted, {
133173
[PropExitCode]: Exit.isSuccess(exit) ? 0 : 1,
134174
[PropDurationMs]: finishedAt - startedAt,
175+
[PropOutputFormat]: resolveOutputFormatForTelemetry(args, output.format),
135176
})
136177
.pipe(withAnalyticsContext(analyticsContext));
137178

@@ -145,12 +186,12 @@ function withLegacyCommandAnalyticsImplementation<Flags extends Record<string, u
145186

146187
export function withLegacyCommandInstrumentation(): <A, E, R>(
147188
self: Effect.Effect<A, E, R>,
148-
) => Effect.Effect<A, E, R | Analytics | CommandRuntime | Stdio.Stdio>;
189+
) => Effect.Effect<A, E, R | Analytics | CommandRuntime | Stdio.Stdio | Output>;
149190
export function withLegacyCommandInstrumentation<Flags extends Record<string, unknown>>(
150191
options: LegacyCommandInstrumentationOptions<Flags>,
151192
): <A, E, R>(
152193
self: Effect.Effect<A, E, R>,
153-
) => Effect.Effect<A, E, R | Analytics | CommandRuntime | Stdio.Stdio>;
194+
) => Effect.Effect<A, E, R | Analytics | CommandRuntime | Stdio.Stdio | Output>;
154195
export function withLegacyCommandInstrumentation<Flags extends Record<string, unknown>>(
155196
options?: LegacyCommandInstrumentationOptions<Flags>,
156197
) {

‎apps/cli/src/legacy/telemetry/legacy-command-instrumentation.unit.test.ts‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { commandRuntimeLayer } from "../../shared/runtime/command-runtime.layer.
44
import { CurrentAnalyticsContext } from "../../shared/telemetry/analytics-context.ts";
55
import { Analytics } from "../../shared/telemetry/analytics.service.ts";
66
import { withLegacyCommandInstrumentation } from "./legacy-command-instrumentation.ts";
7+
import { mockOutput } from "../../../tests/helpers/mocks.ts";
78

89
function mockContextualAnalytics() {
910
const captured: Array<{
@@ -46,6 +47,7 @@ describe("withLegacyCommandInstrumentation", () => {
4647
}).pipe(
4748
withLegacyCommandInstrumentation(),
4849
Effect.provide(analytics.layer),
50+
Effect.provide(mockOutput({ format: "text" }).layer),
4951
Effect.provide(
5052
Stdio.layerTest({
5153
args: Effect.succeed(["backups", "list"]),
@@ -60,6 +62,56 @@ describe("withLegacyCommandInstrumentation", () => {
6062
expect(event?.properties.command).toBe("backups list");
6163
expect(event?.properties.exit_code).toBe(0);
6264
expect(typeof event?.properties.duration_ms).toBe("number");
65+
expect(event?.properties.output_format).toBe("text");
66+
}),
67+
),
68+
);
69+
});
70+
71+
it.live("reports legacy Go machine output formats emitted through the text layer", () => {
72+
const analytics = mockContextualAnalytics();
73+
74+
return Effect.void.pipe(
75+
withLegacyCommandInstrumentation(),
76+
Effect.provide(analytics.layer),
77+
Effect.provide(mockOutput({ format: "text" }).layer),
78+
Effect.provide(
79+
Stdio.layerTest({
80+
args: Effect.succeed(["backups", "list", "--output", "yaml"]),
81+
}),
82+
),
83+
Effect.provide(commandRuntimeLayer(["backups", "list"])),
84+
Effect.tap(() =>
85+
Effect.sync(() => {
86+
expect(analytics.captured[0]?.properties.output_format).toBe("yaml");
87+
}),
88+
),
89+
);
90+
});
91+
92+
it.live("keeps the TS output format when legacy --output pretty defers to it", () => {
93+
const analytics = mockContextualAnalytics();
94+
95+
return Effect.void.pipe(
96+
withLegacyCommandInstrumentation(),
97+
Effect.provide(analytics.layer),
98+
Effect.provide(mockOutput({ format: "json" }).layer),
99+
Effect.provide(
100+
Stdio.layerTest({
101+
args: Effect.succeed([
102+
"backups",
103+
"list",
104+
"--output",
105+
"pretty",
106+
"--output-format",
107+
"json",
108+
]),
109+
}),
110+
),
111+
Effect.provide(commandRuntimeLayer(["backups", "list"])),
112+
Effect.tap(() =>
113+
Effect.sync(() => {
114+
expect(analytics.captured[0]?.properties.output_format).toBe("json");
63115
}),
64116
),
65117
);
@@ -73,6 +125,7 @@ describe("withLegacyCommandInstrumentation", () => {
73125
flags: { projectRef: Option.some("abcdefghijklmnopqrst") },
74126
}),
75127
Effect.provide(analytics.layer),
128+
Effect.provide(mockOutput({ format: "text" }).layer),
76129
Effect.provide(
77130
Stdio.layerTest({
78131
args: Effect.succeed(["secrets", "list", "--project-ref", "abcdefghijklmnopqrst"]),
@@ -99,6 +152,7 @@ describe("withLegacyCommandInstrumentation", () => {
99152
flags: { envFile: Option.some("/path/to/.env") },
100153
}),
101154
Effect.provide(analytics.layer),
155+
Effect.provide(mockOutput({ format: "text" }).layer),
102156
Effect.provide(
103157
Stdio.layerTest({
104158
args: Effect.succeed(["secrets", "set", "--env-file=/path/to/.env"]),
@@ -125,6 +179,7 @@ describe("withLegacyCommandInstrumentation", () => {
125179
},
126180
}),
127181
Effect.provide(analytics.layer),
182+
Effect.provide(mockOutput({ format: "text" }).layer),
128183
Effect.provide(
129184
Stdio.layerTest({
130185
args: Effect.succeed([
@@ -157,6 +212,7 @@ describe("withLegacyCommandInstrumentation", () => {
157212
safeFlags: ["project-ref"],
158213
}),
159214
Effect.provide(analytics.layer),
215+
Effect.provide(mockOutput({ format: "text" }).layer),
160216
Effect.provide(
161217
Stdio.layerTest({
162218
args: Effect.succeed(["link", "--project-ref", "abcdefghijklmnopqrst"]),
@@ -180,6 +236,7 @@ describe("withLegacyCommandInstrumentation", () => {
180236
return Effect.void.pipe(
181237
withLegacyCommandInstrumentation({ flags: {} }),
182238
Effect.provide(analytics.layer),
239+
Effect.provide(mockOutput({ format: "text" }).layer),
183240
Effect.provide(Stdio.layerTest({ args: Effect.succeed(["backups", "list"]) })),
184241
Effect.provide(commandRuntimeLayer(["backups", "list"])),
185242
Effect.tap(() =>
@@ -196,6 +253,7 @@ describe("withLegacyCommandInstrumentation", () => {
196253

197254
return withLegacyCommandInstrumentation()(Effect.fail(new Error("boom"))).pipe(
198255
Effect.provide(analytics.layer),
256+
Effect.provide(mockOutput({ format: "text" }).layer),
199257
Effect.provide(Stdio.layerTest({ args: Effect.succeed(["backups", "list"]) })),
200258
Effect.provide(commandRuntimeLayer(["backups", "list"])),
201259
Effect.exit,
@@ -215,6 +273,7 @@ describe("withLegacyCommandInstrumentation", () => {
215273
return Effect.sync(() => "ok").pipe(
216274
withLegacyCommandInstrumentation({ analytics: false }),
217275
Effect.provide(analytics.layer),
276+
Effect.provide(mockOutput({ format: "text" }).layer),
218277
Effect.provide(Stdio.layerTest({ args: Effect.succeed(["telemetry", "enable"]) })),
219278
Effect.provide(commandRuntimeLayer(["telemetry", "enable"])),
220279
Effect.tap(() =>
@@ -236,6 +295,7 @@ describe("withLegacyCommandInstrumentation", () => {
236295
},
237296
}),
238297
Effect.provide(analytics.layer),
298+
Effect.provide(mockOutput({ format: "text" }).layer),
239299
Effect.provide(
240300
Stdio.layerTest({
241301
args: Effect.succeed([

‎apps/cli/src/next/auth/platform-api.layer.unit.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import { withCommandInstrumentation } from "../../shared/telemetry/command-instr
1313
import { Credentials } from "./credentials.service.ts";
1414
import { PlatformApi } from "./platform-api.service.ts";
1515
import { makePlatformApiServices } from "./platform-api.layer.ts";
16+
import { mockOutput } from "../../../tests/helpers/mocks.ts";
1617

1718
function httpClientLayer(
1819
handler: (
@@ -243,6 +244,7 @@ describe("platformApiLayer", () => {
243244
Effect.provide(layer),
244245
Effect.provide(runtimeLayer),
245246
Effect.provide(analytics.layer),
247+
Effect.provide(mockOutput({ format: "text" }).layer),
246248
Effect.provide(
247249
Stdio.layerTest({
248250
args: Effect.succeed(["branches", "list"]),

‎apps/cli/src/shared/telemetry/command-instrumentation.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import {
44
getCommandRuntimeCommand,
55
getCommandRuntimeSpanName,
66
} from "../runtime/command-runtime.service.ts";
7+
import { Output } from "../output/output.service.ts";
78
import { withAnalyticsContext } from "./analytics-context.ts";
89
import { Analytics } from "./analytics.service.ts";
910

@@ -99,6 +100,7 @@ function withCommandAnalyticsImplementation<Flags extends Record<string, unknown
99100
});
100101

101102
const analytics = yield* Analytics;
103+
const output = yield* Output;
102104
const stdio = yield* Stdio.Stdio;
103105
const args = yield* stdio.args;
104106
const startedAt = yield* Clock.currentTimeMillis;
@@ -120,6 +122,7 @@ function withCommandAnalyticsImplementation<Flags extends Record<string, unknown
120122
.capture("cli_command_executed", {
121123
exit_code: Exit.isSuccess(exit) ? 0 : 1,
122124
duration_ms: finishedAt - startedAt,
125+
output_format: output.format,
123126
})
124127
.pipe(withAnalyticsContext(analyticsContext));
125128

@@ -133,12 +136,12 @@ function withCommandAnalyticsImplementation<Flags extends Record<string, unknown
133136

134137
export function withCommandInstrumentation(): <A, E, R>(
135138
self: Effect.Effect<A, E, R>,
136-
) => Effect.Effect<A, E, R | Analytics | CommandRuntime | Stdio.Stdio>;
139+
) => Effect.Effect<A, E, R | Analytics | CommandRuntime | Stdio.Stdio | Output>;
137140
export function withCommandInstrumentation<Flags extends Record<string, unknown>>(
138141
options: CommandInstrumentationOptions<Flags>,
139142
): <A, E, R>(
140143
self: Effect.Effect<A, E, R>,
141-
) => Effect.Effect<A, E, R | Analytics | CommandRuntime | Stdio.Stdio>;
144+
) => Effect.Effect<A, E, R | Analytics | CommandRuntime | Stdio.Stdio | Output>;
142145
export function withCommandInstrumentation<Flags extends Record<string, unknown>>(
143146
options?: CommandInstrumentationOptions<Flags>,
144147
) {

‎apps/cli/src/shared/telemetry/command-instrumentation.unit.test.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { commandRuntimeLayer } from "../runtime/command-runtime.layer.ts";
44
import { CurrentAnalyticsContext } from "./analytics-context.ts";
55
import { Analytics } from "./analytics.service.ts";
66
import { withCommandInstrumentation } from "./command-instrumentation.ts";
7+
import { mockOutput } from "../../../tests/helpers/mocks.ts";
78

89
function mockContextualAnalytics() {
910
const captured: Array<{
@@ -46,6 +47,7 @@ describe("withCommandInstrumentation", () => {
4647
}).pipe(
4748
withCommandInstrumentation({ analytics: false }),
4849
Effect.provide(analytics.layer),
50+
Effect.provide(mockOutput({ format: "text" }).layer),
4951
Effect.provide(
5052
Stdio.layerTest({
5153
args: Effect.succeed(["branches", "list"]),
@@ -68,6 +70,7 @@ describe("withCommandInstrumentation", () => {
6870
}).pipe(
6971
withCommandInstrumentation(),
7072
Effect.provide(analytics.layer),
73+
Effect.provide(mockOutput({ format: "text" }).layer),
7174
Effect.provide(
7275
Stdio.layerTest({
7376
args: Effect.succeed(["start", "--detach", "--exclude=auth"]),
@@ -99,6 +102,7 @@ describe("withCommandInstrumentation", () => {
99102

100103
const program = withCommandInstrumentation()(Effect.fail(new Error("boom"))).pipe(
101104
Effect.provide(analytics.layer),
105+
Effect.provide(mockOutput({ format: "text" }).layer),
102106
Effect.provide(
103107
Stdio.layerTest({
104108
args: Effect.succeed(["login"]),
@@ -133,6 +137,7 @@ describe("withCommandInstrumentation", () => {
133137
allowedFlagValues: ["exclude", "mode", "stack"],
134138
}),
135139
Effect.provide(analytics.layer),
140+
Effect.provide(mockOutput({ format: "text" }).layer),
136141
Effect.provide(
137142
Stdio.layerTest({
138143
args: Effect.succeed([
@@ -177,6 +182,7 @@ describe("withCommandInstrumentation", () => {
177182
allowedFlagValues: ["token", "name", "noBrowser"],
178183
}),
179184
Effect.provide(analytics.layer),
185+
Effect.provide(mockOutput({ format: "text" }).layer),
180186
Effect.provide(
181187
Stdio.layerTest({
182188
args: Effect.succeed(["login", "--name", "my-machine", "--no-browser"]),
@@ -202,6 +208,7 @@ describe("withCommandInstrumentation", () => {
202208
return Effect.sync(() => "ok").pipe(
203209
withCommandInstrumentation({ analytics: false }),
204210
Effect.provide(analytics.layer),
211+
Effect.provide(mockOutput({ format: "text" }).layer),
205212
Effect.provide(
206213
Stdio.layerTest({
207214
args: Effect.succeed(["telemetry", "enable"]),

0 commit comments

Comments
 (0)