Skip to content

Commit a676b0d

Browse files
authored
Add telemetry for databricks aitools install (#5862)
## 🥞 Stacked PR Use this [link](https://github.com/databricks/cli/pull/5862/files) to review incremental changes. - [stack/telemetry](#5862) [[Files changed](https://github.com/databricks/cli/pull/5862/files)] --------- ## Changes Emit a structured `AitoolsInstallEvent` telemetry event on every run of `databricks aitools install` (and its deprecated `experimental aitools skills install` alias, which delegates to the same command). The event records three closed-set, non-PII dimensions: the agents targeted (`agents`, from the fixed agent registry), the install scope (`scope`, project or global), and whether experimental skills were requested (`experimental`). `aitools install` is otherwise a fully local command (skills are fetched from GitHub), so it never resolves auth on its own. `tryConfigureAuth` resolves auth config best-effort into the context so `telemetry.Upload` at command exit has somewhere to send. A user with no configured profile still installs offline and simply emits nothing. ## Why `aitools install` had no usage signal. This lets us measure adoption of Databricks skills across coding agents — how many installs, for which agents, at which scope, and how often the experimental skills are pulled in. ## Tests Unit tests.
1 parent a28e610 commit a676b0d

7 files changed

Lines changed: 251 additions & 17 deletions

File tree

cmd/aitools/install.go

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,10 @@ func NewInstallCmd() *cobra.Command {
5959
var projectFlag, globalFlag bool
6060

6161
cmd := &cobra.Command{
62-
Use: "install",
63-
Short: "Install Databricks skills and plugins for coding agents",
62+
Use: "install",
63+
// Resolve auth best-effort so telemetry can upload; see tryConfigureAuth.
64+
PreRunE: tryConfigureAuth,
65+
Short: "Install Databricks skills and plugins for coding agents",
6466
Long: `Install Databricks skills and plugins for detected coding agents.
6567
6668
By default this installs the databricks plugin through each agent's own CLI
@@ -150,6 +152,11 @@ Supported agents: Claude Code, Cursor, Codex CLI, OpenCode, GitHub Copilot, Anti
150152
}
151153
}
152154

155+
defer logInstallEvent(ctx, plan, installOpts{
156+
Scope: opts.Scope,
157+
Experimental: opts.IncludeExperimental,
158+
})
159+
153160
return executePlan(ctx, src, plan, opts)
154161
},
155162
}

cmd/aitools/install_test.go

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"github.com/databricks/cli/libs/aitools/agents"
1313
"github.com/databricks/cli/libs/aitools/installer"
1414
"github.com/databricks/cli/libs/cmdio"
15+
"github.com/databricks/cli/libs/telemetry"
1516
"github.com/stretchr/testify/assert"
1617
"github.com/stretchr/testify/require"
1718
)
@@ -238,7 +239,7 @@ func TestInstallSkillsOnlyAllAgents(t *testing.T) {
238239
setupTestAgents(t)
239240
calls := setupInstallMock(t)
240241

241-
ctx := cmdio.MockDiscard(t.Context())
242+
ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context()))
242243
cmd := NewInstallCmd()
243244
cmd.SetContext(ctx)
244245
cmd.SetArgs([]string{"--skills-only"})
@@ -253,7 +254,7 @@ func TestInstallSkillsOnlySpecificSkills(t *testing.T) {
253254
setupTestAgents(t)
254255
calls := setupInstallMock(t)
255256

256-
ctx := cmdio.MockDiscard(t.Context())
257+
ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context()))
257258
cmd := NewInstallCmd()
258259
cmd.SetContext(ctx)
259260
cmd.SetArgs([]string{"--skills-only", "--skills", "databricks,databricks-apps"})
@@ -267,7 +268,7 @@ func TestInstallSkillsOnlyExperimental(t *testing.T) {
267268
setupTestAgents(t)
268269
calls := setupInstallMock(t)
269270

270-
ctx := cmdio.MockDiscard(t.Context())
271+
ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context()))
271272
cmd := NewInstallCmd()
272273
cmd.SetContext(ctx)
273274
cmd.SetArgs([]string{"--skills-only", "--experimental"})
@@ -288,7 +289,7 @@ func TestInstallPluginFirstDefault(t *testing.T) {
288289

289290
ctx, stderr := cmdio.NewTestContextWithStderr(t.Context())
290291
cmd := NewInstallCmd()
291-
cmd.SetContext(ctx)
292+
cmd.SetContext(telemetry.WithNewLogger(ctx))
292293

293294
require.NoError(t, cmd.Execute())
294295
require.Len(t, *plugins, 1)
@@ -327,7 +328,7 @@ func TestInstallInteractivePickerAndConfirm(t *testing.T) {
327328
go drainReader(test.Stderr)
328329

329330
cmd := NewInstallCmd()
330-
cmd.SetContext(ctx)
331+
cmd.SetContext(telemetry.WithNewLogger(ctx))
331332

332333
errc := make(chan error, 1)
333334
go func() { errc <- cmd.RunE(cmd, nil) }()
@@ -350,7 +351,7 @@ func TestInstallExplicitAgentWorksUndetected(t *testing.T) {
350351
t.Setenv("DATABRICKS_SKILLS_REF", "v0.2.6")
351352
plugins := setupPluginMock(t)
352353

353-
ctx := cmdio.MockDiscard(t.Context())
354+
ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context()))
354355
cmd := NewInstallCmd()
355356
cmd.SetContext(ctx)
356357
cmd.SetArgs([]string{"--agents", "codex"})
@@ -463,7 +464,7 @@ func TestInstallScopeFlag(t *testing.T) {
463464
setupTestAgents(t)
464465
calls := setupInstallMock(t)
465466

466-
ctx := cmdio.MockDiscard(t.Context())
467+
ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context()))
467468
cmd := NewInstallCmd()
468469
cmd.SetContext(ctx)
469470
cmd.SetArgs(tt.args)
@@ -503,7 +504,7 @@ func TestInstallNoFlagNonInteractiveUsesGlobal(t *testing.T) {
503504
setupTestAgents(t)
504505
calls := setupInstallMock(t)
505506

506-
ctx := cmdio.MockDiscard(t.Context())
507+
ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context()))
507508
cmd := NewInstallCmd()
508509
cmd.SetContext(ctx)
509510
cmd.SetArgs([]string{"--skills-only"})

cmd/aitools/legacy_skills_test.go

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"testing"
55

66
"github.com/databricks/cli/libs/cmdio"
7+
"github.com/databricks/cli/libs/telemetry"
78
"github.com/spf13/cobra"
89
"github.com/stretchr/testify/assert"
910
"github.com/stretchr/testify/require"
@@ -13,7 +14,7 @@ func TestLegacySkillsInstallDelegatesToInstall(t *testing.T) {
1314
setupTestAgents(t)
1415
calls := setupInstallMock(t)
1516

16-
ctx := cmdio.MockDiscard(t.Context())
17+
ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context()))
1718
cmd := newLegacySkillsInstallCmd()
1819
cmd.SetContext(ctx)
1920

@@ -28,7 +29,7 @@ func TestLegacySkillsInstallForwardsSkillName(t *testing.T) {
2829
setupTestAgents(t)
2930
calls := setupInstallMock(t)
3031

31-
ctx := cmdio.MockDiscard(t.Context())
32+
ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context()))
3233
cmd := newLegacySkillsInstallCmd()
3334
cmd.SetContext(ctx)
3435

@@ -43,7 +44,7 @@ func TestLegacySkillsInstallExecuteNoArgs(t *testing.T) {
4344
setupTestAgents(t)
4445
calls := setupInstallMock(t)
4546

46-
ctx := cmdio.MockDiscard(t.Context())
47+
ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context()))
4748
cmd := newLegacySkillsInstallCmd()
4849
cmd.SetContext(ctx)
4950
cmd.SetArgs([]string{})
@@ -60,7 +61,7 @@ func TestLegacySkillsInstallExecuteWithSkillName(t *testing.T) {
6061
setupTestAgents(t)
6162
calls := setupInstallMock(t)
6263

63-
ctx := cmdio.MockDiscard(t.Context())
64+
ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context()))
6465
cmd := newLegacySkillsInstallCmd()
6566
cmd.SetContext(ctx)
6667
cmd.SetArgs([]string{"databricks"})
@@ -76,7 +77,7 @@ func TestLegacySkillsInstallForwardsExperimental(t *testing.T) {
7677
setupTestAgents(t)
7778
calls := setupInstallMock(t)
7879

79-
ctx := cmdio.MockDiscard(t.Context())
80+
ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context()))
8081
cmd := newLegacySkillsInstallCmd()
8182
cmd.SetContext(ctx)
8283
cmd.SetArgs([]string{"--experimental"})
@@ -89,7 +90,7 @@ func TestLegacySkillsInstallForwardsExperimental(t *testing.T) {
8990
}
9091

9192
func TestLegacySkillsInstallExecuteRejectsTwoArgs(t *testing.T) {
92-
ctx := cmdio.MockDiscard(t.Context())
93+
ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context()))
9394
cmd := newLegacySkillsInstallCmd()
9495
cmd.SetContext(ctx)
9596
cmd.SetArgs([]string{"a", "b"})
@@ -111,7 +112,7 @@ func TestLegacySkillsListDelegatesToListFn(t *testing.T) {
111112
return nil
112113
}
113114

114-
ctx := cmdio.MockDiscard(t.Context())
115+
ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context()))
115116
cmd := newLegacySkillsListCmd()
116117
cmd.SetContext(ctx)
117118

cmd/aitools/telemetry.go

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
package aitools
2+
3+
import (
4+
"context"
5+
"slices"
6+
7+
"github.com/databricks/cli/cmd/root"
8+
"github.com/databricks/cli/libs/aitools/agents"
9+
"github.com/databricks/cli/libs/aitools/installer"
10+
"github.com/databricks/cli/libs/telemetry"
11+
"github.com/databricks/cli/libs/telemetry/protos"
12+
"github.com/spf13/cobra"
13+
)
14+
15+
// tryConfigureAuth resolves auth config onto the context so telemetry can
16+
// upload at command exit. It does not fail if auth is not configured.
17+
func tryConfigureAuth(cmd *cobra.Command, args []string) error {
18+
ctx := root.SkipPrompt(cmd.Context())
19+
ctx = root.SkipLoadBundle(ctx)
20+
cmd.SetContext(ctx)
21+
_, _ = root.MustAnyClient(cmd, args)
22+
return nil
23+
}
24+
25+
// installOpts is the subset of install options telemetry records, kept narrow
26+
// so every field that leaves the machine is visible here.
27+
type installOpts struct {
28+
Scope string
29+
Experimental bool
30+
}
31+
32+
// logInstallEvent buffers an install event; cmd/root uploads it at exit.
33+
func logInstallEvent(ctx context.Context, plan []agentPlanItem, opts installOpts) {
34+
telemetry.Log(ctx, protos.DatabricksCliLog{
35+
AitoolsInstallEvent: &protos.AitoolsInstallEvent{
36+
Agents: agentsField(plan),
37+
Scope: scopeType(opts.Scope),
38+
Experimental: opts.Experimental,
39+
},
40+
})
41+
}
42+
43+
// agentsField returns the deduped agent enums from the plan, sorted so the
44+
// same set of agents always produces the same array on the analytics side.
45+
func agentsField(plan []agentPlanItem) []protos.AitoolsAgentType {
46+
if len(plan) == 0 {
47+
return nil
48+
}
49+
out := make([]protos.AitoolsAgentType, 0, len(plan))
50+
seen := make(map[protos.AitoolsAgentType]struct{}, len(plan))
51+
for _, it := range plan {
52+
if it.agent == nil {
53+
continue
54+
}
55+
t := agentType(it.agent.Name)
56+
if _, ok := seen[t]; ok {
57+
continue
58+
}
59+
seen[t] = struct{}{}
60+
out = append(out, t)
61+
}
62+
slices.Sort(out)
63+
return out
64+
}
65+
66+
// agentType maps a CLI agent registry name to its telemetry enum. Every agent
67+
// in agents.Registry must have a case here; TestAgentTypeCoversRegistry fails
68+
// if one is missing. The default maps genuinely unknown names to Unspecified
69+
// so an older proto never drops an install.
70+
func agentType(name string) protos.AitoolsAgentType {
71+
switch name {
72+
case agents.NameClaudeCode:
73+
return protos.AitoolsAgentTypeClaudeCode
74+
case agents.NameCursor:
75+
return protos.AitoolsAgentTypeCursor
76+
case agents.NameCodex:
77+
return protos.AitoolsAgentTypeCodex
78+
case agents.NameOpenCode:
79+
return protos.AitoolsAgentTypeOpenCode
80+
case agents.NameCopilot:
81+
return protos.AitoolsAgentTypeCopilot
82+
case agents.NameAntigravity:
83+
return protos.AitoolsAgentTypeAntigravity
84+
default:
85+
return protos.AitoolsAgentTypeUnspecified
86+
}
87+
}
88+
89+
// scopeType maps a resolved install scope to its telemetry enum.
90+
func scopeType(scope string) protos.AitoolsInstallScope {
91+
switch scope {
92+
case installer.ScopeGlobal:
93+
return protos.AitoolsInstallScopeGlobal
94+
case installer.ScopeProject:
95+
return protos.AitoolsInstallScopeProject
96+
default:
97+
return protos.AitoolsInstallScopeUnspecified
98+
}
99+
}

cmd/aitools/telemetry_test.go

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
package aitools
2+
3+
import (
4+
"testing"
5+
6+
"github.com/databricks/cli/libs/aitools/agents"
7+
"github.com/databricks/cli/libs/aitools/installer"
8+
"github.com/databricks/cli/libs/telemetry/protos"
9+
"github.com/stretchr/testify/assert"
10+
)
11+
12+
func TestAgentsField(t *testing.T) {
13+
cases := []struct {
14+
name string
15+
plan []agentPlanItem
16+
want []protos.AitoolsAgentType
17+
}{
18+
{
19+
name: "empty plan",
20+
plan: nil,
21+
want: nil,
22+
},
23+
{
24+
name: "single agent",
25+
plan: []agentPlanItem{{agent: &agents.Agent{Name: agents.NameClaudeCode}}},
26+
want: []protos.AitoolsAgentType{protos.AitoolsAgentTypeClaudeCode},
27+
},
28+
{
29+
name: "multiple agents are sorted",
30+
plan: []agentPlanItem{
31+
{agent: &agents.Agent{Name: agents.NameCursor}},
32+
{agent: &agents.Agent{Name: agents.NameClaudeCode}},
33+
{agent: &agents.Agent{Name: agents.NameCodex}},
34+
},
35+
want: []protos.AitoolsAgentType{
36+
protos.AitoolsAgentTypeClaudeCode,
37+
protos.AitoolsAgentTypeCodex,
38+
protos.AitoolsAgentTypeCursor,
39+
},
40+
},
41+
{
42+
name: "duplicates are deduplicated",
43+
plan: []agentPlanItem{
44+
{agent: &agents.Agent{Name: agents.NameClaudeCode}},
45+
{agent: &agents.Agent{Name: agents.NameClaudeCode}},
46+
},
47+
want: []protos.AitoolsAgentType{protos.AitoolsAgentTypeClaudeCode},
48+
},
49+
{
50+
name: "nil agent entries are skipped",
51+
plan: []agentPlanItem{
52+
{agent: nil},
53+
{agent: &agents.Agent{Name: agents.NameCursor}},
54+
},
55+
want: []protos.AitoolsAgentType{protos.AitoolsAgentTypeCursor},
56+
},
57+
{
58+
name: "unknown agent maps to unspecified",
59+
plan: []agentPlanItem{{agent: &agents.Agent{Name: "future-agent"}}},
60+
want: []protos.AitoolsAgentType{protos.AitoolsAgentTypeUnspecified},
61+
},
62+
}
63+
for _, tc := range cases {
64+
t.Run(tc.name, func(t *testing.T) {
65+
got := agentsField(tc.plan)
66+
assert.Equal(t, tc.want, got)
67+
})
68+
}
69+
}
70+
71+
// TestAgentTypeCoversRegistry fails when an agent is added to agents.Registry
72+
// without a matching telemetry enum in agentType. This is the guard against
73+
// silently logging a new agent as UNSPECIFIED.
74+
func TestAgentTypeCoversRegistry(t *testing.T) {
75+
for _, a := range agents.Registry {
76+
assert.NotEqualf(t, protos.AitoolsAgentTypeUnspecified, agentType(a.Name),
77+
"agent %q has no telemetry enum: add a case to agentType and a value to "+
78+
"AitoolsAgentType in enum.proto (Universe) + aitools_install.go (CLI)", a.Name)
79+
}
80+
}
81+
82+
func TestScopeType(t *testing.T) {
83+
assert.Equal(t, protos.AitoolsInstallScopeGlobal, scopeType(installer.ScopeGlobal))
84+
assert.Equal(t, protos.AitoolsInstallScopeProject, scopeType(installer.ScopeProject))
85+
assert.Equal(t, protos.AitoolsInstallScopeUnspecified, scopeType(""))
86+
}

0 commit comments

Comments
 (0)