Skip to content

Commit e79df94

Browse files
StellaHuang95StellaHuang95
committed
Add Kusto telemetry dashboard and queries
1 parent c29567b commit e79df94

12 files changed

+793
-0
lines changed

‎.gitattributes‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
*.ipynb filter=nbstripout
2+
*.zpln filter=nbstripout
3+
*.ipynb diff=ipynb

‎analysis/kusto/00-telemetry-validation.kql‎

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,228 @@
1+
// =============================================================================
2+
// TELEMETRY VALIDATION
3+
// Checks that PR #1365 telemetry events are arriving with correct properties.
4+
// Each query is self-contained — just copy-paste into Azure Data Explorer.
5+
// =============================================================================
6+
7+
8+
// =============================================================================
9+
// CHECK 0: UniqueMachines with Telemetry Changes
10+
// Shows unique machines per version for builds >= 1.23.10781012 (all stable + pre-release).
11+
// Use this to confirm the latest pre-release build has real users.
12+
// =============================================================================
13+
let filtered = RawEventsVSCodeExt
14+
| where ServerTimestamp > ago(7d)
15+
| where ExtensionName == "ms-python.vscode-python-envs"
16+
| where ExtensionVersion != ""
17+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtensionVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtensionVersion))
18+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012);
19+
filtered
20+
| summarize UniqueMachines = dcount(VSCodeMachineId) by ExtensionVersion
21+
| extend TotalUniqueMachines = toscalar(filtered | summarize dcount(VSCodeMachineId))
22+
| order by UniqueMachines desc;
23+
24+
25+
// =============================================================================
26+
// CHECK 1: Are all 4 events arriving? (3 new + 1 enhanced)
27+
// Expected: at least 1 row per event. If any row shows 0, that event is broken.
28+
// Broken down by extension version so you can confirm the new build is reporting.
29+
// =============================================================================
30+
let allEvents = datatable(EventName: string) [
31+
"ms-python.vscode-python-envs/manager_registration.failed",
32+
"ms-python.vscode-python-envs/manager_registration.skipped",
33+
"ms-python.vscode-python-envs/setup.hang_detected",
34+
"ms-python.vscode-python-envs/extension.manager_registration_duration"
35+
];
36+
let observed = RawEventsVSCodeExt
37+
| where ServerTimestamp > ago(7d)
38+
| where EventName in (
39+
"ms-python.vscode-python-envs/manager_registration.failed",
40+
"ms-python.vscode-python-envs/manager_registration.skipped",
41+
"ms-python.vscode-python-envs/setup.hang_detected",
42+
"ms-python.vscode-python-envs/extension.manager_registration_duration"
43+
)
44+
| extend ExtVersion = tostring(Properties["common.extversion"])
45+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
46+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
47+
| summarize EventCount = count(), UniqueMachines = dcount(VSCodeMachineId) by EventName, ExtVersion;
48+
allEvents
49+
| join kind=leftouter observed on EventName
50+
| project
51+
EventName,
52+
ExtVersion = coalesce(ExtVersion, ""),
53+
EventCount = coalesce(EventCount, 0),
54+
UniqueMachines = coalesce(UniqueMachines, 0),
55+
Status = iff(coalesce(EventCount, 0) > 0, "✅ RECEIVING DATA", "⚠️ NO DATA YET")
56+
| order by EventName asc, ExtVersion desc;
57+
58+
59+
// =============================================================================
60+
// CHECK 2: Enhanced event — does "result" property exist?
61+
// The existing event previously had only "duration". Now it should have "result".
62+
// Expected: rows with result = "success" (should be the majority) and maybe a few "error".
63+
// If result is empty/"", the property isn't being sent correctly.
64+
// =============================================================================
65+
RawEventsVSCodeExt
66+
| where ServerTimestamp > ago(7d)
67+
| where EventName == "ms-python.vscode-python-envs/extension.manager_registration_duration"
68+
| extend ExtVersion = tostring(Properties["common.extversion"])
69+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
70+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
71+
| summarize
72+
TotalEvents = count(),
73+
HasResult = countif(isnotempty(tostring(Properties.result))),
74+
ResultSuccess = countif(tostring(Properties.result) == "success"),
75+
ResultError = countif(tostring(Properties.result) == "error"),
76+
HasDuration = countif(todouble(Properties.duration) > 0)
77+
| extend
78+
ResultPopulatedPct = round(todouble(HasResult) / todouble(TotalEvents) * 100, 1),
79+
ErrorPct = round(todouble(ResultError) / todouble(HasResult) * 100, 2),
80+
Status = iff(HasResult == TotalEvents, "✅ ALL EVENTS HAVE result", "⚠️ SOME EVENTS MISSING result");
81+
82+
83+
// =============================================================================
84+
// CHECK 3: MANAGER_REGISTRATION.SKIPPED — property validation
85+
// Expected: managerName in {conda, pyenv, pipenv, poetry}, reason = "tool_not_found"
86+
// These should be common — most users don't have all 4 tools.
87+
// =============================================================================
88+
let totalMachines = toscalar(
89+
RawEventsVSCodeExt
90+
| where ServerTimestamp > ago(7d)
91+
| where ExtensionName == "ms-python.vscode-python-envs"
92+
| where ExtensionVersion != ""
93+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtensionVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtensionVersion))
94+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
95+
| summarize dcount(VSCodeMachineId)
96+
);
97+
RawEventsVSCodeExt
98+
| where ServerTimestamp > ago(7d)
99+
| where EventName == "ms-python.vscode-python-envs/manager_registration.skipped"
100+
| extend ExtVersion = tostring(Properties["common.extversion"])
101+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
102+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
103+
| summarize
104+
EventCount = count(),
105+
UniqueMachines = dcount(VSCodeMachineId)
106+
by ManagerName = tostring(Properties.managername), Reason = tostring(Properties.reason)
107+
| extend TotalUniqueMachines = totalMachines
108+
| extend MachinePct = round(todouble(UniqueMachines) / todouble(TotalUniqueMachines) * 100, 1)
109+
| order by EventCount desc;
110+
111+
112+
// =============================================================================
113+
// CHECK 4: MANAGER_REGISTRATION.FAILED — property validation
114+
// Expected: managerName in {system, conda, pyenv, pipenv, poetry, shellStartupVars}
115+
// errorType in {spawn_timeout, spawn_enoent, permission_denied, canceled, parse_error, unknown}
116+
// This may have 0 rows if no one hit errors — that's fine.
117+
//
118+
// errorType reference (from errorClassifier.ts):
119+
// spawn_timeout — tool process started but hung or didn't respond in time
120+
// spawn_enoent — OS couldn't find the executable (ENOENT: not installed or not on PATH)
121+
// permission_denied — OS returned EACCES/EPERM (exists but no permission to run)
122+
// canceled — operation was stopped (user closed VS Code, workspace changed, CancellationToken fired)
123+
// parse_error — tool ran but returned unparseable output (malformed JSON, unexpected format)
124+
// unknown — didn't match any known pattern (catch-all for unexpected errors)
125+
// =============================================================================
126+
let totalMachines = toscalar(
127+
RawEventsVSCodeExt
128+
| where ServerTimestamp > ago(7d)
129+
| where ExtensionName == "ms-python.vscode-python-envs"
130+
| where ExtensionVersion != ""
131+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtensionVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtensionVersion))
132+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
133+
| summarize dcount(VSCodeMachineId)
134+
);
135+
RawEventsVSCodeExt
136+
| where ServerTimestamp > ago(7d)
137+
| where EventName == "ms-python.vscode-python-envs/manager_registration.failed"
138+
| extend ExtVersion = tostring(Properties["common.extversion"])
139+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
140+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
141+
| summarize
142+
EventCount = count(),
143+
UniqueMachines = dcount(VSCodeMachineId)
144+
by ManagerName = tostring(Properties.managername), ErrorType = tostring(Properties.errortype)
145+
| extend TotalUniqueMachines = totalMachines
146+
| extend MachinePct = round(todouble(UniqueMachines) / todouble(TotalUniqueMachines) * 100, 1)
147+
| order by EventCount desc;
148+
149+
150+
// =============================================================================
151+
// CHECK 5: SETUP.HANG_DETECTED — property validation
152+
// Expected: failureStage in {nativeFinder, managerRegistration, envSelection, terminalWatcher, settingsListener}
153+
// This should have very few or 0 rows (hangs are rare). 0 is normal.
154+
// =============================================================================
155+
let totalMachines = toscalar(
156+
RawEventsVSCodeExt
157+
| where ServerTimestamp > ago(7d)
158+
| where ExtensionName == "ms-python.vscode-python-envs"
159+
| where ExtensionVersion != ""
160+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtensionVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtensionVersion))
161+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
162+
| summarize dcount(VSCodeMachineId)
163+
);
164+
RawEventsVSCodeExt
165+
| where ServerTimestamp > ago(7d)
166+
| where EventName == "ms-python.vscode-python-envs/setup.hang_detected"
167+
| extend ExtVersion = tostring(Properties["common.extversion"])
168+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
169+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
170+
| summarize
171+
EventCount = count(),
172+
UniqueMachines = dcount(VSCodeMachineId)
173+
by FailureStage = tostring(Properties.failurestage)
174+
| extend TotalUniqueMachines = totalMachines
175+
| extend MachinePct = round(todouble(UniqueMachines) / todouble(TotalUniqueMachines) * 100, 1)
176+
| order by EventCount desc;
177+
178+
179+
// =============================================================================
180+
// CHECK 6: Registered vs Skipped — unique machine counts per manager
181+
// For each manager that can be skipped, compare registered vs skipped machine counts.
182+
// If a manager shows 0 registered AND 0 skipped, the telemetry path may be broken.
183+
// =============================================================================
184+
let totalMachines = toscalar(
185+
RawEventsVSCodeExt
186+
| where ServerTimestamp > ago(7d)
187+
| where ExtensionName == "ms-python.vscode-python-envs"
188+
| where ExtensionVersion != ""
189+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtensionVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtensionVersion))
190+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
191+
| summarize dcount(VSCodeMachineId)
192+
);
193+
let skipped = RawEventsVSCodeExt
194+
| where ServerTimestamp > ago(7d)
195+
| where EventName == "ms-python.vscode-python-envs/manager_registration.skipped"
196+
| extend ExtVersion = tostring(Properties["common.extversion"])
197+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
198+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
199+
| summarize SkippedMachines = dcount(VSCodeMachineId) by Manager = tostring(Properties.managername);
200+
let registered = RawEventsVSCodeExt
201+
| where ServerTimestamp > ago(7d)
202+
| where EventName == "ms-python.vscode-python-envs/environment_manager.registered"
203+
| extend ExtVersion = tostring(Properties["common.extversion"])
204+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
205+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
206+
| extend RawManagerId = tostring(Properties.managerid)
207+
| where RawManagerId has_any ("conda", "pyenv", "pipenv", "poetry")
208+
| extend Manager = case(
209+
RawManagerId has "conda", "conda",
210+
RawManagerId has "pyenv", "pyenv",
211+
RawManagerId has "pipenv", "pipenv",
212+
RawManagerId has "poetry", "poetry",
213+
RawManagerId)
214+
| summarize RegisteredMachines = dcount(VSCodeMachineId) by Manager;
215+
skipped
216+
| join kind=fullouter registered on Manager
217+
| project
218+
Manager = coalesce(Manager, Manager1),
219+
RegisteredMachines = coalesce(RegisteredMachines, 0),
220+
SkippedMachines = coalesce(SkippedMachines, 0),
221+
TotalUniqueMachines = totalMachines
222+
| extend
223+
RegisteredPct = round(todouble(RegisteredMachines) / todouble(TotalUniqueMachines) * 100, 1),
224+
SkippedPct = round(todouble(SkippedMachines) / todouble(TotalUniqueMachines) * 100, 1)
225+
| order by Manager asc;
226+
227+
228+

‎analysis/kusto/01-overall-setup-success-rate.kql‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
// Query 1 — Overall Setup Success Rate (Last 28 Days)
2+
// Of all machines that started manager setup, what % completed successfully?
3+
// This is the top-level health metric. If this drops, something is broken.
4+
RawEventsVSCodeExt
5+
| where ServerTimestamp > ago(28d)
6+
| where EventName == "ms-python.vscode-python-envs/extension.manager_registration_duration"
7+
| extend ExtVersion = tostring(Properties["common.extversion"])
8+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
9+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
10+
| summarize
11+
TotalMachines = dcount(VSCodeMachineId),
12+
SuccessMachines = dcountif(VSCodeMachineId, tostring(Properties.result) == "success"),
13+
ErrorMachines = dcountif(VSCodeMachineId, tostring(Properties.result) == "error")
14+
| extend SuccessRate = round(todouble(SuccessMachines) / todouble(TotalMachines) * 100, 1)
15+
| project TotalMachines, SuccessMachines, ErrorMachines, SuccessRate

‎analysis/kusto/02-manager-availability.kql‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
// Query 2 — Manager Availability: What Tools Do Users Have Installed?
2+
// For each manager, counts: registered (tool found), skipped (tool not found), failed (crashed).
3+
// InstalledRate shows what % of users actually have each tool.
4+
let endDate = startofday(now()-1d);
5+
let skipped = RawEventsVSCodeExt
6+
| where ServerTimestamp > endDate-28d and ServerTimestamp < endDate
7+
| where EventName == "ms-python.vscode-python-envs/manager_registration.skipped"
8+
| extend ExtVersion = tostring(Properties["common.extversion"])
9+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
10+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
11+
| summarize SkippedMachines = dcount(VSCodeMachineId) by ManagerName = tostring(Properties.managername);
12+
let registered = RawEventsVSCodeExt
13+
| where ServerTimestamp > endDate-28d and ServerTimestamp < endDate
14+
| where EventName == "ms-python.vscode-python-envs/environment_manager.registered"
15+
| extend ExtVersion = tostring(Properties["common.extversion"])
16+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
17+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
18+
| extend RawManagerId = tostring(Properties.managerid)
19+
| extend ManagerName = extract("[^:]+$", 0, RawManagerId)
20+
| summarize RegisteredMachines = dcount(VSCodeMachineId) by ManagerName;
21+
let failed = RawEventsVSCodeExt
22+
| where ServerTimestamp > endDate-28d and ServerTimestamp < endDate
23+
| where EventName == "ms-python.vscode-python-envs/manager_registration.failed"
24+
| extend ExtVersion = tostring(Properties["common.extversion"])
25+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
26+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
27+
| summarize FailedMachines = dcount(VSCodeMachineId) by ManagerName = tostring(Properties.managername);
28+
skipped
29+
| join kind=fullouter registered on ManagerName
30+
| join kind=fullouter failed on ManagerName
31+
| project
32+
Manager = coalesce(ManagerName, ManagerName1, ManagerName2),
33+
Registered = coalesce(RegisteredMachines, 0),
34+
Skipped = coalesce(SkippedMachines, 0),
35+
Failed = coalesce(FailedMachines, 0)
36+
| extend Total = Registered + Skipped + Failed
37+
| extend InstalledRate = iff(Total > 0, round(todouble(Registered) / todouble(Total) * 100, 1), 0.0)
38+
| order by Total desc

‎analysis/kusto/03-daily-trend.kql‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
// Query 3 — Daily Trend: Are Failures Increasing or Decreasing?
2+
// Day-by-day trend of setup results. Check this after shipping a new version.
3+
// A sudden SuccessRate drop means a regression; a rise means a fix is working.
4+
let endDate = startofday(now()-1d);
5+
RawEventsVSCodeExt
6+
| where ServerTimestamp > endDate-14d and ServerTimestamp < endDate
7+
| where EventName == "ms-python.vscode-python-envs/extension.manager_registration_duration"
8+
| extend ExtVersion = tostring(Properties["common.extversion"])
9+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
10+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
11+
| extend Day = startofday(ServerTimestamp)
12+
| extend Result = tostring(Properties.result)
13+
| where Result in ("success", "error")
14+
| summarize
15+
SuccessCount = countif(Result == "success"),
16+
ErrorCount = countif(Result == "error")
17+
by Day
18+
| extend TotalCount = SuccessCount + ErrorCount
19+
| extend SuccessRate = round(todouble(SuccessCount) / todouble(TotalCount) * 100, 1)
20+
| order by Day asc

‎analysis/kusto/04-error-type-distribution.kql‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
// Query 4 — Error Type Distribution Across All Failures
2+
// Combines errors from overall setup and individual managers, grouped by error type.
3+
// Action depends on results:
4+
// spawn_timeout → native finder or tool is too slow
5+
// spawn_enoent → binary path wrong or not on PATH
6+
// permission_denied → file system permission issue
7+
// parse_error → a tool returned unexpected output
8+
// unknown → needs deeper investigation
9+
let endDate = startofday(now()-1d);
10+
let setupErrors = RawEventsVSCodeExt
11+
| where ServerTimestamp > endDate-28d and ServerTimestamp < endDate
12+
| where EventName == "ms-python.vscode-python-envs/extension.manager_registration_duration"
13+
| extend ExtVersion = tostring(Properties["common.extversion"])
14+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
15+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
16+
| where tostring(Properties.result) == "error"
17+
| project ErrorType = tostring(Properties.errortype), Source = "setup_overall", VSCodeMachineId;
18+
let managerErrors = RawEventsVSCodeExt
19+
| where ServerTimestamp > endDate-28d and ServerTimestamp < endDate
20+
| where EventName == "ms-python.vscode-python-envs/manager_registration.failed"
21+
| extend ExtVersion = tostring(Properties["common.extversion"])
22+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
23+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
24+
| project ErrorType = tostring(Properties.errortype), Source = "individual_manager", VSCodeMachineId;
25+
union setupErrors, managerErrors
26+
| summarize
27+
EventCount = count(),
28+
AffectedMachines = dcount(VSCodeMachineId)
29+
by ErrorType, Source
30+
| order by AffectedMachines desc

0 commit comments

Comments
 (0)