Skip to content

Commit 87686fd

Browse files
StellaHuang95StellaHuang95
committed
add dashboard
1 parent 4e46444 commit 87686fd

24 files changed

+987
-876
lines changed

‎.gitattributes‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
*.ipynb filter=nbstripout
2+
*.zpln filter=nbstripout
3+
*.ipynb diff=ipynb

‎analysis/kusto/00-telemetry-validation.kql‎

Lines changed: 236 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,236 @@
1+
// =============================================================================
2+
// TELEMETRY VALIDATION
3+
// Checks that PR #1365 telemetry events are arriving with correct properties.
4+
// Each query is self-contained — just copy-paste into Azure Data Explorer.
5+
// =============================================================================
6+
7+
8+
// =============================================================================
9+
// CHECK 0: UniqueMachines with new telemetry
10+
// Shows unique machines per version for builds >= 1.23.10781012 (all stable + pre-release).
11+
// Use this to confirm the latest pre-release build has real users.
12+
// =============================================================================
13+
let filtered = RawEventsVSCodeExt
14+
| where ServerTimestamp > ago(7d)
15+
| where ExtensionName == "ms-python.vscode-python-envs"
16+
| where ExtensionVersion != ""
17+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtensionVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtensionVersion))
18+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012);
19+
filtered
20+
| summarize UniqueMachines = dcount(VSCodeMachineId) by ExtensionVersion
21+
| extend TotalUniqueMachines = toscalar(filtered | summarize dcount(VSCodeMachineId))
22+
| order by UniqueMachines desc;
23+
24+
25+
// =============================================================================
26+
// CHECK 1: Are all 4 events arriving? (Only versions with ⚠️ NO DATA YET are displayed)
27+
// =============================================================================
28+
let allEvents = datatable(EventName: string) [
29+
"ms-python.vscode-python-envs/manager_registration.failed",
30+
"ms-python.vscode-python-envs/manager_registration.skipped",
31+
"ms-python.vscode-python-envs/setup.hang_detected",
32+
"ms-python.vscode-python-envs/extension.manager_registration_duration"
33+
];
34+
let observed = RawEventsVSCodeExt
35+
| where ServerTimestamp > ago(7d)
36+
| where EventName in (
37+
"ms-python.vscode-python-envs/manager_registration.failed",
38+
"ms-python.vscode-python-envs/manager_registration.skipped",
39+
"ms-python.vscode-python-envs/setup.hang_detected",
40+
"ms-python.vscode-python-envs/extension.manager_registration_duration"
41+
)
42+
| extend ExtVersion = tostring(Properties["common.extversion"])
43+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
44+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
45+
| summarize EventCount = count(), UniqueMachines = dcount(VSCodeMachineId) by EventName, ExtVersion;
46+
allEvents
47+
| join kind=leftouter observed on EventName
48+
| project
49+
EventName,
50+
ExtVersion = coalesce(ExtVersion, ""),
51+
EventCount = coalesce(EventCount, 0),
52+
UniqueMachines = coalesce(UniqueMachines, 0),
53+
Status = iff(coalesce(EventCount, 0) > 0, "✅ RECEIVING DATA", "⚠️ NO DATA YET")
54+
| where Status == "⚠️ NO DATA YET"
55+
| order by EventName asc, ExtVersion desc;
56+
57+
58+
// =============================================================================
59+
// CHECK 2: MANAGER_REGISTRATION_DURATION event success rate
60+
// The existing event previously had only "duration". Now it should have "result".
61+
// Expected: rows with result = "success" (should be the majority) and maybe a few "error".
62+
// If result is empty/"", the property isn't being sent correctly.
63+
// =============================================================================
64+
RawEventsVSCodeExt
65+
| where ServerTimestamp > ago(7d)
66+
| where EventName == "ms-python.vscode-python-envs/extension.manager_registration_duration"
67+
| extend ExtVersion = tostring(Properties["common.extversion"])
68+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
69+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
70+
| summarize
71+
TotalEvents = count(),
72+
HasResult = countif(isnotempty(tostring(Properties.result))),
73+
ResultSuccess = countif(tostring(Properties.result) == "success"),
74+
ResultError = countif(tostring(Properties.result) == "error"),
75+
HasDuration = countif(todouble(Properties.duration) > 0)
76+
| extend
77+
ResultPopulatedPct = round(todouble(HasResult) / todouble(TotalEvents) * 100, 1),
78+
ErrorPct = round(todouble(ResultError) / todouble(HasResult) * 100, 2),
79+
Status = iff(HasResult == TotalEvents, "✅ ALL EVENTS HAVE result", "⚠️ SOME EVENTS MISSING result");
80+
81+
82+
// =============================================================================
83+
// CHECK 3: MANAGER_REGISTRATION.SKIPPED (Percentage of users don't have each tool)
84+
// Expected: managerName in {conda, pyenv, pipenv, poetry}, reason = "tool_not_found"
85+
// These should be common — most users don't have all 4 tools.
86+
// =============================================================================
87+
let totalMachines = toscalar(
88+
RawEventsVSCodeExt
89+
| where ServerTimestamp > ago(7d)
90+
| where ExtensionName == "ms-python.vscode-python-envs"
91+
| where ExtensionVersion != ""
92+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtensionVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtensionVersion))
93+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
94+
| summarize dcount(VSCodeMachineId)
95+
);
96+
RawEventsVSCodeExt
97+
| where ServerTimestamp > ago(7d)
98+
| where EventName == "ms-python.vscode-python-envs/manager_registration.skipped"
99+
| extend ExtVersion = tostring(Properties["common.extversion"])
100+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
101+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
102+
| extend ManagerName = tostring(Properties.managername), Reason = tostring(Properties.reason)
103+
| where isnotempty(ManagerName) and isnotempty(Reason)
104+
| summarize
105+
EventCount = count(),
106+
UniqueMachines = dcount(VSCodeMachineId)
107+
by ManagerName, Reason
108+
| extend TotalUniqueMachines = totalMachines
109+
| extend MachinePct = round(todouble(UniqueMachines) / todouble(TotalUniqueMachines) * 100, 1)
110+
| order by EventCount desc;
111+
112+
113+
// =============================================================================
114+
// CHECK 4: MANAGER_REGISTRATION.FAILED — property validation
115+
// Expected: managerName in {system, conda, pyenv, pipenv, poetry, shellStartupVars}
116+
// errorType in {spawn_timeout, spawn_enoent, permission_denied, canceled, parse_error, unknown}
117+
// This may have 0 rows if no one hit errors — that's fine.
118+
//
119+
// errorType reference (from errorClassifier.ts):
120+
// spawn_timeout — tool process started but hung or didn't respond in time
121+
// spawn_enoent — OS couldn't find the executable (ENOENT: not installed or not on PATH)
122+
// permission_denied — OS returned EACCES/EPERM (exists but no permission to run)
123+
// canceled — operation was stopped (user closed VS Code, workspace changed, CancellationToken fired)
124+
// parse_error — tool ran but returned unparseable output (malformed JSON, unexpected format)
125+
// unknown — didn't match any known pattern (catch-all for unexpected errors)
126+
// =============================================================================
127+
let totalMachines = toscalar(
128+
RawEventsVSCodeExt
129+
| where ServerTimestamp > ago(7d)
130+
| where ExtensionName == "ms-python.vscode-python-envs"
131+
| where ExtensionVersion != ""
132+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtensionVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtensionVersion))
133+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
134+
| summarize dcount(VSCodeMachineId)
135+
);
136+
RawEventsVSCodeExt
137+
| where ServerTimestamp > ago(7d)
138+
| where EventName == "ms-python.vscode-python-envs/manager_registration.failed"
139+
| extend ExtVersion = tostring(Properties["common.extversion"])
140+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
141+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
142+
| summarize
143+
EventCount = count(),
144+
UniqueMachines = dcount(VSCodeMachineId)
145+
by ManagerName = tostring(Properties.managername), ErrorType = tostring(Properties.errortype)
146+
| extend TotalUniqueMachines = totalMachines
147+
| extend MachinePct = round(todouble(UniqueMachines) / todouble(TotalUniqueMachines) * 100, 2)
148+
| order by EventCount desc;
149+
150+
151+
// =============================================================================
152+
// CHECK 4a: SPAWN_TIMEOUT failures broken down by manager × extension version
153+
// Shows whether spawn_timeout is improving or worsening across versions.
154+
// If a new version shows higher MachinePct → that release regressed timeout handling.
155+
// =============================================================================
156+
let totalByVersion = RawEventsVSCodeExt
157+
| where ServerTimestamp > ago(7d)
158+
| where ExtensionName == "ms-python.vscode-python-envs"
159+
| where ExtensionVersion != ""
160+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtensionVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtensionVersion))
161+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
162+
| summarize TotalMachines = dcount(VSCodeMachineId) by ExtVersion = ExtensionVersion;
163+
RawEventsVSCodeExt
164+
| where ServerTimestamp > ago(7d)
165+
| where EventName == "ms-python.vscode-python-envs/manager_registration.failed"
166+
| extend ExtVersion = tostring(Properties["common.extversion"])
167+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
168+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
169+
| where tostring(Properties.errortype) == "spawn_timeout"
170+
| summarize
171+
EventCount = count(),
172+
UniqueMachines = dcount(VSCodeMachineId)
173+
by ManagerName = tostring(Properties.managername), ExtVersion
174+
| join kind=inner totalByVersion on ExtVersion
175+
| extend MachinePct = round(todouble(UniqueMachines) / todouble(TotalMachines) * 100, 2)
176+
| project ManagerName, ExtVersion, EventCount, UniqueMachines, TotalMachines, MachinePct
177+
| order by ManagerName asc, ExtVersion desc;
178+
179+
180+
// =============================================================================
181+
// CHECK 4b: UNKNOWN failures broken down by manager × extension version
182+
// Shows whether unknown errors are improving or worsening across versions.
183+
// High counts in the latest version → new unclassified error paths need investigation.
184+
// =============================================================================
185+
let totalByVersion = RawEventsVSCodeExt
186+
| where ServerTimestamp > ago(7d)
187+
| where ExtensionName == "ms-python.vscode-python-envs"
188+
| where ExtensionVersion != ""
189+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtensionVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtensionVersion))
190+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
191+
| summarize TotalMachines = dcount(VSCodeMachineId) by ExtVersion = ExtensionVersion;
192+
RawEventsVSCodeExt
193+
| where ServerTimestamp > ago(7d)
194+
| where EventName == "ms-python.vscode-python-envs/manager_registration.failed"
195+
| extend ExtVersion = tostring(Properties["common.extversion"])
196+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
197+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
198+
| where tostring(Properties.errortype) == "unknown"
199+
| summarize
200+
EventCount = count(),
201+
UniqueMachines = dcount(VSCodeMachineId)
202+
by ManagerName = tostring(Properties.managername), ExtVersion
203+
| join kind=inner totalByVersion on ExtVersion
204+
| extend MachinePct = round(todouble(UniqueMachines) / todouble(TotalMachines) * 100, 2)
205+
| project ManagerName, ExtVersion, EventCount, UniqueMachines, TotalMachines, MachinePct
206+
| order by ManagerName asc, ExtVersion desc;
207+
208+
209+
// =============================================================================
210+
// CHECK 5: SETUP.HANG_DETECTED — property validation
211+
// Expected: failureStage in {nativeFinder, managerRegistration, envSelection, terminalWatcher, settingsListener}
212+
// This should have very few or 0 rows (hangs are rare). 0 is normal.
213+
// =============================================================================
214+
let totalMachines = toscalar(
215+
RawEventsVSCodeExt
216+
| where ServerTimestamp > ago(7d)
217+
| where ExtensionName == "ms-python.vscode-python-envs"
218+
| where ExtensionVersion != ""
219+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtensionVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtensionVersion))
220+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
221+
| summarize dcount(VSCodeMachineId)
222+
);
223+
RawEventsVSCodeExt
224+
| where ServerTimestamp > ago(7d)
225+
| where EventName == "ms-python.vscode-python-envs/setup.hang_detected"
226+
| extend ExtVersion = tostring(Properties["common.extversion"])
227+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
228+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
229+
| summarize
230+
EventCount = count(),
231+
UniqueMachines = dcount(VSCodeMachineId)
232+
by FailureStage = tostring(Properties.failurestage), ExtVersion
233+
| extend TotalUniqueMachines = totalMachines
234+
| extend MachinePct = round(todouble(UniqueMachines) / todouble(TotalUniqueMachines) * 100, 1)
235+
| order by FailureStage asc, ExtVersion desc;
236+

‎analysis/kusto/01-failure-stage-breakdown.kql‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
// Query 1 — Registration Failure Stage Breakdown per Manager
2+
// For events that DO have failureStage, what values are present?
3+
// High counts at a specific stage → that code path is the priority fix target.
4+
RawEventsVSCodeExt
5+
| where ServerTimestamp > ago(7d)
6+
| where EventName == "ms-python.vscode-python-envs/manager_registration.failed"
7+
| extend ExtVersion = tostring(Properties["common.extversion"])
8+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion))
9+
| extend _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
10+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
11+
| extend FailureStage = tostring(Properties.failurestage)
12+
| where isnotempty(FailureStage)
13+
| summarize EventCount = count(), Machines = dcount(VSCodeMachineId)
14+
by ManagerName = tostring(Properties.managername), FailureStage, ExtVersion
15+
| order by Machines desc

‎analysis/kusto/02-error-type-x-failure-stage.kql‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
// Query 2 — Error Type × Failure Stage Matrix
2+
// Cross-tabulates errorType (what kind of error) with failureStage (where it happened).
3+
// This is the key diagnostic view: e.g., "connection_error at nativeFinderRefresh" means
4+
// PET process dies during native finder, while "tool_not_found at pathLookup" means the
5+
// tool binary wasn't on PATH. Prioritize cells with the highest AffectedMachines.
6+
RawEventsVSCodeExt
7+
| where ServerTimestamp > ago(7d)
8+
| where EventName == "ms-python.vscode-python-envs/manager_registration.failed"
9+
| extend ExtVersion = tostring(Properties["common.extversion"])
10+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion))
11+
| extend _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
12+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
13+
| extend ManagerName = tostring(Properties.managername)
14+
| extend ErrorType = tostring(Properties.errortype)
15+
| extend FailureStage = tostring(Properties.failurestage)
16+
| where isnotempty(FailureStage)
17+
| summarize
18+
FailureCount = count(),
19+
AffectedMachines = dcount(VSCodeMachineId)
20+
by ManagerName, ErrorType, FailureStage
21+
| order by AffectedMachines desc, ManagerName asc

‎analysis/kusto/03-overall-setup-success-rate.kql‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
// Query 1 — Overall Setup Success Rate (Last 28 Days)
2+
// Of all machines that started manager setup, what % completed successfully?
3+
// This is the top-level health metric. If this drops, something is broken.
4+
RawEventsVSCodeExt
5+
| where ServerTimestamp > ago(28d)
6+
| where EventName == "ms-python.vscode-python-envs/extension.manager_registration_duration"
7+
| extend ExtVersion = tostring(Properties["common.extversion"])
8+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
9+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
10+
| summarize
11+
TotalMachines = dcount(VSCodeMachineId),
12+
SuccessMachines = dcountif(VSCodeMachineId, tostring(Properties.result) == "success"),
13+
ErrorMachines = dcountif(VSCodeMachineId, tostring(Properties.result) == "error")
14+
| extend SuccessRate = round(todouble(SuccessMachines) / todouble(TotalMachines) * 100, 1)
15+
| project TotalMachines, SuccessMachines, ErrorMachines, SuccessRate

‎analysis/kusto/04-manager-availability.kql‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
// Query 2 — Manager Availability: What Tools Do Users Have Installed?
2+
// For each manager, counts: registered (tool found), skipped (tool not found), failed (crashed).
3+
// InstalledRate shows what % of users actually have each tool.
4+
let endDate = startofday(now()-1d);
5+
let skipped = RawEventsVSCodeExt
6+
| where ServerTimestamp > endDate-28d and ServerTimestamp < endDate
7+
| where EventName == "ms-python.vscode-python-envs/manager_registration.skipped"
8+
| extend ExtVersion = tostring(Properties["common.extversion"])
9+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
10+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
11+
| summarize SkippedMachines = dcount(VSCodeMachineId) by ManagerName = tostring(Properties.managername);
12+
let registered = RawEventsVSCodeExt
13+
| where ServerTimestamp > endDate-28d and ServerTimestamp < endDate
14+
| where EventName == "ms-python.vscode-python-envs/environment_manager.registered"
15+
| extend ExtVersion = tostring(Properties["common.extversion"])
16+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
17+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
18+
| extend RawManagerId = tostring(Properties.managerid)
19+
| extend ManagerName = extract("[^:]+$", 0, RawManagerId)
20+
| summarize RegisteredMachines = dcount(VSCodeMachineId) by ManagerName;
21+
let failed = RawEventsVSCodeExt
22+
| where ServerTimestamp > endDate-28d and ServerTimestamp < endDate
23+
| where EventName == "ms-python.vscode-python-envs/manager_registration.failed"
24+
| extend ExtVersion = tostring(Properties["common.extversion"])
25+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
26+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
27+
| summarize FailedMachines = dcount(VSCodeMachineId) by ManagerName = tostring(Properties.managername);
28+
skipped
29+
| join kind=fullouter registered on ManagerName
30+
| join kind=fullouter failed on ManagerName
31+
| project
32+
Manager = coalesce(ManagerName, ManagerName1, ManagerName2),
33+
Registered = coalesce(RegisteredMachines, 0),
34+
Skipped = coalesce(SkippedMachines, 0),
35+
Failed = coalesce(FailedMachines, 0)
36+
| extend Total = Registered + Skipped + Failed
37+
| extend InstalledRate = iff(Total > 0, round(todouble(Registered) / todouble(Total) * 100, 1), 0.0)
38+
| order by Total desc

‎analysis/kusto/05-daily-trend.kql‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
// Query 3 — Daily Trend: Are Failures Increasing or Decreasing?
2+
// Day-by-day trend of setup results. Check this after shipping a new version.
3+
// A sudden SuccessRate drop means a regression; a rise means a fix is working.
4+
let endDate = startofday(now()-1d);
5+
RawEventsVSCodeExt
6+
| where ServerTimestamp > endDate-14d and ServerTimestamp < endDate
7+
| where EventName == "ms-python.vscode-python-envs/extension.manager_registration_duration"
8+
| extend ExtVersion = tostring(Properties["common.extversion"])
9+
| extend _minor = toint(extract("^1\\.(\\d+)", 1, ExtVersion)), _patch = tolong(extract("^1\\.\\d+\\.(\\d+)", 1, ExtVersion))
10+
| where _minor > 23 or (_minor == 23 and _patch >= 10781012)
11+
| extend Day = startofday(ServerTimestamp)
12+
| extend Result = tostring(Properties.result)
13+
| where Result in ("success", "error")
14+
| summarize
15+
SuccessCount = countif(Result == "success"),
16+
ErrorCount = countif(Result == "error")
17+
by Day
18+
| extend TotalCount = SuccessCount + ErrorCount
19+
| extend SuccessRate = round(todouble(SuccessCount) / todouble(TotalCount) * 100, 1)
20+
| order by Day asc

0 commit comments

Comments
 (0)