Skip to content

Commit 987defa

Browse files
Prachig-MicrosoftCopilot
Prachig-Microsoft
and
Copilot
committed
feat: Add portable Azure Workbook template for token usage dashboard
Adds a standalone workbook JSON template that can be imported into any Application Insights instance without being tied to a specific resource. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 2dab021 commit 987defa

1 file changed

Lines changed: 236 additions & 0 deletions

File tree

‎infra/dashboards/token-usage-workbook.json‎

Lines changed: 236 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,236 @@
1+
{
2+
"version": "Notebook/1.0",
3+
"$schema": "https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json",
4+
"items": [
5+
{
6+
"type": 1,
7+
"content": {
8+
"json": "# LLM Token Usage Dashboard\n\nMonitors token consumption across all pipeline agents in Content Processing Solution Accelerator.\n\n---"
9+
},
10+
"name": "title"
11+
},
12+
{
13+
"type": 9,
14+
"content": {
15+
"version": "KqlParameterItem/1.0",
16+
"parameters": [
17+
{
18+
"id": "a0b1c2d3-e4f5-6789-abcd-ef0123456789",
19+
"version": "KqlParameterItem/1.0",
20+
"name": "TimeRange",
21+
"label": "Time Range",
22+
"type": 4,
23+
"isRequired": true,
24+
"value": {
25+
"durationMs": 604800000
26+
},
27+
"typeSettings": {
28+
"selectableValues": [
29+
{ "durationMs": 3600000 },
30+
{ "durationMs": 86400000 },
31+
{ "durationMs": 604800000 },
32+
{ "durationMs": 2592000000 }
33+
]
34+
}
35+
}
36+
],
37+
"style": "pills"
38+
},
39+
"name": "parameters"
40+
},
41+
{
42+
"type": 1,
43+
"content": {
44+
"json": "## Overview"
45+
},
46+
"name": "section-overview"
47+
},
48+
{
49+
"type": 3,
50+
"content": {
51+
"version": "KqlItem/1.0",
52+
"query": "customEvents\n| where name == 'LLM_Token_Usage_Summary'\n| where timestamp {TimeRange}\n| extend input_tokens = toint(customDimensions['total_input_tokens'])\n| extend output_tokens = toint(customDimensions['total_output_tokens'])\n| extend total_tokens = toint(customDimensions['total_tokens'])\n| summarize\n TotalDocuments = count(),\n TotalInputTokens = sum(input_tokens),\n TotalOutputTokens = sum(output_tokens),\n TotalTokens = sum(total_tokens),\n AvgTokensPerDocument = round(avg(total_tokens), 0)",
53+
"size": 4,
54+
"title": "Overall Token Usage Summary",
55+
"queryType": 0,
56+
"resourceType": "microsoft.insights/components",
57+
"visualization": "tiles",
58+
"tileSettings": {
59+
"showBorder": true
60+
}
61+
},
62+
"name": "overall-summary"
63+
},
64+
{
65+
"type": 3,
66+
"content": {
67+
"version": "KqlItem/1.0",
68+
"query": "customEvents\n| where name == 'LLM_Agent_Token_Usage'\n| where timestamp {TimeRange}\n| extend agent = tostring(customDimensions['agent_name'])\n| extend input_tokens = toint(customDimensions['input_tokens'])\n| extend output_tokens = toint(customDimensions['output_tokens'])\n| extend total_tokens = toint(customDimensions['total_tokens'])\n| summarize\n InputTokens = sum(input_tokens),\n OutputTokens = sum(output_tokens),\n TotalTokens = sum(total_tokens),\n Invocations = count()\n by Step = agent\n| order by TotalTokens desc",
69+
"size": 0,
70+
"title": "Token Usage by Pipeline Step",
71+
"queryType": 0,
72+
"resourceType": "microsoft.insights/components",
73+
"visualization": "barchart"
74+
},
75+
"name": "tokens-by-agent"
76+
},
77+
{
78+
"type": 3,
79+
"content": {
80+
"version": "KqlItem/1.0",
81+
"query": "customEvents\n| where name == 'LLM_Token_Usage_Summary'\n| where timestamp {TimeRange}\n| extend input_tokens = toint(customDimensions['total_input_tokens'])\n| extend output_tokens = toint(customDimensions['total_output_tokens'])\n| summarize InputTokens = sum(input_tokens), OutputTokens = sum(output_tokens) by bin(timestamp, 1h)\n| order by timestamp asc",
82+
"size": 0,
83+
"title": "Token Usage Over Time (Hourly)",
84+
"queryType": 0,
85+
"resourceType": "microsoft.insights/components",
86+
"visualization": "areachart"
87+
},
88+
"name": "tokens-over-time"
89+
},
90+
{
91+
"type": 3,
92+
"content": {
93+
"version": "KqlItem/1.0",
94+
"query": "customEvents\n| where name == 'LLM_Agent_Token_Usage'\n| where timestamp {TimeRange}\n| extend agent = tostring(customDimensions['agent_name'])\n| extend total_tokens = toint(customDimensions['total_tokens'])\n| summarize TotalTokens = sum(total_tokens) by agent",
95+
"size": 0,
96+
"title": "Token Distribution by Agent",
97+
"queryType": 0,
98+
"resourceType": "microsoft.insights/components",
99+
"visualization": "piechart"
100+
},
101+
"name": "token-distribution-pie"
102+
},
103+
{
104+
"type": 1,
105+
"content": {
106+
"json": "## Cost Estimation"
107+
},
108+
"name": "section-cost"
109+
},
110+
{
111+
"type": 3,
112+
"content": {
113+
"version": "KqlItem/1.0",
114+
"query": "let input_price_per_million = 2.50;\nlet output_price_per_million = 10.00;\ncustomEvents\n| where name == 'LLM_Token_Usage_Summary'\n| where timestamp {TimeRange}\n| extend input_tokens = toint(customDimensions['total_input_tokens'])\n| extend output_tokens = toint(customDimensions['total_output_tokens'])\n| summarize TotalInput = sum(input_tokens), TotalOutput = sum(output_tokens) by bin(timestamp, 1d)\n| extend InputCost = round(TotalInput * input_price_per_million / 1000000.0, 4)\n| extend OutputCost = round(TotalOutput * output_price_per_million / 1000000.0, 4)\n| extend TotalCost = InputCost + OutputCost\n| project Day = timestamp, TotalInput, TotalOutput, InputCost, OutputCost, TotalCost\n| order by Day desc",
115+
"size": 0,
116+
"title": "Estimated Daily Cost (GPT-4o Pricing: $2.50/1M input, $10.00/1M output)",
117+
"queryType": 0,
118+
"resourceType": "microsoft.insights/components",
119+
"visualization": "table"
120+
},
121+
"name": "cost-estimation"
122+
},
123+
{
124+
"type": 3,
125+
"content": {
126+
"version": "KqlItem/1.0",
127+
"query": "let gpt4o_input = 2.50;\nlet gpt4o_output = 10.00;\nlet gpt4o_mini_input = 0.15;\nlet gpt4o_mini_output = 0.60;\ncustomEvents\n| where name == 'LLM_Model_Token_Usage'\n| where timestamp {TimeRange}\n| extend model = tostring(customDimensions['model_deployment_name'])\n| extend input_tokens = toint(customDimensions['input_tokens'])\n| extend output_tokens = toint(customDimensions['output_tokens'])\n| summarize TotalInput = sum(input_tokens), TotalOutput = sum(output_tokens) by model\n| extend InputPrice = case(model has \"mini\", gpt4o_mini_input, gpt4o_input)\n| extend OutputPrice = case(model has \"mini\", gpt4o_mini_output, gpt4o_output)\n| extend InputCost = round(TotalInput * InputPrice / 1000000.0, 4)\n| extend OutputCost = round(TotalOutput * OutputPrice / 1000000.0, 4)\n| extend TotalCost = InputCost + OutputCost\n| project Model = model, TotalInput, TotalOutput, InputCost, OutputCost, TotalCost\n| order by TotalCost desc",
128+
"size": 0,
129+
"title": "Estimated Cost by Model",
130+
"queryType": 0,
131+
"resourceType": "microsoft.insights/components",
132+
"visualization": "table"
133+
},
134+
"name": "cost-by-model"
135+
},
136+
{
137+
"type": 1,
138+
"content": {
139+
"json": "## Model & Document Details"
140+
},
141+
"name": "section-details"
142+
},
143+
{
144+
"type": 3,
145+
"content": {
146+
"version": "KqlItem/1.0",
147+
"query": "customEvents\n| where name == 'LLM_Model_Token_Usage'\n| where timestamp {TimeRange}\n| extend model = tostring(customDimensions['model_deployment_name'])\n| extend input_tokens = toint(customDimensions['input_tokens'])\n| extend output_tokens = toint(customDimensions['output_tokens'])\n| extend total_tokens = toint(customDimensions['total_tokens'])\n| summarize\n InputTokens = sum(input_tokens),\n OutputTokens = sum(output_tokens),\n TotalTokens = sum(total_tokens),\n Invocations = count()\n by Model = model\n| order by TotalTokens desc",
148+
"size": 0,
149+
"title": "Token Usage by Model",
150+
"queryType": 0,
151+
"resourceType": "microsoft.insights/components",
152+
"visualization": "table"
153+
},
154+
"name": "tokens-by-model"
155+
},
156+
{
157+
"type": 3,
158+
"content": {
159+
"version": "KqlItem/1.0",
160+
"query": "customEvents\n| where name == 'LLM_Agent_Token_Usage'\n| where timestamp {TimeRange}\n| extend agent = tostring(customDimensions['agent_name'])\n| extend model = tostring(customDimensions['model_deployment_name'])\n| extend input_tokens = toint(customDimensions['input_tokens'])\n| extend output_tokens = toint(customDimensions['output_tokens'])\n| extend total_tokens = toint(customDimensions['total_tokens'])\n| summarize\n InputTokens = sum(input_tokens),\n OutputTokens = sum(output_tokens),\n TotalTokens = sum(total_tokens),\n Invocations = count()\n by Step = agent, Model = model\n| order by TotalTokens desc",
161+
"size": 0,
162+
"title": "Step-to-Model Token Mapping",
163+
"queryType": 0,
164+
"resourceType": "microsoft.insights/components",
165+
"visualization": "table"
166+
},
167+
"name": "step-model-mapping"
168+
},
169+
{
170+
"type": 3,
171+
"content": {
172+
"version": "KqlItem/1.0",
173+
"query": "customEvents\n| where name == 'LLM_Token_Usage_Summary'\n| where timestamp {TimeRange}\n| extend total_tokens = toint(customDimensions['total_tokens'])\n| extend process_id = tostring(customDimensions['process_id'])\n| extend file_name = tostring(customDimensions['file_name'])\n| summarize TotalTokens = sum(total_tokens) by process_id, file_name\n| order by TotalTokens desc\n| take 20",
174+
"size": 0,
175+
"title": "Top 20 Token Consumers by Document",
176+
"queryType": 0,
177+
"resourceType": "microsoft.insights/components",
178+
"visualization": "table"
179+
},
180+
"name": "top-consumers"
181+
},
182+
{
183+
"type": 3,
184+
"content": {
185+
"version": "KqlItem/1.0",
186+
"query": "customEvents\n| where name == 'LLM_Token_Usage_Summary'\n| where timestamp {TimeRange}\n| extend total_tokens = toint(customDimensions['total_tokens'])\n| extend input_tokens = toint(customDimensions['total_input_tokens'])\n| extend output_tokens = toint(customDimensions['total_output_tokens'])\n| extend mime_type = tostring(customDimensions['file_mime_type'])\n| extend file_type = case(\n mime_type has \"pdf\", \"PDF\",\n mime_type has \"image\", \"Image\",\n mime_type has \"word\" or mime_type has \"docx\", \"Word\",\n mime_type has \"excel\" or mime_type has \"xlsx\", \"Excel\",\n mime_type has \"text\", \"Text\",\n \"Other\")\n| summarize\n Documents = count(),\n TotalInputTokens = sum(input_tokens),\n TotalOutputTokens = sum(output_tokens),\n TotalTokens = sum(total_tokens),\n AvgTokensPerDoc = round(avg(total_tokens), 0)\n by FileType = file_type\n| order by TotalTokens desc",
187+
"size": 0,
188+
"title": "Token Usage by File Type",
189+
"queryType": 0,
190+
"resourceType": "microsoft.insights/components",
191+
"visualization": "barchart"
192+
},
193+
"name": "tokens-by-filetype"
194+
},
195+
{
196+
"type": 1,
197+
"content": {
198+
"json": "## Percentiles & Trends"
199+
},
200+
"name": "section-percentiles"
201+
},
202+
{
203+
"type": 3,
204+
"content": {
205+
"version": "KqlItem/1.0",
206+
"query": "customEvents\n| where name == 'LLM_Token_Usage_Summary'\n| where timestamp {TimeRange}\n| extend total_tokens = toint(customDimensions['total_tokens'])\n| summarize\n p50 = percentile(total_tokens, 50),\n p90 = percentile(total_tokens, 90),\n p95 = percentile(total_tokens, 95),\n p99 = percentile(total_tokens, 99),\n Max = max(total_tokens)",
207+
"size": 4,
208+
"title": "Token Usage Percentiles Per Document",
209+
"queryType": 0,
210+
"resourceType": "microsoft.insights/components",
211+
"visualization": "tiles",
212+
"tileSettings": {
213+
"showBorder": true
214+
}
215+
},
216+
"name": "token-percentiles"
217+
},
218+
{
219+
"type": 3,
220+
"content": {
221+
"version": "KqlItem/1.0",
222+
"query": "customEvents\n| where name == 'LLM_Token_Usage_Summary'\n| where timestamp {TimeRange}\n| extend total_tokens = toint(customDimensions['total_tokens'])\n| extend file_name = tostring(customDimensions['file_name'])\n| summarize\n DocumentsProcessed = count(),\n TotalTokens = sum(total_tokens),\n AvgTokensPerDoc = round(avg(total_tokens), 0),\n MaxTokensPerDoc = max(total_tokens)\n by Day = bin(timestamp, 1d)\n| order by Day desc",
223+
"size": 0,
224+
"title": "Daily Processing Volume with Token Usage",
225+
"queryType": 0,
226+
"resourceType": "microsoft.insights/components",
227+
"visualization": "table"
228+
},
229+
"name": "daily-volume"
230+
}
231+
],
232+
"isLocked": false,
233+
"fallbackResourceIds": [
234+
"Azure Monitor"
235+
]
236+
}

0 commit comments

Comments
 (0)