feat: Update workbook with grid layout and add deploy-workbook.bicep

- Updated token-usage-workbook.json with grid layout for graphs
- Added deploy-workbook.bicep for standalone workbook deployment to any RG

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: Prachig-Microsoft

infra/dashboards/deploy-workbook.bicep

@@ -0,0 +1,27 @@
+// Standalone deployment for LLM Token Usage Workbook
+// Connects to an existing Application Insights instance from any content processing RG
+
+targetScope = 'resourceGroup'
+
+@description('Full resource ID of the Application Insights instance to query.')
+param appInsightsResourceId string
+
+@description('Azure region for the workbook resource.')
+param location string = resourceGroup().location
+
+var workbookId = guid(resourceGroup().id, 'token-usage-workbook')
+
+resource workbook 'Microsoft.Insights/workbooks@2022-04-01' = {
+  name: workbookId
+  location: location
+  kind: 'shared'
+  properties: {
+    displayName: 'LLM Token Usage Dashboard'
+    category: 'workbook'
+    sourceId: appInsightsResourceId
+    serializedData: loadTextContent('token-usage-workbook.json')
+  }
+}
+
+output workbookName string = workbook.name
+output workbookId string = workbook.id

infra/dashboards/token-usage-workbook.json

@@ -25,10 +25,18 @@
             },
             "typeSettings": {
               "selectableValues": [
-                {"durationMs": 3600000},
-                {"durationMs": 86400000},
-                {"durationMs": 604800000},
-                {"durationMs": 2592000000}
+                {
+                  "durationMs": 3600000
+                },
+                {
+                  "durationMs": 86400000
+                },
+                {
+                  "durationMs": 604800000
+                },
+                {
+                  "durationMs": 2592000000
+                }
               ],
               "allowCustom": true
             }
@@ -54,9 +62,6 @@
         "query": "customEvents\r\n| where name == 'LLM_Token_Usage_Summary'\r\n| where timestamp > ago(7d)\r\n| extend process_id = tostring(customDimensions['process_id'])\r\n| extend input_tokens = toint(customDimensions['total_input_tokens'])\r\n| extend output_tokens = toint(customDimensions['total_output_tokens'])\r\n| extend total_tokens = toint(customDimensions['total_tokens'])\r\n| summarize input_tokens=max(input_tokens), output_tokens=max(output_tokens), total_tokens=max(total_tokens) by process_id\r\n| summarize\r\n    TotalDocuments = count(),\r\n    TotalInputTokens = sum(input_tokens),\r\n    TotalOutputTokens = sum(output_tokens),\r\n    TotalTokens = sum(total_tokens),\r\n    AvgTokensPerDocument = round(avg(total_tokens), 0)",
         "size": 3,
         "title": "Overall Token Usage Summary",
-        "timeContext": {
-          "durationMs": 604800000
-        },
         "timeContextFromParameter": "TimeRange",
         "queryType": 0,
         "resourceType": "microsoft.insights/components",
@@ -76,16 +81,13 @@
       "type": 3,
       "content": {
         "version": "KqlItem/1.0",
-        "query": "customEvents\r\n| where name == 'LLM_Agent_Token_Usage'\r\n| where timestamp > ago(7d)\r\n| extend agent = tostring(customDimensions['agent_name'])\r\n| extend process_id = tostring(customDimensions['process_id'])\r\n| extend input_tokens = toint(customDimensions['input_tokens'])\r\n| extend output_tokens = toint(customDimensions['output_tokens'])\r\n| extend total_tokens = toint(customDimensions['total_tokens'])\r\n| summarize input_tokens=max(input_tokens), output_tokens=max(output_tokens), total_tokens=max(total_tokens) by agent, process_id\r\n| summarize\r\n    InputTokens = sum(input_tokens),\r\n    OutputTokens = sum(output_tokens),\r\n    TotalTokens = sum(total_tokens),\r\n    Invocations = count()\r\n    by Step = agent\r\n| order by TotalTokens desc",
+        "query": "customEvents\r\n| where name == 'LLM_Agent_Token_Usage'\r\n| where timestamp > ago(7d)\r\n| extend agent = tostring(customDimensions['agent_name'])\r\n| extend process_id = tostring(customDimensions['process_id'])\r\n| extend input_tokens = toint(customDimensions['input_tokens'])\r\n| extend output_tokens = toint(customDimensions['output_tokens'])\r\n| extend total_tokens = toint(customDimensions['total_tokens'])\r\n| summarize input_tokens=max(input_tokens), output_tokens=max(output_tokens), total_tokens=max(total_tokens) by agent, process_id\r\n| summarize\r\n    InputTokens = sum(input_tokens),\r\n    OutputTokens = sum(output_tokens),\r\n    TotalTokens = sum(total_tokens),\r\n    Invocations = count()\r\n    by Step = agent\r\n| project Step, InputTokens, OutputTokens, TotalTokens, Invocations\r\n| order by TotalTokens desc",
         "size": 0,
         "title": "Token Usage by Pipeline Step",
-        "timeContext": {
-          "durationMs": 604800000
-        },
         "timeContextFromParameter": "TimeRange",
         "queryType": 0,
         "resourceType": "microsoft.insights/components",
-        "visualization": "barchart"
+        "visualization": "table"
       },
       "name": "tokens-by-agent"
     },
@@ -96,9 +98,6 @@
         "query": "customEvents\r\n| where name == 'LLM_Token_Usage_Summary'\r\n| where timestamp > ago(7d)\r\n| extend process_id = tostring(customDimensions['process_id'])\r\n| extend input_tokens = toint(customDimensions['total_input_tokens'])\r\n| extend output_tokens = toint(customDimensions['total_output_tokens'])\r\n| summarize input_tokens=max(input_tokens), output_tokens=max(output_tokens), timestamp=min(timestamp) by process_id\r\n| summarize InputTokens = sum(input_tokens), OutputTokens = sum(output_tokens) by bin(timestamp, 1h)\r\n| order by timestamp asc",
         "size": 0,
         "title": "Token Usage Over Time (Hourly)",
-        "timeContext": {
-          "durationMs": 604800000
-        },
         "timeContextFromParameter": "TimeRange",
         "queryType": 0,
         "resourceType": "microsoft.insights/components",
@@ -113,9 +112,6 @@
         "query": "customEvents\r\n| where name == 'LLM_Agent_Token_Usage'\r\n| where timestamp > ago(7d)\r\n| extend agent = tostring(customDimensions['agent_name'])\r\n| extend process_id = tostring(customDimensions['process_id'])\r\n| extend total_tokens = toint(customDimensions['total_tokens'])\r\n| summarize total_tokens=max(total_tokens) by agent, process_id\r\n| summarize TotalTokens = sum(total_tokens) by agent",
         "size": 0,
         "title": "Token Distribution by Agent",
-        "timeContext": {
-          "durationMs": 604800000
-        },
         "timeContextFromParameter": "TimeRange",
         "queryType": 0,
         "resourceType": "microsoft.insights/components",
@@ -137,9 +133,6 @@
         "query": "let input_price_per_million = 2.50;\r\nlet output_price_per_million = 10.00;\r\ncustomEvents\r\n| where name == 'LLM_Token_Usage_Summary'\r\n| where timestamp > ago(7d)\r\n| extend process_id = tostring(customDimensions['process_id'])\r\n| extend input_tokens = toint(customDimensions['total_input_tokens'])\r\n| extend output_tokens = toint(customDimensions['total_output_tokens'])\r\n| summarize input_tokens=max(input_tokens), output_tokens=max(output_tokens), timestamp=min(timestamp) by process_id\r\n| summarize TotalInput = sum(input_tokens), TotalOutput = sum(output_tokens) by bin(timestamp, 1d)\r\n| extend InputCost = round(TotalInput * input_price_per_million / 1000000.0, 4)\r\n| extend OutputCost = round(TotalOutput * output_price_per_million / 1000000.0, 4)\r\n| extend TotalCost = InputCost + OutputCost\r\n| project Day = timestamp, TotalInput, TotalOutput, InputCost, OutputCost, TotalCost\r\n| order by Day desc",
         "size": 0,
         "title": "Estimated Daily Cost (GPT-4o Pricing: $2.50/1M input, $10.00/1M output)",
-        "timeContext": {
-          "durationMs": 2592000000
-        },
         "timeContextFromParameter": "TimeRange",
         "queryType": 0,
         "resourceType": "microsoft.insights/components",
@@ -154,9 +147,6 @@
         "query": "let gpt4o_input = 2.50;\r\nlet gpt4o_output = 10.00;\r\nlet gpt4o_mini_input = 0.15;\r\nlet gpt4o_mini_output = 0.60;\r\ncustomEvents\r\n| where name == 'LLM_Model_Token_Usage'\r\n| where timestamp > ago(7d)\r\n| extend model = tostring(customDimensions['model_deployment_name'])\r\n| extend input_tokens = toint(customDimensions['input_tokens'])\r\n| extend output_tokens = toint(customDimensions['output_tokens'])\r\n| summarize TotalInput = sum(input_tokens), TotalOutput = sum(output_tokens) by model\r\n| extend InputPrice = case(model has \"mini\", gpt4o_mini_input, gpt4o_input)\r\n| extend OutputPrice = case(model has \"mini\", gpt4o_mini_output, gpt4o_output)\r\n| extend InputCost = round(TotalInput * InputPrice / 1000000.0, 4)\r\n| extend OutputCost = round(TotalOutput * OutputPrice / 1000000.0, 4)\r\n| extend TotalCost = InputCost + OutputCost\r\n| project Model = model, TotalInput, TotalOutput, InputCost, OutputCost, TotalCost\r\n| order by TotalCost desc",
         "size": 0,
         "title": "Estimated Cost by Model",
-        "timeContext": {
-          "durationMs": 2592000000
-        },
         "timeContextFromParameter": "TimeRange",
         "queryType": 0,
         "resourceType": "microsoft.insights/components",
@@ -178,9 +168,6 @@
         "query": "customEvents\r\n| where name == 'LLM_Model_Token_Usage'\r\n| where timestamp > ago(7d)\r\n| extend model = tostring(customDimensions['model_deployment_name'])\r\n| extend input_tokens = toint(customDimensions['input_tokens'])\r\n| extend output_tokens = toint(customDimensions['output_tokens'])\r\n| extend total_tokens = toint(customDimensions['total_tokens'])\r\n| summarize\r\n    InputTokens = sum(input_tokens),\r\n    OutputTokens = sum(output_tokens),\r\n    TotalTokens = sum(total_tokens),\r\n    Invocations = count()\r\n    by Model = model\r\n| order by TotalTokens desc",
         "size": 0,
         "title": "Token Usage by Model",
-        "timeContext": {
-          "durationMs": 604800000
-        },
         "timeContextFromParameter": "TimeRange",
         "queryType": 0,
         "resourceType": "microsoft.insights/components",
@@ -195,9 +182,6 @@
         "query": "customEvents\r\n| where name == 'LLM_Agent_Token_Usage'\r\n| where timestamp > ago(7d)\r\n| extend agent = tostring(customDimensions['agent_name'])\r\n| extend model = tostring(customDimensions['model_deployment_name'])\r\n| extend input_tokens = toint(customDimensions['input_tokens'])\r\n| extend output_tokens = toint(customDimensions['output_tokens'])\r\n| extend total_tokens = toint(customDimensions['total_tokens'])\r\n| summarize\r\n    InputTokens = sum(input_tokens),\r\n    OutputTokens = sum(output_tokens),\r\n    TotalTokens = sum(total_tokens),\r\n    Invocations = count()\r\n    by Step = agent, Model = model\r\n| order by TotalTokens desc",
         "size": 0,
         "title": "Step-to-Model Token Mapping",
-        "timeContext": {
-          "durationMs": 604800000
-        },
         "timeContextFromParameter": "TimeRange",
         "queryType": 0,
         "resourceType": "microsoft.insights/components",
@@ -212,9 +196,6 @@
         "query": "customEvents\r\n| where name == 'LLM_Token_Usage_Summary'\r\n| where timestamp > ago(7d)\r\n| extend total_tokens = toint(customDimensions['total_tokens'])\r\n| extend process_id = tostring(customDimensions['process_id'])\r\n| extend file_name = tostring(customDimensions['file_name'])\r\n| summarize TotalTokens = max(total_tokens) by process_id, file_name\r\n| order by TotalTokens desc\r\n| take 20",
         "size": 0,
         "title": "Top 20 Token Consumers by Document",
-        "timeContext": {
-          "durationMs": 604800000
-        },
         "timeContextFromParameter": "TimeRange",
         "queryType": 0,
         "resourceType": "microsoft.insights/components",
@@ -229,9 +210,6 @@
         "query": "customEvents\r\n| where name == 'LLM_Token_Usage_Summary'\r\n| where timestamp > ago(7d)\r\n| extend process_id = tostring(customDimensions['process_id'])\r\n| extend total_tokens = toint(customDimensions['total_tokens'])\r\n| extend input_tokens = toint(customDimensions['total_input_tokens'])\r\n| extend output_tokens = toint(customDimensions['total_output_tokens'])\r\n| extend mime_type = tostring(customDimensions['file_mime_type'])\r\n| extend file_type = case(\r\n    mime_type has \"pdf\", \"PDF\",\r\n    mime_type has \"image\", \"Image\",\r\n    mime_type has \"word\" or mime_type has \"docx\", \"Word\",\r\n    mime_type has \"excel\" or mime_type has \"xlsx\", \"Excel\",\r\n    mime_type has \"text\", \"Text\",\r\n    \"Other\")\r\n| summarize input_tokens=max(input_tokens), output_tokens=max(output_tokens), total_tokens=max(total_tokens), file_type=take_any(file_type) by process_id\r\n| summarize\r\n    Documents = count(),\r\n    TotalInputTokens = sum(input_tokens),\r\n    TotalOutputTokens = sum(output_tokens),\r\n    TotalTokens = sum(total_tokens),\r\n    AvgTokensPerDoc = round(avg(total_tokens), 0)\r\n    by FileType = file_type\r\n| order by TotalTokens desc",
         "size": 0,
         "title": "Token Usage by File Type",
-        "timeContext": {
-          "durationMs": 604800000
-        },
         "timeContextFromParameter": "TimeRange",
         "queryType": 0,
         "resourceType": "microsoft.insights/components",
@@ -253,9 +231,6 @@
         "query": "customEvents\r\n| where name == 'LLM_Token_Usage_Summary'\r\n| where timestamp > ago(7d)\r\n| extend process_id = tostring(customDimensions['process_id'])\r\n| extend total_tokens = toint(customDimensions['total_tokens'])\r\n| summarize total_tokens=max(total_tokens) by process_id\r\n| summarize\r\n    p50 = percentile(total_tokens, 50),\r\n    p90 = percentile(total_tokens, 90),\r\n    p95 = percentile(total_tokens, 95),\r\n    p99 = percentile(total_tokens, 99),\r\n    Max = max(total_tokens)",
         "size": 3,
         "title": "Token Usage Percentiles Per Document",
-        "timeContext": {
-          "durationMs": 604800000
-        },
         "timeContextFromParameter": "TimeRange",
         "queryType": 0,
         "resourceType": "microsoft.insights/components",
@@ -278,9 +253,6 @@
         "query": "customEvents\r\n| where name == 'LLM_Token_Usage_Summary'\r\n| where timestamp > ago(7d)\r\n| extend process_id = tostring(customDimensions['process_id'])\r\n| extend total_tokens = toint(customDimensions['total_tokens'])\r\n| summarize total_tokens=max(total_tokens), timestamp=min(timestamp) by process_id\r\n| summarize\r\n    DocumentsProcessed = count(),\r\n    TotalTokens = sum(total_tokens),\r\n    AvgTokensPerDoc = round(avg(total_tokens), 0),\r\n    MaxTokensPerDoc = max(total_tokens)\r\n    by Day = bin(timestamp, 1d)\r\n| order by Day desc",
         "size": 0,
         "title": "Daily Processing Volume with Token Usage",
-        "timeContext": {
-          "durationMs": 2592000000
-        },
         "timeContextFromParameter": "TimeRange",
         "queryType": 0,
         "resourceType": "microsoft.insights/components",
@@ -289,8 +261,14 @@
       "name": "daily-volume"
     }
   ],
+  "isLocked": true,
+  "defaultResourceIds": [
+    "/subscriptions/1d5876cd-7603-407a-96d2-ae5ca9a9c5f3/resourcegroups/rg-content-processing-token1/providers/microsoft.insights/components/appi-cptoken1kbxf2",
+    "/subscriptions/1d5876cd-7603-407a-96d2-ae5ca9a9c5f3/resourceGroups/rg-pgcpfeat/providers/Microsoft.Insights/components/appi-pgcpfeatuw333"
+  ],
   "fallbackResourceIds": [
-    "Azure Monitor"
+    "/subscriptions/1d5876cd-7603-407a-96d2-ae5ca9a9c5f3/resourcegroups/rg-content-processing-token1/providers/microsoft.insights/components/appi-cptoken1kbxf2",
+    "/subscriptions/1d5876cd-7603-407a-96d2-ae5ca9a9c5f3/resourceGroups/rg-pgcpfeat/providers/Microsoft.Insights/components/appi-pgcpfeatuw333"
   ],
   "fromTemplateId": "community-Workbooks/Common/Templates"
-}
+}