Skip to content

Commit 09a3f85

Browse files
adamdotdevinadamdotdevin
committed
wip: stats
1 parent ece6098 commit 09a3f85

4 files changed

Lines changed: 112 additions & 95 deletions

File tree

‎infra/stats.ts‎

Lines changed: 34 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -28,22 +28,40 @@ const inferenceEventTable = new aws.s3tables.Table("LakeInferenceEventTable", {
2828
{ name: "event_date", type: "string", required: false },
2929
{ name: "event_type", type: "string", required: false },
3030
{ name: "dataset", type: "string", required: false },
31+
{ name: "cf_continent", type: "string", required: false },
32+
{ name: "cf_country", type: "string", required: false },
33+
{ name: "cf_city", type: "string", required: false },
34+
{ name: "cf_region", type: "string", required: false },
35+
{ name: "cf_latitude", type: "double", required: false },
36+
{ name: "cf_longitude", type: "double", required: false },
37+
{ name: "cf_timezone", type: "string", required: false },
38+
{ name: "duration", type: "double", required: false },
39+
{ name: "request_length", type: "long", required: false },
40+
{ name: "status", type: "int", required: false },
41+
{ name: "ip", type: "string", required: false },
42+
{ name: "is_stream", type: "boolean", required: false },
43+
{ name: "session", type: "string", required: false },
44+
{ name: "request", type: "string", required: false },
3145
{ name: "client", type: "string", required: false },
46+
{ name: "user_agent", type: "string", required: false },
47+
{ name: "model_variant", type: "string", required: false },
3248
{ name: "source", type: "string", required: false },
33-
{ name: "tier", type: "string", required: false },
3449
{ name: "provider", type: "string", required: false },
3550
{ name: "provider_model", type: "string", required: false },
3651
{ name: "model", type: "string", required: false },
37-
{ name: "session", type: "string", required: false },
38-
{ name: "request", type: "string", required: false },
39-
{ name: "user_agent", type: "string", required: false },
40-
{ name: "ip", type: "string", required: false },
41-
{ name: "status", type: "int", required: false },
42-
{ name: "is_stream", type: "boolean", required: false },
43-
{ name: "duration_ms", type: "long", required: false },
44-
{ name: "ttfb_ms", type: "long", required: false },
45-
{ name: "request_length", type: "long", required: false },
52+
{ name: "llm_error_code", type: "int", required: false },
53+
{ name: "llm_error_message", type: "string", required: false },
54+
{ name: "error_response", type: "string", required: false },
55+
{ name: "error_type", type: "string", required: false },
56+
{ name: "error_message", type: "string", required: false },
57+
{ name: "error_cause", type: "string", required: false },
58+
{ name: "error_cause2", type: "string", required: false },
59+
{ name: "api_key", type: "string", required: false },
60+
{ name: "workspace", type: "string", required: false },
61+
{ name: "is_subscription", type: "boolean", required: false },
62+
{ name: "subscription", type: "string", required: false },
4663
{ name: "response_length", type: "long", required: false },
64+
{ name: "time_to_first_byte", type: "long", required: false },
4765
{ name: "timestamp_first_byte", type: "long", required: false },
4866
{ name: "timestamp_last_byte", type: "long", required: false },
4967
{ name: "tokens_input", type: "long", required: false },
@@ -52,20 +70,17 @@ const inferenceEventTable = new aws.s3tables.Table("LakeInferenceEventTable", {
5270
{ name: "tokens_cache_read", type: "long", required: false },
5371
{ name: "tokens_cache_write_5m", type: "long", required: false },
5472
{ name: "tokens_cache_write_1h", type: "long", required: false },
55-
{ name: "tokens_total", type: "long", required: false },
5673
{ name: "cost_input_microcents", type: "long", required: false },
5774
{ name: "cost_output_microcents", type: "long", required: false },
5875
{ name: "cost_cache_read_microcents", type: "long", required: false },
5976
{ name: "cost_cache_write_microcents", type: "long", required: false },
6077
{ name: "cost_total_microcents", type: "long", required: false },
61-
{ name: "output_tps", type: "double", required: false },
62-
{ name: "cf_continent", type: "string", required: false },
63-
{ name: "cf_country", type: "string", required: false },
64-
{ name: "cf_city", type: "string", required: false },
65-
{ name: "cf_region", type: "string", required: false },
66-
{ name: "cf_latitude", type: "double", required: false },
67-
{ name: "cf_longitude", type: "double", required: false },
68-
{ name: "cf_timezone", type: "string", required: false },
78+
{ name: "cost_input", type: "long", required: false },
79+
{ name: "cost_output", type: "long", required: false },
80+
{ name: "cost_cache_read", type: "long", required: false },
81+
{ name: "cost_cache_write_5m", type: "long", required: false },
82+
{ name: "cost_cache_write_1h", type: "long", required: false },
83+
{ name: "cost_total", type: "long", required: false },
6984
],
7085
},
7186
},

‎packages/console/function/src/log-processor.ts‎

Lines changed: 43 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -78,75 +78,68 @@ export default {
7878
}
7979

8080
function toLakeEvent(time: string, data: Record<string, unknown>) {
81-
const tokensInput = integer(data, "tokens.input")
82-
const tokensOutput = integer(data, "tokens.output")
83-
const tokensReasoning = integer(data, "tokens.reasoning")
84-
const tokensCacheRead = integer(data, "tokens.cache_read")
85-
const tokensCacheWrite5m = integer(data, "tokens.cache_write_5m")
86-
const tokensCacheWrite1h = integer(data, "tokens.cache_write_1h")
87-
const timestampFirstByte = integer(data, "timestamp.first_byte")
88-
const timestampLastByte = integer(data, "timestamp.last_byte")
89-
const source = string(data, "source")
90-
9181
return {
9282
_datalake_key: "inference.event",
9383
event_timestamp: time,
9484
event_date: time.slice(0, 10),
9585
event_type: string(data, "event_type"),
9686
dataset: "zen",
87+
cf_continent: string(data, "cf.continent"),
88+
cf_country: string(data, "cf.country"),
89+
cf_city: string(data, "cf.city"),
90+
cf_region: string(data, "cf.region"),
91+
cf_latitude: number(data, "cf.latitude"),
92+
cf_longitude: number(data, "cf.longitude"),
93+
cf_timezone: string(data, "cf.timezone"),
94+
duration: number(data, "duration"),
95+
request_length: integer(data, "request_length"),
96+
status: integer(data, "status"),
97+
ip: string(data, "ip"),
98+
is_stream: boolean(data, "is_stream"),
99+
session: string(data, "session"),
100+
request: string(data, "request"),
97101
client: string(data, "client"),
98-
source,
99-
tier: source,
102+
user_agent: string(data, "user_agent"),
103+
model_variant: string(data, "model.variant"),
104+
source: string(data, "source"),
100105
provider: string(data, "provider"),
101106
provider_model: string(data, "provider.model"),
102107
model: string(data, "model"),
103-
session: string(data, "session"),
104-
request: string(data, "request"),
105-
user_agent: string(data, "user_agent"),
106-
ip: string(data, "ip"),
107-
status: integer(data, "status"),
108-
is_stream: boolean(data, "is_stream"),
109-
duration_ms: integer(data, "duration"),
110-
ttfb_ms: integer(data, "time_to_first_byte"),
111-
request_length: integer(data, "request_length"),
108+
llm_error_code: integer(data, "llm.error.code"),
109+
llm_error_message: string(data, "llm.error.message"),
110+
error_response: string(data, "error.response"),
111+
error_type: string(data, "error.type"),
112+
error_message: string(data, "error.message"),
113+
error_cause: string(data, "error.cause"),
114+
error_cause2: string(data, "error.cause2"),
115+
api_key: string(data, "api_key"),
116+
workspace: string(data, "workspace"),
117+
is_subscription: boolean(data, "isSubscription"),
118+
subscription: string(data, "subscription"),
112119
response_length: integer(data, "response_length"),
113-
timestamp_first_byte: timestampFirstByte,
114-
timestamp_last_byte: timestampLastByte,
115-
tokens_input: tokensInput,
116-
tokens_output: tokensOutput,
117-
tokens_reasoning: tokensReasoning,
118-
tokens_cache_read: tokensCacheRead,
119-
tokens_cache_write_5m: tokensCacheWrite5m,
120-
tokens_cache_write_1h: tokensCacheWrite1h,
121-
tokens_total:
122-
integer(data, "tokens") ??
123-
(tokensInput ?? 0) +
124-
(tokensOutput ?? 0) +
125-
(tokensReasoning ?? 0) +
126-
(tokensCacheRead ?? 0) +
127-
(tokensCacheWrite5m ?? 0) +
128-
(tokensCacheWrite1h ?? 0),
120+
time_to_first_byte: integer(data, "time_to_first_byte"),
121+
timestamp_first_byte: integer(data, "timestamp.first_byte"),
122+
timestamp_last_byte: integer(data, "timestamp.last_byte"),
123+
tokens_input: integer(data, "tokens.input"),
124+
tokens_output: integer(data, "tokens.output"),
125+
tokens_reasoning: integer(data, "tokens.reasoning"),
126+
tokens_cache_read: integer(data, "tokens.cache_read"),
127+
tokens_cache_write_5m: integer(data, "tokens.cache_write_5m"),
128+
tokens_cache_write_1h: integer(data, "tokens.cache_write_1h"),
129129
cost_input_microcents: integer(data, "cost.input.microcents"),
130130
cost_output_microcents: integer(data, "cost.output.microcents"),
131131
cost_cache_read_microcents: integer(data, "cost.cache_read.microcents"),
132132
cost_cache_write_microcents: integer(data, "cost.cache_write.microcents"),
133133
cost_total_microcents: integer(data, "cost.total.microcents"),
134-
output_tps: number(data, "tps.output") ?? outputTps(tokensOutput, timestampFirstByte, timestampLastByte),
135-
cf_continent: string(data, "cf.continent"),
136-
cf_country: string(data, "cf.country"),
137-
cf_city: string(data, "cf.city"),
138-
cf_region: string(data, "cf.region"),
139-
cf_latitude: number(data, "cf.latitude"),
140-
cf_longitude: number(data, "cf.longitude"),
141-
cf_timezone: string(data, "cf.timezone"),
134+
cost_input: integer(data, "cost.input"),
135+
cost_output: integer(data, "cost.output"),
136+
cost_cache_read: integer(data, "cost.cache_read"),
137+
cost_cache_write_5m: integer(data, "cost.cache_write_5m"),
138+
cost_cache_write_1h: integer(data, "cost.cache_write_1h"),
139+
cost_total: integer(data, "cost.total"),
142140
}
143141
}
144142

145-
function outputTps(tokens: number | undefined, firstByte: number | undefined, lastByte: number | undefined) {
146-
if (!tokens || !firstByte || !lastByte || lastByte <= firstByte) return undefined
147-
return Number(((tokens / (lastByte - firstByte)) * 1000).toFixed(6))
148-
}
149-
150143
function string(data: Record<string, unknown>, key: string) {
151144
const value = data[key]
152145
if (typeof value === "string") return value

‎packages/stats/core/src/domain/inference.ts‎

Lines changed: 26 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -55,30 +55,46 @@ export function buildStatsQuery(periodStart: Date, periodEnd: Date, dimension: S
5555
WITH filtered AS (
5656
SELECT
5757
from_iso8601_timestamp(event_timestamp) AS event_time,
58-
COALESCE(NULLIF(tier, ''), 'unknown') AS tier,
59-
COALESCE(NULLIF(provider, ''), 'unknown') AS provider,
58+
CASE
59+
WHEN source = 'lite' THEN 'Go'
60+
WHEN model IN ('gpt-5-nano', 'grok-code', 'big-pickle') OR model LIKE '%-free' THEN 'Free'
61+
ELSE 'Paid'
62+
END AS tier,
63+
COALESCE(NULLIF(
64+
CASE
65+
WHEN starts_with(provider, 'minimax-plan') THEN 'minimax-plan'
66+
WHEN starts_with(provider, 'zai-plan') THEN 'zai-plan'
67+
WHEN starts_with(provider, 'azure-databricks') THEN 'azure-databricks'
68+
WHEN regexp_like(provider, '^azure[0-9]+') THEN 'azure-openai'
69+
ELSE provider
70+
END,
71+
''
72+
), 'unknown') AS provider,
6073
COALESCE(NULLIF(provider_model, ''), '') AS provider_model,
6174
COALESCE(NULLIF(model, ''), 'unknown') AS model,
6275
UPPER(COALESCE(NULLIF(cf_country, ''), 'ZZ')) AS country,
6376
COALESCE(NULLIF(cf_continent, ''), '') AS continent,
6477
session,
6578
status,
66-
duration_ms,
67-
ttfb_ms,
68-
output_tps,
79+
duration AS duration_ms,
80+
time_to_first_byte AS ttfb_ms,
81+
CASE
82+
WHEN timestamp_last_byte - timestamp_first_byte < 100 THEN null
83+
ELSE CAST(tokens_output AS double) / (timestamp_last_byte - timestamp_first_byte) * 1000
84+
END AS output_tps,
6985
tokens_input,
7086
tokens_output,
7187
tokens_reasoning,
7288
tokens_cache_read,
73-
tokens_total,
74-
cost_input_microcents,
75-
cost_output_microcents,
76-
cost_total_microcents
89+
COALESCE(tokens_cache_read, 0) + COALESCE(tokens_cache_write_5m, 0) + COALESCE(tokens_input, 0) + COALESCE(tokens_output, 0) AS tokens_total,
90+
COALESCE(cost_input_microcents, cost_input * 1000000) AS cost_input_microcents,
91+
COALESCE(cost_output_microcents, cost_output * 1000000) AS cost_output_microcents,
92+
COALESCE(cost_total_microcents, cost_total * 1000000) AS cost_total_microcents
7793
FROM ${sourceTable}
7894
WHERE event_type = 'completions'
7995
AND model IS NOT NULL
8096
AND model <> ''
81-
AND user_agent LIKE '%opencode%'
97+
AND (strpos(COALESCE(user_agent, ''), 'ai-sdk') > 0 OR strpos(COALESCE(user_agent, ''), 'opencode') > 0)
8298
AND event_timestamp >= ${periodStartValue}
8399
AND event_timestamp < ${periodEndValue}
84100
), daily AS (

‎sst-env.d.ts‎

Lines changed: 9 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -155,22 +155,7 @@ declare module "sst" {
155155
"type": "sst.sst.Linkable"
156156
"value": string
157157
}
158-
"Stat": {
159-
"type": "sst.cloudflare.Worker"
160-
"url": string
161-
}
162-
"UpstashRedisRestToken": {
163-
"type": "sst.sst.Secret"
164-
"value": string
165-
}
166-
"UpstashRedisRestUrl": {
167-
"type": "sst.sst.Secret"
168-
"value": string
169-
}
170-
"Stats": {
171-
"type": "sst.cloudflare.SolidStart"
172-
"url": string
173-
}
158+
"Stat": import("@cloudflare/workers-types").Service
174159
"StatsDatabase": {
175160
"database": string
176161
"host": string
@@ -192,6 +177,14 @@ declare module "sst" {
192177
"type": "sst.cloudflare.SolidStart"
193178
"url": string
194179
}
180+
"UpstashRedisRestToken": {
181+
"type": "sst.sst.Secret"
182+
"value": string
183+
}
184+
"UpstashRedisRestUrl": {
185+
"type": "sst.sst.Secret"
186+
"value": string
187+
}
195188
"Web": {
196189
"type": "sst.cloudflare.Astro"
197190
"url": string

0 commit comments

Comments
 (0)