Skip to content

Commit 961ff4a

Browse files
committed
new metrics for game server
1 parent 20132f7 commit 961ff4a

12 files changed

Lines changed: 1066 additions & 3643 deletions

File tree

apps/game-server/package.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,9 @@
2323
"@mp/keycloak": "workspace:*",
2424
"@mp/logger": "workspace:*",
2525
"@mp/std": "workspace:*",
26+
"@mp/telemetry": "workspace:*",
2627
"@mp/world": "workspace:*",
28+
"@rift/core": "workspace:*",
2729
"@rift/wss": "workspace:*",
2830
"dotenv": "catalog:",
2931
"ws": "8.20.0"

apps/game-server/src/main.ts

Lines changed: 73 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import {
3333
import * as fixtures from "@mp/fixtures";
3434
import "dotenv/config";
3535
import { opt } from "./options";
36+
import { attachRiftMetrics } from "./metrics";
3637
import type { UserSessionIdentity } from "@mp/world";
3738

3839
const shutdownCleanups: Array<() => unknown> = [];
@@ -64,39 +65,7 @@ logger.info(`Loaded ${areas.size} area(s)`);
6465

6566
const userByWs = new WeakMap<WebSocket, UserSessionIdentity>();
6667

67-
const httpServer = createHttpServer((req, res) => {
68-
if (req.url === "/health") {
69-
res.writeHead(200);
70-
res.end("ok");
71-
return;
72-
}
73-
res.writeHead(404);
74-
res.end();
75-
});
76-
7768
const wss = new WebSocketServer({ noServer: true });
78-
79-
httpServer.on("upgrade", async (req, socket, head) => {
80-
const url = new URL(req.url ?? "", "ws://x");
81-
const token = url.searchParams.get(wsHandshakeAccessTokenParam);
82-
if (!token) {
83-
socket.write("HTTP/1.1 401 Unauthorized\r\n\r\n");
84-
socket.destroy();
85-
return;
86-
}
87-
const result = await resolveAccessToken(token as AccessToken);
88-
if (result.isErr()) {
89-
socket.write("HTTP/1.1 401 Unauthorized\r\n\r\n");
90-
socket.destroy();
91-
return;
92-
}
93-
wss.handleUpgrade(req, socket, head, (ws) => {
94-
const { id, roles, name } = result.value;
95-
userByWs.set(ws, { id, roles, name });
96-
wss.emit("connection", ws, req);
97-
});
98-
});
99-
10069
const transport = wssTransport(wss);
10170
const registry = new SessionRegistry();
10271
const itemLookup = createItemDefinitionLookup(
@@ -167,10 +136,65 @@ transport.on((event) => {
167136
}
168137
});
169138

139+
const metrics = attachRiftMetrics(riftServer);
140+
141+
const httpServer = createHttpServer(async (req, res) => {
142+
switch (req.url) {
143+
case "/health":
144+
res.writeHead(200);
145+
res.end("ok");
146+
break;
147+
148+
case "/metrics":
149+
if (!isLoopbackOrPrivate(req.socket.remoteAddress)) {
150+
res.writeHead(403);
151+
res.end();
152+
break;
153+
}
154+
155+
try {
156+
const body = await metrics.renderText();
157+
res.writeHead(200, { "Content-Type": "text/plain; version=0.0.4" });
158+
res.end(body);
159+
} catch (err: unknown) {
160+
logger.error(err, "Failed to render metrics");
161+
res.writeHead(500);
162+
res.end();
163+
}
164+
break;
165+
default:
166+
res.writeHead(404);
167+
res.end();
168+
break;
169+
}
170+
});
171+
172+
httpServer.on("upgrade", async (req, socket, head) => {
173+
const url = new URL(req.url ?? "", "ws://x");
174+
const token = url.searchParams.get(wsHandshakeAccessTokenParam);
175+
if (!token) {
176+
socket.write("HTTP/1.1 401 Unauthorized\r\n\r\n");
177+
socket.destroy();
178+
return;
179+
}
180+
const result = await resolveAccessToken(token as AccessToken);
181+
if (result.isErr()) {
182+
socket.write("HTTP/1.1 401 Unauthorized\r\n\r\n");
183+
socket.destroy();
184+
return;
185+
}
186+
wss.handleUpgrade(req, socket, head, (ws) => {
187+
const { id, roles, name } = result.value;
188+
userByWs.set(ws, { id, roles, name });
189+
wss.emit("connection", ws, req);
190+
});
191+
});
192+
170193
httpServer.listen(opt.port, opt.hostname);
171194
logger.info(`game service connected on ${opt.hostname}:${opt.port}`);
172195

173196
shutdownCleanups.push(async () => {
197+
metrics.dispose();
174198
await riftServer.dispose();
175199
for (const client of wss.clients) {
176200
client.terminate();
@@ -186,3 +210,20 @@ shutdownCleanups.push(async () => {
186210
// full-reload, which does not run dispose hooks.
187211
import.meta.hot?.accept();
188212
import.meta.hot?.dispose(() => Promise.all(shutdownCleanups.map((fn) => fn())));
213+
214+
function isLoopbackOrPrivate(addr: string | undefined): boolean {
215+
if (!addr) {
216+
return false;
217+
}
218+
return (
219+
addr.startsWith("127.") ||
220+
addr.startsWith("10.") ||
221+
addr.startsWith("172.") ||
222+
addr.startsWith("192.168.") ||
223+
addr === "::1" ||
224+
addr.startsWith("::ffff:127.") ||
225+
addr.startsWith("::ffff:10.") ||
226+
addr.startsWith("::ffff:172.") ||
227+
addr.startsWith("::ffff:192.168.")
228+
);
229+
}

apps/game-server/src/metrics.ts

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
import {
2+
ClientConnected,
3+
ClientDisconnected,
4+
Opcode,
5+
PacketReceived,
6+
PacketSent,
7+
RiftCloseCode,
8+
type RiftServer,
9+
Tick,
10+
TickCompleted,
11+
} from "@rift/core";
12+
import {
13+
collectDefaultMetrics,
14+
exponentialBuckets,
15+
MetricsCounter,
16+
MetricsGague,
17+
MetricsHistogram,
18+
metricsRegister,
19+
} from "@mp/telemetry/prom";
20+
21+
export interface AttachedRiftMetrics {
22+
readonly renderText: () => Promise<string>;
23+
readonly dispose: () => void;
24+
}
25+
26+
// Attaches a passive observer to a RiftServer that derives prom-client metrics
27+
// purely from the server's existing event bus and getters. No coupling back
28+
// into rift internals — if the observable surface ever grows, this is the
29+
// only place that needs to learn about it.
30+
export function attachRiftMetrics(server: RiftServer): AttachedRiftMetrics {
31+
collectDefaultMetrics({ register: metricsRegister });
32+
33+
const ticksTotal = new MetricsCounter({
34+
name: "rift_ticks_total",
35+
help: "Cumulative count of server ticks observed.",
36+
});
37+
38+
// Histogram of wall-clock spacing between consecutive ticks. A healthy
39+
// tick loop produces values near 1/tickRateHz; sustained higher values
40+
// mean the loop is overrunning its budget.
41+
const tickIntervalSeconds = new MetricsHistogram({
42+
name: "rift_tick_interval_seconds",
43+
help: "Wall-clock seconds between consecutive ticks.",
44+
buckets: exponentialBuckets(0.001, 2, 12),
45+
});
46+
47+
// Histogram of how long the tick body itself spent (event drain + delta
48+
// build + per-client flush). Headroom = tick_interval - tick_duration.
49+
const tickDurationSeconds = new MetricsHistogram({
50+
name: "rift_tick_duration_seconds",
51+
help: "Wall-clock seconds spent inside one tick.",
52+
buckets: exponentialBuckets(0.0005, 2, 12),
53+
});
54+
55+
const clientsConnected = new MetricsGague({
56+
name: "rift_clients_connected",
57+
help: "Number of clients currently tracked by the server (sampled per tick).",
58+
});
59+
60+
const entities = new MetricsGague({
61+
name: "rift_entities",
62+
help: "Number of entities currently in the world (sampled per tick).",
63+
});
64+
65+
const connectionsOpenedTotal = new MetricsCounter({
66+
name: "rift_client_connections_opened_total",
67+
help: "Cumulative count of clients that completed the handshake.",
68+
});
69+
70+
const connectionsClosedTotal = new MetricsCounter({
71+
name: "rift_client_connections_closed_total",
72+
help: "Cumulative count of post-handshake disconnections, labeled by rift close code.",
73+
labelNames: ["code"] as const,
74+
});
75+
76+
// event_index is the schema-assigned position. Stable for the lifetime of
77+
// a schema version; if the schema changes order, series rotate. A future
78+
// pass can layer human names on top via a Map<RiftType, string>.
79+
const eventsSentTotal = new MetricsCounter({
80+
name: "rift_events_sent_total",
81+
help: "Cumulative count of events dispatched toward connected clients.",
82+
labelNames: ["event_index"] as const,
83+
});
84+
85+
const eventsReceivedTotal = new MetricsCounter({
86+
name: "rift_events_received_total",
87+
help: "Cumulative count of events received from connected clients.",
88+
labelNames: ["event_index"] as const,
89+
});
90+
91+
const bytesSentTotal = new MetricsCounter({
92+
name: "rift_bytes_sent_total",
93+
help: "Total bytes dispatched to clients (byteSize * recipientCount), labeled by opcode.",
94+
labelNames: ["opcode"] as const,
95+
});
96+
97+
const bytesReceivedTotal = new MetricsCounter({
98+
name: "rift_bytes_received_total",
99+
help: "Total bytes received from clients, labeled by opcode.",
100+
labelNames: ["opcode"] as const,
101+
});
102+
103+
const packetSizeBytes = new MetricsHistogram({
104+
name: "rift_packet_size_bytes",
105+
help: "Size distribution of outbound packets (one observation per logical send, regardless of recipientCount).",
106+
buckets: exponentialBuckets(32, 2, 14),
107+
labelNames: ["opcode"] as const,
108+
});
109+
110+
let lastTickAt = 0;
111+
112+
const unsubs: Array<() => void> = [
113+
server.on(Tick, () => {
114+
ticksTotal.inc();
115+
const now = performance.now();
116+
if (lastTickAt !== 0) {
117+
tickIntervalSeconds.observe((now - lastTickAt) / 1000);
118+
}
119+
lastTickAt = now;
120+
clientsConnected.set(server.getClientIds().length);
121+
entities.set(server.world.entities().size);
122+
}),
123+
server.on(TickCompleted, (ev) => {
124+
tickDurationSeconds.observe(ev.data.durationUs / 1_000_000);
125+
}),
126+
server.on(ClientConnected, () => {
127+
connectionsOpenedTotal.inc();
128+
}),
129+
server.on(ClientDisconnected, (ev) => {
130+
connectionsClosedTotal.inc({ code: closeCodeLabel(ev.data.code) });
131+
}),
132+
server.on(PacketSent, (ev) => {
133+
const opcode = opcodeLabel(ev.data.opcode);
134+
bytesSentTotal.inc(
135+
{ opcode },
136+
ev.data.byteSize * ev.data.recipientCount,
137+
);
138+
packetSizeBytes.observe({ opcode }, ev.data.byteSize);
139+
}),
140+
server.on(PacketReceived, (ev) => {
141+
bytesReceivedTotal.inc(
142+
{ opcode: opcodeLabel(ev.data.opcode) },
143+
ev.data.byteSize,
144+
);
145+
}),
146+
server.onAny((ev) => {
147+
// source.type === "wire" means the event came in from a client;
148+
// target.type === "wire" means it's being dispatched out to clients.
149+
// Lifecycle events (Tick/Packet*/ClientConnected/ClientDisconnected)
150+
// are local on both ends and are accounted for above instead.
151+
if (ev.source.type === "wire") {
152+
eventsReceivedTotal.inc({
153+
event_index: eventIndexLabel(server, ev.type),
154+
});
155+
} else if (ev.target.type === "wire") {
156+
eventsSentTotal.inc({
157+
event_index: eventIndexLabel(server, ev.type),
158+
});
159+
}
160+
}),
161+
];
162+
163+
return {
164+
renderText: () => metricsRegister.metrics(),
165+
dispose: () => {
166+
for (const u of unsubs) {
167+
u();
168+
}
169+
},
170+
};
171+
}
172+
173+
function eventIndexLabel(
174+
server: RiftServer,
175+
ty: Parameters<RiftServer["schema"]["eventIndexOf"]>[0],
176+
): string {
177+
const idx = server.schema.eventIndexOf(ty);
178+
return idx === undefined ? "unknown" : String(idx);
179+
}
180+
181+
function opcodeLabel(opcode: Opcode): string {
182+
switch (opcode) {
183+
case Opcode.Hello:
184+
return "hello";
185+
case Opcode.Accept:
186+
return "accept";
187+
case Opcode.Delta:
188+
return "delta";
189+
case Opcode.EventFromClient:
190+
return "event_from_client";
191+
case Opcode.EventToClient:
192+
return "event_to_client";
193+
default:
194+
return String(opcode);
195+
}
196+
}
197+
198+
function closeCodeLabel(code: RiftCloseCode): string {
199+
switch (code) {
200+
case RiftCloseCode.Normal:
201+
return "normal";
202+
case RiftCloseCode.SchemaMismatch:
203+
return "schema_mismatch";
204+
case RiftCloseCode.ProtocolError:
205+
return "protocol_error";
206+
case RiftCloseCode.HandshakeTimeout:
207+
return "handshake_timeout";
208+
case RiftCloseCode.ServerShutdown:
209+
return "server_shutdown";
210+
default:
211+
return String(code);
212+
}
213+
}

docker/grafana/alloy.alloy

Lines changed: 3 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,9 @@
1-
prometheus.scrape "mp_api" {
1+
prometheus.scrape "mp_game_server" {
22
targets = [{
3-
__address__ = env("MP_API_PROXY_HOST"),
3+
__address__ = env("MP_GAME_SERVER_PROXY_HOST"),
44
}]
55
forward_to = [prometheus.remote_write.default.receiver]
6-
job_name = "mp_api"
7-
scrape_interval = env("PROMETHEUS_SCRAPE_INTERVAL")
8-
scrape_timeout = env("PROMETHEUS_SCRAPE_TIMEOUT")
9-
}
10-
11-
prometheus.scrape "mp_gateway" {
12-
targets = [{
13-
__address__ = env("MP_GATEWAY_PROXY_HOST"),
14-
}]
15-
forward_to = [prometheus.remote_write.default.receiver]
16-
job_name = "mp_gateway"
6+
job_name = "mp_game_server"
177
scrape_interval = env("PROMETHEUS_SCRAPE_INTERVAL")
188
scrape_timeout = env("PROMETHEUS_SCRAPE_TIMEOUT")
199
}

0 commit comments

Comments
 (0)