Skip to content

Commit 22acc22

Browse files
OgeonX-AiAitomates
andauthored
feat: add tested local chat proxy (#20)
* feat: add tested local chat proxy * test: stabilize proxy and tournament integration * test: make event-store soak deterministic on windows --------- Co-authored-by: Kim Harjamäki <kim.harjamaki@prosimo.fi>
1 parent fe977e9 commit 22acc22

4 files changed

Lines changed: 314 additions & 3 deletions

File tree

universal-refiner/scripts/stress/event-store-soak-worker.mjs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,14 @@ import { EventStore } from "../../dist/src/history/event-store.js";
22

33
const workerId = process.argv[2];
44
const durationMs = Number.parseInt(process.argv[3] || "10000", 10);
5+
const minOperations = Number.parseInt(process.argv[4] || "0", 10);
56
const deadline = Date.now() + durationMs;
67
const counts = { events: 0, prompts: 0, executions: 0, operations: 0 };
78
const store = EventStore.getInstance();
89
let index = 0;
910
let lastPromptId;
1011

11-
while (Date.now() < deadline) {
12+
while (Date.now() < deadline || counts.operations < minOperations) {
1213
const id = `soak-${workerId}-${index}`;
1314
const operation = index % 3;
1415

universal-refiner/scripts/stress/event-store-soak.mjs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,14 @@ export async function runEventStoreSoak(options = {}) {
2525
const workers = readPositiveInteger(options.workers, 4, "workers");
2626
const durationMs = readPositiveInteger(options.durationMs, 10_000, "durationMs");
2727
const minOperations = readPositiveInteger(options.minOperations, workers * 10, "minOperations");
28+
const minOperationsPerWorker = Math.ceil(minOperations / workers);
2829
const maxLossRatio = readRatio(options.maxLossRatio, 0, "maxLossRatio");
2930
const directory = await mkdtemp(join(tmpdir(), "prompt-refiner-soak-"));
3031
const databasePath = join(directory, "events.db");
3132

3233
try {
3334
const results = await Promise.all(Array.from({ length: workers }, async (_, index) => {
34-
const result = await runProcess(process.execPath, [workerScript, String(index), String(durationMs)], {
35+
const result = await runProcess(process.execPath, [workerScript, String(index), String(durationMs), String(minOperationsPerWorker)], {
3536
cwd: repoRoot,
3637
env: { ...process.env, PROMPT_REFINER_GLOBAL_DIR: directory, PROMPT_REFINER_LOG_LEVEL: "error" },
3738
timeoutMs: durationMs + 30_000,

universal-refiner/src/core/dashboard.ts

Lines changed: 105 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,12 @@ import { streamSSE } from "hono/streaming";
99
import { getDisplayVersion } from "./version.js";
1010
import { RuntimeLogger } from "./logger.js";
1111
import { ConfigManager } from "./config.js";
12+
import { randomUUID } from "crypto";
1213

1314
import { TimelineProvider } from "../history/timeline.js";
1415
import { EventStore } from "../history/event-store.js";
1516
import { AutoPilotStatus } from "./autopilot-status.js";
1617
import { createABEvaluationRecord } from "../evaluation/prompt-evaluator.js";
17-
import { randomUUID } from "crypto";
1818

1919
const __dirname = path.dirname(fileURLToPath(import.meta.url));
2020

@@ -394,6 +394,110 @@ export class CommandCenterDashboard {
394394
return c.json({ error: "Provider health unavailable" }, 500);
395395
}
396396
});
397+
app.post("/proxy/v1/chat/completions", async (c) => {
398+
const selectedPath = this.resolveSelectedPath(c.req.query("project"));
399+
try {
400+
if (!isJsonContentType(c.req.header("content-type"))) {
401+
return c.json({ error: "Proxy requests must use application/json" }, 415);
402+
}
403+
404+
const config = ConfigManager.getSemanticConfig(selectedPath);
405+
let upstreamBase: URL;
406+
try {
407+
upstreamBase = new URL(config.baseUrl.endsWith("/") ? config.baseUrl : `${config.baseUrl}/`);
408+
} catch {
409+
return c.json({ error: "Configured semantic baseUrl is invalid" }, 400);
410+
}
411+
if (!config.allowNonLoopback && !isLoopbackHost(upstreamBase.hostname)) {
412+
return c.json({ error: "Proxy upstream must be loopback unless allowNonLoopback is enabled" }, 403);
413+
}
414+
415+
let body: any;
416+
try {
417+
body = await c.req.json();
418+
} catch {
419+
return c.json({ error: "Proxy request body must be valid JSON" }, 400);
420+
}
421+
422+
const messages = Array.isArray(body.messages) ? body.messages : [];
423+
const lastMessage = messages[messages.length - 1];
424+
const rawPrompt = typeof lastMessage?.content === "string" && lastMessage.content.trim()
425+
? lastMessage.content
426+
: "Unknown proxy prompt";
427+
428+
const store = EventStore.getInstance();
429+
const repoId = store.ensureRepository(selectedPath).id;
430+
const promptId = `prm_${randomUUID()}`;
431+
const execId = `exec_${randomUUID()}`;
432+
store.recordPrompt({
433+
id: promptId,
434+
client: "API_PROXY",
435+
agent_name: "ProxyClient",
436+
raw_prompt: rawPrompt,
437+
repo_id: repoId,
438+
});
439+
440+
const upstreamUrl = new URL("chat/completions", upstreamBase).toString();
441+
let upstreamResponse: Response;
442+
try {
443+
upstreamResponse = await fetch(upstreamUrl, {
444+
method: "POST",
445+
headers: {
446+
"Content-Type": "application/json",
447+
...(c.req.header("authorization") ? { "Authorization": c.req.header("authorization") as string } : {}),
448+
},
449+
body: JSON.stringify(body),
450+
});
451+
} catch (error) {
452+
const message = error instanceof Error ? error.message : String(error);
453+
store.recordExecution({
454+
id: execId,
455+
prompt_id: promptId,
456+
workflow_name: "proxy_forward",
457+
executor_name: "LocalLLM",
458+
status: "failed",
459+
result_summary: `Network error reaching upstream: ${message}`,
460+
artifacts_json: JSON.stringify({ error: redactSensitive(message) }),
461+
});
462+
return c.json({ error: "Proxy request failed" }, 502);
463+
}
464+
465+
if (!upstreamResponse.ok) {
466+
const errText = await upstreamResponse.text();
467+
store.recordExecution({
468+
id: execId,
469+
prompt_id: promptId,
470+
workflow_name: "proxy_forward",
471+
executor_name: "LocalLLM",
472+
status: "failed",
473+
result_summary: `Upstream error: ${upstreamResponse.status} ${upstreamResponse.statusText}`,
474+
artifacts_json: JSON.stringify({ error: redactSensitive(errText) }),
475+
});
476+
return new Response(errText, {
477+
status: upstreamResponse.status,
478+
headers: upstreamResponse.headers,
479+
});
480+
}
481+
482+
store.recordExecution({
483+
id: execId,
484+
prompt_id: promptId,
485+
workflow_name: "proxy_forward",
486+
executor_name: "LocalLLM",
487+
status: "completed",
488+
result_summary: `Proxied successfully to ${sanitizeEndpoint(config.baseUrl)}`,
489+
artifacts_json: "{}",
490+
});
491+
492+
return new Response(upstreamResponse.body, {
493+
status: upstreamResponse.status,
494+
headers: upstreamResponse.headers,
495+
});
496+
} catch (error) {
497+
this.logRouteError("proxy/v1/chat/completions", error, selectedPath);
498+
return c.json({ error: "Proxy request failed" }, 502);
499+
}
500+
});
397501

398502
app.get("/api/tournaments", async (c) => {
399503
const selectedPath = this.resolveSelectedPath(c.req.query("project"));
Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
// secret-scan: allow-fixture
2+
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
3+
import * as fs from "node:fs";
4+
import * as os from "node:os";
5+
import * as path from "node:path";
6+
import { CommandCenterDashboard } from "../src/core/dashboard.js";
7+
import { ConfigManager, type SemanticConfig } from "../src/core/config.js";
8+
import { EventStore } from "../src/history/event-store.js";
9+
10+
describe("dashboard OpenAI-compatible proxy route", () => {
11+
let testDir: string;
12+
let repoDir: string;
13+
let store: EventStore;
14+
const semanticConfig: SemanticConfig = {
15+
localEnabled: true,
16+
mcpSamplingEnabled: false,
17+
baseUrl: "http://127.0.0.1:9000/v1",
18+
models: ["gemma3:12b"],
19+
timeoutMs: 1_000,
20+
temperature: 0.2,
21+
allowNonLoopback: false,
22+
};
23+
24+
beforeEach(() => {
25+
testDir = fs.mkdtempSync(path.join(os.tmpdir(), "proxy-route-"));
26+
repoDir = path.join(testDir, "repo");
27+
fs.mkdirSync(repoDir, { recursive: true });
28+
process.env.PROMPT_REFINER_GLOBAL_DIR = path.join(testDir, "global");
29+
(EventStore as unknown as { instance: EventStore | null }).instance = null;
30+
store = EventStore.getInstance();
31+
vi.spyOn(ConfigManager, "getSemanticConfig").mockReturnValue(semanticConfig);
32+
});
33+
34+
afterEach(() => {
35+
vi.restoreAllMocks();
36+
store.close();
37+
(EventStore as unknown as { instance: EventStore | null }).instance = null;
38+
delete process.env.PROMPT_REFINER_GLOBAL_DIR;
39+
fs.rmSync(testDir, { recursive: true, force: true });
40+
});
41+
42+
it("records the prompt, forwards the request, and records a completed execution", async () => {
43+
const fetchMock = vi.fn().mockResolvedValue(new Response("mock LLM response", {
44+
status: 200,
45+
headers: { "content-type": "text/plain" },
46+
}));
47+
vi.stubGlobal("fetch", fetchMock);
48+
const app = CommandCenterDashboard.createApp(repoDir);
49+
50+
const response = await app.request("/proxy/v1/chat/completions", {
51+
method: "POST",
52+
headers: {
53+
"content-type": "application/json",
54+
authorization: "Bearer local-token",
55+
},
56+
body: JSON.stringify({
57+
messages: [{ role: "user", content: "Improve this prompt" }],
58+
}),
59+
});
60+
61+
expect(response.status).toBe(200);
62+
expect(await response.text()).toBe("mock LLM response");
63+
expect(fetchMock).toHaveBeenCalledWith("http://127.0.0.1:9000/v1/chat/completions", expect.objectContaining({
64+
method: "POST",
65+
headers: expect.objectContaining({
66+
"Content-Type": "application/json",
67+
"Authorization": "Bearer local-token",
68+
}),
69+
}));
70+
71+
const db = (store as any).db;
72+
const prompt = db.prepare("SELECT * FROM prompts WHERE client = ?").get("API_PROXY");
73+
expect(prompt.raw_prompt).toBe("Improve this prompt");
74+
const execution = db.prepare("SELECT * FROM executions WHERE prompt_id = ?").get(prompt.id);
75+
expect(execution).toMatchObject({
76+
workflow_name: "proxy_forward",
77+
executor_name: "LocalLLM",
78+
status: "completed",
79+
artifacts_json: "{}",
80+
});
81+
});
82+
83+
it("supports trailing-slash base URLs and falls back when no prompt message is present", async () => {
84+
vi.spyOn(ConfigManager, "getSemanticConfig").mockReturnValueOnce({
85+
...semanticConfig,
86+
baseUrl: "http://127.0.0.1:9000/v1/",
87+
});
88+
const fetchMock = vi.fn().mockResolvedValue(new Response("ok", { status: 200 }));
89+
vi.stubGlobal("fetch", fetchMock);
90+
const app = CommandCenterDashboard.createApp(repoDir);
91+
92+
const response = await app.request("/proxy/v1/chat/completions", {
93+
method: "POST",
94+
headers: { "content-type": "application/json" },
95+
body: JSON.stringify({ messages: "not-an-array" }),
96+
});
97+
98+
expect(response.status).toBe(200);
99+
expect(fetchMock).toHaveBeenCalledWith("http://127.0.0.1:9000/v1/chat/completions", expect.objectContaining({
100+
headers: { "Content-Type": "application/json" },
101+
}));
102+
const db = (store as any).db;
103+
expect(db.prepare("SELECT raw_prompt FROM prompts WHERE client = ?").get("API_PROXY"))
104+
.toEqual({ raw_prompt: "Unknown proxy prompt" });
105+
});
106+
107+
it("rejects malformed proxy requests before forwarding", async () => {
108+
const fetchMock = vi.fn();
109+
vi.stubGlobal("fetch", fetchMock);
110+
const app = CommandCenterDashboard.createApp(repoDir);
111+
112+
expect((await app.request("/proxy/v1/chat/completions", { method: "POST", body: "{}" })).status).toBe(415);
113+
expect((await app.request("/proxy/v1/chat/completions", {
114+
method: "POST",
115+
headers: { "content-type": "application/json" },
116+
body: "{",
117+
})).status).toBe(400);
118+
119+
vi.spyOn(ConfigManager, "getSemanticConfig").mockReturnValueOnce({
120+
...semanticConfig,
121+
baseUrl: "https://remote.example/v1",
122+
allowNonLoopback: false,
123+
});
124+
expect((await app.request("/proxy/v1/chat/completions", {
125+
method: "POST",
126+
headers: { "content-type": "application/json" },
127+
body: "{}",
128+
})).status).toBe(403);
129+
130+
vi.spyOn(ConfigManager, "getSemanticConfig").mockReturnValueOnce({
131+
...semanticConfig,
132+
baseUrl: "not a url",
133+
});
134+
expect((await app.request("/proxy/v1/chat/completions", {
135+
method: "POST",
136+
headers: { "content-type": "application/json" },
137+
body: "{}",
138+
})).status).toBe(400);
139+
expect(fetchMock).not.toHaveBeenCalled();
140+
});
141+
142+
it("records failed executions for upstream HTTP and network failures", async () => {
143+
const app = CommandCenterDashboard.createApp(repoDir);
144+
vi.stubGlobal("fetch", vi.fn()
145+
.mockResolvedValueOnce(new Response("token=secret-value", { status: 503, statusText: "Unavailable" }))
146+
.mockRejectedValueOnce(new Error("connection refused with token=secret-value")));
147+
148+
const upstreamFailure = await app.request("/proxy/v1/chat/completions", {
149+
method: "POST",
150+
headers: { "content-type": "application/json" },
151+
body: JSON.stringify({ messages: [{ role: "user", content: "First" }] }),
152+
});
153+
expect(upstreamFailure.status).toBe(503);
154+
expect(await upstreamFailure.text()).toBe("token=secret-value");
155+
156+
const networkFailure = await app.request("/proxy/v1/chat/completions", {
157+
method: "POST",
158+
headers: { "content-type": "application/json" },
159+
body: JSON.stringify({ messages: [{ role: "user", content: "Second" }] }),
160+
});
161+
expect(networkFailure.status).toBe(502);
162+
expect(await networkFailure.json()).toEqual({ error: "Proxy request failed" });
163+
164+
const db = (store as any).db;
165+
const failures = db.prepare("SELECT * FROM executions WHERE status = 'failed' ORDER BY started_at ASC").all();
166+
expect(failures).toHaveLength(2);
167+
expect(failures[0].result_summary).toContain("Upstream error: 503");
168+
expect(failures[0].artifacts_json).toContain("token=[REDACTED]");
169+
expect(failures[1].result_summary).toContain("Network error reaching upstream");
170+
expect(failures[1].artifacts_json).toContain("token=[REDACTED]");
171+
});
172+
173+
it("records non-Error network failures", async () => {
174+
const app = CommandCenterDashboard.createApp(repoDir);
175+
vi.stubGlobal("fetch", vi.fn().mockRejectedValue("string failure token=secret-value"));
176+
177+
const response = await app.request("/proxy/v1/chat/completions", {
178+
method: "POST",
179+
headers: { "content-type": "application/json" },
180+
body: JSON.stringify({ messages: [{ role: "user", content: "Prompt" }] }),
181+
});
182+
183+
expect(response.status).toBe(502);
184+
const db = (store as any).db;
185+
const failure = db.prepare("SELECT * FROM executions WHERE status = 'failed'").get();
186+
expect(failure.result_summary).toContain("string failure");
187+
expect(failure.artifacts_json).toContain("token=[REDACTED]");
188+
});
189+
190+
it("returns a sanitized route failure when proxy bookkeeping fails", async () => {
191+
const app = CommandCenterDashboard.createApp(repoDir);
192+
vi.spyOn(EventStore, "getInstance").mockImplementationOnce(() => {
193+
throw new Error("store secret");
194+
});
195+
196+
const response = await app.request("/proxy/v1/chat/completions", {
197+
method: "POST",
198+
headers: { "content-type": "application/json" },
199+
body: JSON.stringify({ messages: [] }),
200+
});
201+
202+
expect(response.status).toBe(502);
203+
expect(await response.json()).toEqual({ error: "Proxy request failed" });
204+
});
205+
});

0 commit comments

Comments
 (0)