Telemetry and tuning loop

Author: 7vignesh

README.md

@@ -5,6 +5,8 @@ Users enter a product idea in plain English, and the system coordinates multiple
 
 Currently, this repository contains the **Core Backend & Orchestration Layer** (v1), which features a clean provider abstraction, strict Zod validation, and real-time Server-Sent Events (SSE) streaming for agent progress.
 
+The backend now runs on **real OpenRouter provider calls** with per-agent token optimization (input compression, output caps, and budget guardrails).
+
 ---
 
 ## 🚀 Tech Stack & Tools Needed
@@ -92,10 +94,73 @@ apps/
   web/          # Frontend Web App (React/Next.js stub) 
 
 packages/
-  agents/       # Core Orchestration, 6 Subagents, and LLM Provider mock
+   agents/       # Core Orchestration, 6 Subagents, OpenRouter provider, token optimizer
   shared/       # Zod Schemas, Constant Enums, and TS Contract Types
   ui/           # Reusable UI primitives stub
   config/       # ESLint/TSConfig stubs
 ```
 
+---
+
+## ⚙️ OpenRouter Runtime Configuration
+
+Set these variables in `apps/api/.env`:
+
+```bash
+OPENROUTER_API_KEY=your_key_here
+OPENROUTER_ENDPOINT=https://openrouter.ai/api/v1/chat/completions
+OPENROUTER_APP_NAME=stackforge-api
+OPENROUTER_APP_URL=http://localhost:3001
+```
+
+---
+
+## 📉 Token Tuning Guide
+
+Per-agent tuning lives in `packages/agents/src/config/agent.configs.ts`.
+
+- `maxInputTokens`: hard input cap used by optimizer compression.
+- `maxOutputTokens`: maximum completion tokens requested from provider.
+- `minOutputTokens`: minimum output budget required after compression.
+- `tokenBudget`: total budget target used to derive dynamic output caps.
+- `compressionLevel`: default compression aggressiveness (`low` / `medium` / `high`).
+- `budgetOverflowRetries`: number of extra compression passes before fail-fast.
+
+**Suggested workflow:**
+1. Run 3–5 representative prompts.
+2. Inspect per-agent SSE `agent_completed` telemetry.
+3. Lower `maxInputTokens` or raise `compressionLevel` for agents with high `inputTokens`.
+4. Lower `maxOutputTokens` for agents with consistently low `outputTokens`.
+5. Raise `minOutputTokens` only if quality drops from over-compression.
+
+---
+
+## 📡 SSE Agent Telemetry
+
+Each `agent_completed` event includes token and optimizer metrics:
+
+```json
+{
+   "type": "agent_completed",
+   "agent": "schema",
+   "payload": {
+      "durationMs": 842,
+      "cached": false,
+      "inputTokens": 612,
+      "outputTokens": 431,
+      "totalTokens": 1043,
+      "tokensUsed": 1043,
+      "estimatedInputTokens": 590,
+      "compressionPasses": 2,
+      "providerInputTokens": 612,
+      "providerOutputTokens": 431,
+      "model": "openai/gpt-4o-mini"
+   }
+}
+```
+
+Per-job aggregates are available in REST responses:
+- `GET /api/jobs` returns all jobs with `tokenUsage` summaries.
+- `GET /api/jobs/:jobId` includes the same `tokenUsage` object for a single run.
+
 

apps/api/src/controllers/jobs.controller.ts

@@ -1,8 +1,24 @@
 import type { Request, Response, NextFunction } from "express";
 import { JobIdParamSchema, JOB_STATUS } from "@stackforge/shared";
-import { getJob } from "../store/job.store.js";
+import { getJob, listJobs, summarizeJobTokenUsage } from "../store/job.store.js";
 import { subscribe, unsubscribe } from "../services/sse.service.js";
 
+export function listJobsController(_req: Request, res: Response): void {
+  const jobs = listJobs().map((job) => ({
+    id: job.id,
+    status: job.status,
+    projectName: job.projectName,
+    createdAt: job.createdAt,
+    updatedAt: job.updatedAt,
+    completedAt: job.completedAt,
+    agentsCompleted: job.agentsCompleted,
+    error: job.error,
+    tokenUsage: summarizeJobTokenUsage(job),
+  }));
+
+  res.json({ jobs });
+}
+
 export function getJobController(req: Request, res: Response, next: NextFunction): void {
   const parsed = JobIdParamSchema.safeParse(req.params);
   if (!parsed.success) {
@@ -26,6 +42,7 @@ export function getJobController(req: Request, res: Response, next: NextFunction
     agentsCompleted: job.agentsCompleted,
     error: job.error,
     blueprint: job.blueprint,
+    tokenUsage: summarizeJobTokenUsage(job),
   });
 }
 

apps/api/src/routes/index.ts

@@ -1,10 +1,11 @@
 import { Router, type IRouter } from "express";
 import { generateController } from "../controllers/generate.controller.js";
-import { getJobController, streamController } from "../controllers/jobs.controller.js";
+import { listJobsController, getJobController, streamController } from "../controllers/jobs.controller.js";
 
 const router: IRouter = Router();
 
 router.post("/generate", generateController);
+router.get("/jobs", listJobsController);
 router.get("/jobs/:jobId", getJobController);
 router.get("/stream/:jobId", streamController);
 

apps/api/src/store/job.store.ts

@@ -1,5 +1,10 @@
 import { randomUUID } from "node:crypto";
-import type { Blueprint, SSEEvent, AgentName } from "@stackforge/shared";
+import type {
+  Blueprint,
+  SSEEvent,
+  AgentName,
+  AgentCompletedEvent,
+} from "@stackforge/shared";
 import { JOB_STATUS } from "@stackforge/shared";
 
 export type StoredJob = {
@@ -18,6 +23,62 @@ export type StoredJob = {
 
 const store = new Map<string, StoredJob>();
 
+export type JobTokenUsage = {
+  totalTokens: number;
+  inputTokens: number;
+  outputTokens: number;
+  completionEvents: number;
+  byAgent: Partial<Record<AgentName, {
+    totalTokens: number;
+    inputTokens: number;
+    outputTokens: number;
+    count: number;
+  }>>;
+};
+
+function isAgentCompletedEvent(event: SSEEvent): event is AgentCompletedEvent {
+  return event.type === "agent_completed";
+}
+
+export function summarizeJobTokenUsage(job: StoredJob): JobTokenUsage {
+  const initial: JobTokenUsage = {
+    totalTokens: 0,
+    inputTokens: 0,
+    outputTokens: 0,
+    completionEvents: 0,
+    byAgent: {},
+  };
+
+  for (const event of job.events) {
+    if (!isAgentCompletedEvent(event)) {
+      continue;
+    }
+
+    initial.totalTokens += event.payload.totalTokens;
+    initial.inputTokens += event.payload.inputTokens;
+    initial.outputTokens += event.payload.outputTokens;
+    initial.completionEvents += 1;
+
+    const agent = event.agent as AgentName;
+
+    const existing = initial.byAgent[agent] ?? {
+      totalTokens: 0,
+      inputTokens: 0,
+      outputTokens: 0,
+      count: 0,
+    };
+
+    initial.byAgent[agent] = {
+      totalTokens: existing.totalTokens + event.payload.totalTokens,
+      inputTokens: existing.inputTokens + event.payload.inputTokens,
+      outputTokens: existing.outputTokens + event.payload.outputTokens,
+      count: existing.count + 1,
+    };
+  }
+
+  return initial;
+}
+
 export function createJob(prompt: string, projectName: string): StoredJob {
   const now = new Date().toISOString();
   const job: StoredJob = {
@@ -38,6 +99,10 @@ export function getJob(id: string): StoredJob | undefined {
   return store.get(id);
 }
 
+export function listJobs(): StoredJob[] {
+  return [...store.values()].sort((a, b) => b.createdAt.localeCompare(a.createdAt));
+}
+
 export function updateJob(
   id: string,
   patch: Partial<Omit<StoredJob, "id" | "createdAt" | "events">>,

apps/api/test/integration.test.ts

@@ -52,5 +52,16 @@ describe("StackForge API Integration", () => {
     const jobData = await jobRes.json();
     expect(jobData.id).toBe(data.jobId);
     expect(["queued", "running", "completed", "failed"]).toContain(jobData.status);
+    expect(jobData.tokenUsage).toBeDefined();
+    expect(typeof jobData.tokenUsage.totalTokens).toBe("number");
+
+    const jobsRes = await fetch(`${baseUrl}/api/jobs`);
+    expect(jobsRes.status).toBe(200);
+    const jobsData = await jobsRes.json();
+    expect(Array.isArray(jobsData.jobs)).toBe(true);
+    const createdJob = jobsData.jobs.find((job: { id: string }) => job.id === data.jobId);
+    expect(createdJob).toBeDefined();
+    expect(createdJob.tokenUsage).toBeDefined();
+    expect(typeof createdJob.tokenUsage.totalTokens).toBe("number");
   });
 });

packages/agents/src/agents/base.agent.ts

@@ -10,6 +10,9 @@ export type AgentRunResult<T> = {
   cached: boolean;
   durationMs: number;
   tokensUsed: number;
+  inputTokens: number;
+  outputTokens: number;
+  totalTokens: number;
   estimatedInputTokens: number;
   compressionPasses: number;
   providerInputTokens: number;
@@ -33,6 +36,9 @@ export async function runAgent<TInput, TOutput>(
       cached: true,
       durationMs: 0,
       tokensUsed: 0,
+      inputTokens: 0,
+      outputTokens: 0,
+      totalTokens: 0,
       estimatedInputTokens: 0,
       compressionPasses: 0,
       providerInputTokens: 0,
@@ -67,6 +73,9 @@ export async function runAgent<TInput, TOutput>(
     cached: false,
     durationMs,
     tokensUsed: response.tokensUsed,
+    inputTokens: response.inputTokens ?? optimized.estimatedInputTokens,
+    outputTokens: response.outputTokens ?? optimized.maxOutputTokens,
+    totalTokens: response.tokensUsed,
     estimatedInputTokens: optimized.estimatedInputTokens,
     compressionPasses: optimized.compressionPasses,
     providerInputTokens: response.inputTokens ?? optimized.estimatedInputTokens,

packages/agents/src/orchestrator/orchestrator.service.ts

@@ -35,6 +35,9 @@ function agentCompleted(
   agent: AgentName,
   durationMs: number,
   cached: boolean,
+  inputTokens: number,
+  outputTokens: number,
+  totalTokens: number,
   tokensUsed: number,
   estimatedInputTokens: number,
   compressionPasses: number,
@@ -50,6 +53,9 @@ function agentCompleted(
     payload: {
       durationMs,
       cached,
+      inputTokens,
+      outputTokens,
+      totalTokens,
       tokensUsed,
       estimatedInputTokens,
       compressionPasses,
@@ -72,6 +78,9 @@ async function runWithEmit<T>(
     output: T;
     cached: boolean;
     durationMs: number;
+    inputTokens: number;
+    outputTokens: number;
+    totalTokens: number;
     tokensUsed: number;
     estimatedInputTokens: number;
     compressionPasses: number;
@@ -89,6 +98,9 @@ async function runWithEmit<T>(
         agentName,
         result.durationMs,
         result.cached,
+        result.inputTokens,
+        result.outputTokens,
+        result.totalTokens,
         result.tokensUsed,
         result.estimatedInputTokens,
         result.compressionPasses,

packages/shared/src/schemas/sse.schema.ts

@@ -26,6 +26,9 @@ export const AgentCompletedEventSchema = SSEBaseSchema.extend({
   payload: z.object({
     durationMs: z.number(),
     cached: z.boolean(),
+    inputTokens: z.number(),
+    outputTokens: z.number(),
+    totalTokens: z.number(),
     tokensUsed: z.number(),
     estimatedInputTokens: z.number(),
     compressionPasses: z.number(),