Merge pull request #67 from ChainSafe/feat/metrics

feat: metrics and reward handling

Author: sqhell

autobots/README.md

@@ -146,6 +146,37 @@ sudo journalctl -fu autobots | grep -E "STATUS|CRITICAL"
 {container="autobots"} |~ "CRITICAL|WARN"         # anything needing attention
 ```
 
+## Growth Mode — simulate organic user growth
+
+Instead of a single pre-provisioned `--key`, growth mode mints new Canton parties on-the-fly via the billing portal and runs each one as an independent bot on its own task loop. Both the number of bots and the per-bot task rate ramp up over time following a sqrt curve, producing a decelerating-growth trend suitable for e2e soak testing.
+
+```bash
+# Short demo run — 5 users over 6 minutes, 0.2 → 2 tasks/min per bot
+node runner.js --growth \
+  --server https://mcp-dev1.01.chainsafe.dev \
+  --billing-portal https://billing-dev1.01.chainsafe.dev \
+  --max-users 5 --ramp-hours 0.1 \
+  --min-rate-per-min 0.2 --max-rate-per-min 2
+```
+
+Generated keyfiles are saved to `./keys/<name-prefix>-<N>.json` and reused on restart, so killing and re-starting the process does not re-mint parties or double the fleet.
+
+| Flag | Default | Purpose |
+|------|---------|---------|
+| `--growth` | off | Enable growth mode |
+| `--billing-portal` | (required) | Portal URL for party registration |
+| `--max-users` | 20 | Ramp target (concurrent bots at plateau) |
+| `--ramp-hours` | 24 | Hours from 1 bot to `--max-users` bots |
+| `--min-rate-per-min` | 0.2 | Per-bot task rate at ramp start |
+| `--max-rate-per-min` | 2 | Per-bot task rate at ramp end |
+| `--keys-dir` | `./keys` | Where generated keyfiles are persisted |
+| `--name-prefix` | `autobot` | Prefix for minted party names |
+| `--growth-tick-seconds` | 30 | Orchestrator tick cadence |
+
+Growth-mode `[STATUS]` lines include a `bots=N` segment alongside the existing task stats. Each bot spawn emits `[EVENT] ... party_provisioned` (new mint) or reuses an existing keyfile silently.
+
+> Note: newly minted parties on devnet are not auto-funded (faucet/tap is localnet-only). If your MCP server charges strict CC balance, pre-fund the keys in `./keys/` manually or run against localnet.
+
 ## Custom Tasks
 
 Create a JSON file with your own tasks:

autobots/keys/finn_wang.json

@@ -0,0 +1,6 @@
+{
+  "partyId": "finn_wang::12209f228b507dbeb27eed4648bfc8eb42aacfad11cc4b73d89803bcf00df11b899c",
+  "privateKey": "GK8HRsoFbvHlLnuovEQE0PraJYD73VhJ0H+PMofzYuA=",
+  "publicKey": "R1+QdrBUvTU5Hh3ZVUrwCCnh4BQIWalr9xl9N4XCXBk=",
+  "fingerprint": "12209f228b507dbeb27eed4648bfc8eb42aacfad11cc4b73d89803bcf00df11b899c"
+}

autobots/keys/lars49.json

@@ -0,0 +1,6 @@
+{
+  "partyId": "lars49::1220ce0d49ac2416cea749758382c11ee00802635f46f7d70482ad516fc2e716574f",
+  "privateKey": "ErWfJ67t/iJ8TkWh32dqUWC8Oy3TmEtT8xhSPkeJEWA=",
+  "publicKey": "oZukI5j2Xj2ltozOmkboiQzVRIv2XbiCx5CYL0rxxg0=",
+  "fingerprint": "1220ce0d49ac2416cea749758382c11ee00802635f46f7d70482ad516fc2e716574f"
+}

autobots/keys/leila84.json

@@ -0,0 +1,6 @@
+{
+  "partyId": "leila84::12200a03f84e6c59b5fb9d9838ba46f95b21feae84c8ab5f076a1111d7a08a69bf7c",
+  "privateKey": "t6uIud+K7Makdf95gR6mrN+e9BEiE3aMNwNO+jKLZhE=",
+  "publicKey": "mKiyvcIU14QJB0PmNEnGWRJ58W0jQ+3vdnT0YmfD0SQ=",
+  "fingerprint": "12200a03f84e6c59b5fb9d9838ba46f95b21feae84c8ab5f076a1111d7a08a69bf7c"
+}

autobots/keys/neonwolf379.json

@@ -0,0 +1,6 @@
+{
+  "partyId": "neonwolf379::12207be2811a8ddb74785206db1ea3f5b4da0eed66df14d13112219a8ce5937035f7",
+  "privateKey": "xYkS0+StkMHpeL7331OBdAiRn8O++Ms7AKdBNf10KfM=",
+  "publicKey": "W+730zW5fc69c2aXJYDnEo9xbJhT9ktFAsrOl+XcAys=",
+  "fingerprint": "12207be2811a8ddb74785206db1ea3f5b4da0eed66df14d13112219a8ce5937035f7"
+}

autobots/keys/zara.json

@@ -0,0 +1,6 @@
+{
+  "partyId": "zara::1220576dabbeefbc686bafdb196a9739ecb918461c81e2d10d66bcd8153ec435fda7",
+  "privateKey": "d1rpPEbPGaGqAtQGgWQSO4PCqCva2rkZZ2ieaW3Qps4=",
+  "publicKey": "1oAtla1RFP0xs9h1MHItYiNFGGChPMtRr/kb8TDqxIA=",
+  "fingerprint": "1220576dabbeefbc686bafdb196a9739ecb918461c81e2d10d66bcd8153ec435fda7"
+}

autobots/names.js

@@ -0,0 +1,105 @@
+/**
+ * Realistic username generation for autobot party hints.
+ *
+ * Party IDs look like `<hint>::<fingerprint>` on Canton; the hint is the
+ * human-readable part a dashboard shows. Rather than `autobot-42`, we want
+ * hints that look like real usernames — first names, first.last patterns,
+ * handle + digits — so stakeholder views feel organic.
+ *
+ * All hints are lowercased `[a-z0-9_]+` (Canton-safe). No hyphens, so they
+ * visually distinguish from any pre-existing `autobot-N` keys.
+ */
+
+const FIRST_NAMES = [
+  "alex", "sam", "jamie", "taylor", "jordan", "morgan", "riley", "casey",
+  "quinn", "avery", "chris", "maya", "priya", "arjun", "raj", "ananya",
+  "yuki", "ken", "haruto", "mei", "wei", "jun", "lin", "chen",
+  "sofia", "diego", "luca", "emma", "liam", "noah", "olivia", "ava",
+  "sarah", "mike", "dave", "jen", "rachel", "ben", "nate", "kate",
+  "amir", "leila", "omar", "zara", "kwame", "adaeze", "tomas", "anna",
+  "erik", "lars", "freya", "nora", "finn", "ivy", "eli", "ada",
+  "hiro", "miko", "sana", "aiko", "ravi", "neha", "vikram", "asha",
+];
+
+const LAST_NAMES = [
+  "smith", "jones", "brown", "taylor", "lee", "wang", "kim", "patel",
+  "singh", "garcia", "martinez", "lopez", "kumar", "sharma", "tanaka",
+  "suzuki", "nguyen", "tran", "cohen", "muller", "schmidt", "novak",
+  "rossi", "bianchi", "silva", "oliveira", "kowalski", "andersson",
+  "kapoor", "chen", "liu", "zhang", "yamada", "ito", "park", "choi",
+];
+
+const HANDLE_WORDS = [
+  "fox", "wolf", "raven", "kite", "otter", "atlas", "nova", "echo",
+  "river", "orbit", "pixel", "drift", "spark", "bloom", "loop", "forge",
+  "neon", "sable", "lumen", "quartz", "flux", "cipher", "nebula",
+];
+
+const HANDLE_ADJECTIVES = [
+  "cool", "silent", "neon", "fast", "lucky", "quiet", "brave", "sunny",
+  "misty", "shady", "wild", "urban", "retro", "rapid", "clever",
+];
+
+function pick(arr, rng) {
+  return arr[Math.floor(rng() * arr.length)];
+}
+
+function maybe(rng, p) {
+  return rng() < p;
+}
+
+function twoDigit(rng) {
+  // 10..99 — avoid the very common "1"/"2" suffix that looks incremental
+  return String(10 + Math.floor(rng() * 90));
+}
+
+/**
+ * Generate one realistic username-style party hint.
+ * Patterns are roughly distributed across common real-world shapes.
+ */
+export function generateHint(rng = Math.random) {
+  const pattern = rng();
+  const first = pick(FIRST_NAMES, rng);
+  const last = pick(LAST_NAMES, rng);
+  const lastInitial = last[0];
+
+  if (pattern < 0.25) {
+    // "alex"
+    return first;
+  }
+  if (pattern < 0.5) {
+    // "alex42"
+    return `${first}${twoDigit(rng)}`;
+  }
+  if (pattern < 0.65) {
+    // "alex_kim"
+    return `${first}_${last}`;
+  }
+  if (pattern < 0.8) {
+    // "alexk" or "alexk21"
+    const base = `${first}${lastInitial}`;
+    return maybe(rng, 0.5) ? `${base}${twoDigit(rng)}` : base;
+  }
+  // "coolfox7" / "neonkite42"
+  const adj = pick(HANDLE_ADJECTIVES, rng);
+  const word = pick(HANDLE_WORDS, rng);
+  const digit = maybe(rng, 0.6) ? String(Math.floor(rng() * 1000)) : "";
+  return `${adj}${word}${digit}`;
+}
+
+/**
+ * Generate a hint that doesn't collide with any existing one. Collision on
+ * hint is not a Canton uniqueness violation (fingerprint disambiguates), but
+ * local keyfiles live at `<hint>.json` so filename collision matters.
+ *
+ * @param {(hint: string) => boolean} exists — returns true if hint already used
+ * @param {number} maxTries
+ */
+export function generateUniqueHint(exists, rng = Math.random, maxTries = 50) {
+  for (let i = 0; i < maxTries; i++) {
+    const h = generateHint(rng);
+    if (!exists(h)) return h;
+  }
+  // Fallback: append random hex suffix
+  return `${generateHint(rng)}${Math.floor(rng() * 1e6).toString(36)}`;
+}

autobots/orchestrator.js

@@ -0,0 +1,207 @@
+/**
+ * GrowthOrchestrator — manages a fleet of persona-driven VirtualBots.
+ *
+ * The orchestrator decides *when* to add new bots; each bot's cadence is
+ * driven by its persona (see personas.js), not by the orchestrator. Signup
+ * follows a sqrt-ramp from 0 → maxUsers over rampMs, producing an organic
+ * cohort curve rather than a flat spawn.
+ *
+ * On spawn, each new bot is assigned a persona drawn from the configured
+ * weighted distribution (Casual / Regular / Churned by default).
+ *
+ * On startup the orchestrator adopts any pre-existing keyfiles before minting
+ * new ones, so restarts are idempotent.
+ */
+
+import { PartyPool } from "./party-pool.js";
+import { drawPersona, PERSONAS } from "./personas.js";
+import { VirtualBot } from "./virtual-bot.js";
+
+const iso = () => new Date().toISOString();
+
+function logInfo(msg) {
+  console.log(`[INFO] ${iso()} ${msg}`);
+}
+function logWarn(msg) {
+  console.log(`[WARN] ${iso()} ${msg}`);
+}
+function logCritical(msg) {
+  console.log(`[CRITICAL] ${iso()} ${msg}`);
+}
+
+function sleep(ms) {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+export class GrowthOrchestrator {
+  #serverUrl;
+  #billingApiKey;
+  #maxUsers;
+  #rampMs;
+  #tickMs;
+  #tasks;
+  #stats;
+  #personas;
+  #demoTimeScale;
+  #pool;
+
+  #startedAt = 0;
+  #bots = [];
+  #stopped = false;
+
+  constructor({
+    serverUrl,
+    billingPortalUrl,
+    billingApiKey = null,
+    keysDir,
+    maxUsers,
+    rampHours,
+    tickSeconds = 30,
+    tasks,
+    stats,
+    personas = PERSONAS,
+    demoTimeScale = 1,
+    poolSize = 100,
+  }) {
+    this.#serverUrl = serverUrl;
+    this.#billingApiKey = billingApiKey;
+    this.#maxUsers = maxUsers;
+    this.#rampMs = rampHours * 3600 * 1000;
+    this.#tickMs = tickSeconds * 1000;
+    this.#tasks = tasks;
+    this.#stats = stats;
+    this.#personas = personas;
+    this.#demoTimeScale = demoTimeScale;
+    this.#pool = new PartyPool({ billingPortalUrl, keysDir, maxSize: poolSize });
+  }
+
+  get botCount() {
+    return this.#bots.length;
+  }
+
+  get activeBotCount() {
+    return this.#bots.filter((b) => !b.dormant).length;
+  }
+
+  get targetBotCount() {
+    return this.#usersAt(this.#elapsedFraction());
+  }
+
+  #elapsedFraction() {
+    if (this.#rampMs <= 0) return 1;
+    const elapsed = Date.now() - this.#startedAt;
+    return Math.min(1, Math.max(0, elapsed / this.#rampMs));
+  }
+
+  #usersAt(fraction) {
+    return Math.max(1, Math.round(this.#maxUsers * Math.sqrt(fraction)));
+  }
+
+  async start() {
+    this.#startedAt = Date.now();
+    const personaNames = Object.keys(this.#personas).join("/");
+    const { adopted, capacity } = this.#pool.init();
+    logInfo(
+      `Growth orchestrator starting — maxUsers=${this.#maxUsers} rampHours=${(this.#rampMs / 3600_000).toFixed(2)} personas=${personaNames} demoTimeScale=${this.#demoTimeScale} billingApiKey=${this.#billingApiKey ? "configured" : "missing"} poolSize=${capacity} adoptedKeys=${adopted}`
+    );
+    if (this.#maxUsers > capacity) {
+      logWarn(
+        `maxUsers=${this.#maxUsers} exceeds pool capacity=${capacity}; active bots will cap at pool size`
+      );
+    }
+
+    // Warm-start: if we adopted keyfiles from a previous run, spawn bots for
+    // them immediately (up to maxUsers) instead of re-ramping from zero.
+    // Bots get freshly drawn personas — we don't persist persona-to-party
+    // mapping across runs.
+    const warmStart = Math.min(adopted, this.#maxUsers);
+    for (let i = 0; i < warmStart && !this.#stopped; i++) {
+      try {
+        const reservation = await this.#pool.checkout();
+        if (!reservation) break;
+        await this.#spawnBot(reservation.keyPath, reservation.reused);
+      } catch (err) {
+        logWarn(`Warm-start spawn failed: ${err.message} — continuing`);
+      }
+    }
+
+    await this.#tickLoop();
+  }
+
+  stop() {
+    this.#stopped = true;
+    for (const bot of this.#bots) bot.stop();
+  }
+
+  async #spawnBot(keyPath, reused = false) {
+    const persona = drawPersona(this.#personas);
+    const bot = new VirtualBot({
+      keyPath,
+      serverUrl: this.#serverUrl,
+      tasks: this.#tasks,
+      stats: this.#stats,
+      persona,
+      demoTimeScale: this.#demoTimeScale,
+      billingApiKey: this.#billingApiKey,
+      onRelease: () => this.#handleBotReleased(bot, keyPath),
+    });
+    try {
+      await bot.authenticate();
+    } catch (err) {
+      logCritical(`Bot auth failed for ${keyPath}: ${err.message}`);
+      this.#pool.release(keyPath);
+      throw err;
+    }
+    this.#bots.push(bot);
+    this.#stats.addBotSpawned?.(bot.partyId, persona.name);
+    bot.startLoop();
+    logInfo(
+      `Bot online (${this.#bots.length}/${this.targetBotCount}) persona=${persona.name} party=${reused ? "recycled" : "new"} partyId=${bot.partyId.slice(0, 50)}...`
+    );
+  }
+
+  #handleBotReleased(bot, keyPath) {
+    const idx = this.#bots.indexOf(bot);
+    if (idx >= 0) this.#bots.splice(idx, 1);
+    this.#pool.release(keyPath);
+  }
+
+  async #tickLoop() {
+    while (!this.#stopped) {
+      try {
+        const fraction = this.#elapsedFraction();
+        const target = this.#usersAt(fraction);
+
+        while (this.#bots.length < target && !this.#stopped) {
+          let reservation;
+          try {
+            reservation = await this.#pool.checkout();
+          } catch (err) {
+            logWarn(`Provisioning failed: ${err.message} — will retry next tick`);
+            break;
+          }
+          if (!reservation) {
+            logWarn(
+              `Party pool exhausted (${this.#pool.inUseCount}/${this.#pool.maxSize}); will retry next tick`
+            );
+            break;
+          }
+          try {
+            await this.#spawnBot(reservation.keyPath, reservation.reused);
+          } catch {
+            // #spawnBot already released the party and logged.
+            break;
+          }
+        }
+
+        console.log(
+          `[STATUS] ${iso().slice(0, 19)}Z growth bots=${this.#bots.length} active=${this.activeBotCount} target=${target} ramp=${(fraction * 100).toFixed(1)}% pool=${this.#pool.size}/${this.#pool.maxSize} idle=${this.#pool.idleCount}`
+        );
+      } catch (err) {
+        logCritical(`Orchestrator tick error: ${err.message}`);
+        if (err.stack) console.error(err.stack);
+      }
+      await sleep(this.#tickMs);
+    }
+  }
+}