QA: speed up Matrix live lane

Author: gumadeiras

extensions/qa-matrix/src/runners/contract/runtime.test.ts

@@ -325,6 +325,26 @@ describe("matrix live qa runtime", () => {
     });
   });
 
+  it("batches Matrix scenarios by config key while preserving stable in-group order", () => {
+    const scenarios = liveTesting.findMatrixQaScenarios([
+      "matrix-top-level-reply-shape",
+      "matrix-room-thread-reply-override",
+      "matrix-thread-follow-up",
+      "matrix-room-quiet-streaming-preview",
+      "matrix-reaction-notification",
+    ]);
+
+    expect(
+      liveTesting.scheduleMatrixQaScenariosByConfig(scenarios).map(({ scenario }) => scenario.id),
+    ).toEqual([
+      "matrix-thread-follow-up",
+      "matrix-top-level-reply-shape",
+      "matrix-reaction-notification",
+      "matrix-room-thread-reply-override",
+      "matrix-room-quiet-streaming-preview",
+    ]);
+  });
+
   it("treats only connected, healthy Matrix accounts as ready", () => {
     expect(liveTesting.isMatrixAccountReady({ running: true, connected: true })).toBe(true);
     expect(liveTesting.isMatrixAccountReady({ running: true, connected: false })).toBe(false);

extensions/qa-matrix/src/runners/contract/runtime.ts

@@ -61,6 +61,11 @@ type MatrixQaScenarioResult = {
   title: string;
 };
 
+type MatrixQaScheduledScenario = {
+  originalIndex: number;
+  scenario: (typeof MATRIX_QA_SCENARIOS)[number];
+};
+
 type MatrixQaScenarioConfigEntry = MatrixQaSummary["config"]["scenarios"][number];
 
 type MatrixQaSummary = {
@@ -178,6 +183,25 @@ function buildMatrixQaScenarioResult(params: {
   };
 }
 
+function scheduleMatrixQaScenariosByConfig(
+  scenarios: readonly (typeof MATRIX_QA_SCENARIOS)[number][],
+): MatrixQaScheduledScenario[] {
+  const grouped = new Map<string, MatrixQaScheduledScenario[]>();
+
+  scenarios.forEach((scenario, originalIndex) => {
+    const configKey = buildMatrixQaGatewayConfigKey(scenario.configOverrides);
+    const existing = grouped.get(configKey);
+    const scheduled = { originalIndex, scenario };
+    if (existing) {
+      existing.push(scheduled);
+      return;
+    }
+    grouped.set(configKey, [scheduled]);
+  });
+
+  return [...grouped.values()].flat();
+}
+
 export type MatrixQaRunResult = {
   observedEventsPath: string;
   outputDir: string;
@@ -368,7 +392,9 @@ export async function runMatrixQaLive(params: {
       ].join("\n"),
     },
   ];
-  const scenarioResults: MatrixQaScenarioResult[] = [];
+  const scenarioResults: Array<MatrixQaScenarioResult | undefined> = Array.from({
+    length: scenarios.length,
+  });
   const cleanupErrors: string[] = [];
   let canaryArtifact: MatrixQaCanaryArtifact | undefined;
   let gatewayHarness: MatrixQaLiveLaneGatewayHarness | null = null;
@@ -388,6 +414,8 @@ export async function runMatrixQaLive(params: {
   const defaultConfigSnapshot = buildMatrixQaConfigSnapshot(gatewayConfigParams);
   const scenarioConfigSnapshots: MatrixQaScenarioConfigEntry[] = [];
 
+  const scheduledScenarios = scheduleMatrixQaScenariosByConfig(scenarios);
+
   try {
     const ensureGatewayHarness = async (overrides?: MatrixQaConfigOverrides) => {
       const nextKey = buildMatrixQaGatewayConfigKey(overrides);
@@ -460,13 +488,13 @@ export async function runMatrixQaLive(params: {
     }
 
     if (!canaryFailed) {
-      for (const scenario of scenarios) {
+      for (const { scenario, originalIndex } of scheduledScenarios) {
         const { entry: scenarioConfigEntry, summary: scenarioConfigSummary } =
           buildMatrixQaScenarioConfigEntry({
             gatewayConfigParams,
             scenario,
           });
-        scenarioConfigSnapshots.push(scenarioConfigEntry);
+        scenarioConfigSnapshots[originalIndex] = scenarioConfigEntry;
         try {
           const scenarioGateway = await ensureGatewayHarness(scenario.configOverrides);
           const result = await runMatrixQaScenario(scenario, {
@@ -497,24 +525,20 @@ export async function runMatrixQaLive(params: {
             timeoutMs: scenario.timeoutMs,
             topology: provisioning.topology,
           });
-          scenarioResults.push(
-            buildMatrixQaScenarioResult({
-              artifacts: result.artifacts,
-              configSummary: scenarioConfigSummary,
-              details: result.details,
-              scenario,
-              status: "pass",
-            }),
-          );
+          scenarioResults[originalIndex] = buildMatrixQaScenarioResult({
+            artifacts: result.artifacts,
+            configSummary: scenarioConfigSummary,
+            details: result.details,
+            scenario,
+            status: "pass",
+          });
         } catch (error) {
-          scenarioResults.push(
-            buildMatrixQaScenarioResult({
-              configSummary: scenarioConfigSummary,
-              details: formatErrorMessage(error),
-              scenario,
-              status: "fail",
-            }),
-          );
+          scenarioResults[originalIndex] = buildMatrixQaScenarioResult({
+            configSummary: scenarioConfigSummary,
+            details: formatErrorMessage(error),
+            scenario,
+            status: "fail",
+          });
         }
       }
     }
@@ -532,6 +556,9 @@ export async function runMatrixQaLive(params: {
       appendLiveLaneIssue(cleanupErrors, "Matrix harness cleanup", error);
     }
   }
+  const completedScenarioResults = scenarioResults.filter(
+    (scenario): scenario is MatrixQaScenarioResult => scenario !== undefined,
+  );
   if (cleanupErrors.length > 0) {
     checks.push({
       name: "Matrix cleanup",
@@ -555,7 +582,7 @@ export async function runMatrixQaLive(params: {
     startedAt: startedAtDate,
     finishedAt: finishedAtDate,
     checks,
-    scenarios: scenarioResults.map((scenario) => ({
+    scenarios: completedScenarioResults.map((scenario) => ({
       details: scenario.details,
       name: scenario.title,
       status: scenario.status,
@@ -592,7 +619,7 @@ export async function runMatrixQaLive(params: {
       serverName: harness.serverName,
     },
     observedEventCount: observedEvents.length,
-    scenarios: scenarioResults,
+    scenarios: completedScenarioResults,
     startedAt,
     sutAccountId,
     userIds: {
@@ -623,7 +650,7 @@ export async function runMatrixQaLive(params: {
   const failedChecks = checks.filter(
     (check) => check.status === "fail" && check.name !== "Matrix cleanup",
   );
-  const failedScenarios = scenarioResults.filter((scenario) => scenario.status === "fail");
+  const failedScenarios = completedScenarioResults.filter((scenario) => scenario.status === "fail");
   if (failedChecks.length > 0 || failedScenarios.length > 0) {
     throw new Error(
       buildLiveLaneArtifactsError({
@@ -651,16 +678,18 @@ export async function runMatrixQaLive(params: {
     observedEventsPath,
     outputDir,
     reportPath,
-    scenarios: scenarioResults,
+    scenarios: completedScenarioResults,
     summaryPath,
   };
 }
 
 export const __testing = {
   buildMatrixQaSummary,
+  scheduleMatrixQaScenariosByConfig,
   MATRIX_QA_SCENARIOS,
   buildMatrixQaConfig,
   buildMatrixQaConfigSnapshot,
+  findMatrixQaScenarios,
   isMatrixAccountReady,
   resolveMatrixQaModels,
   summarizeMatrixQaConfigSnapshot,

extensions/qa-matrix/src/substrate/client.ts

@@ -1,4 +1,5 @@
 import { randomUUID } from "node:crypto";
+import { setTimeout as sleep } from "node:timers/promises";
 import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
 import type { MatrixQaObservedEvent } from "./events.js";
 import { requestMatrixJson, type MatrixQaFetchLike } from "./request.js";
@@ -474,7 +475,7 @@ async function joinRoomWithRetry(params: {
       return;
     } catch (error) {
       lastError = error;
-      await new Promise((resolve) => setTimeout(resolve, 300 * attempt));
+      await sleep(300 * attempt);
     }
   }
   throw new Error(`Matrix join retry failed: ${formatErrorMessage(lastError)}`);
@@ -504,40 +505,42 @@ async function provisionMatrixQaTopology(params: {
   fetchImpl?: MatrixQaFetchLike;
   spec: MatrixQaTopologySpec;
 }): Promise<MatrixQaProvisionedTopology> {
-  const rooms = [];
-
-  for (const room of params.spec.rooms) {
-    const members = resolveTopologyMemberAccounts(params.accounts, room.members);
-    const creator = members[0];
-    const invitees = members.slice(1);
-    const creatorClient = createMatrixQaClient({
-      accessToken: creator.account.accessToken,
-      baseUrl: params.baseUrl,
-      fetchImpl: params.fetchImpl,
-    });
-    const roomId = await creatorClient.createPrivateRoom({
-      inviteUserIds: invitees.map((entry) => entry.account.userId),
-      isDirect: room.kind === "dm",
-      name: room.name,
-    });
-    for (const invitee of invitees) {
-      await joinRoomWithRetry({
-        accessToken: invitee.account.accessToken,
+  const rooms = await Promise.all(
+    params.spec.rooms.map(async (room) => {
+      const members = resolveTopologyMemberAccounts(params.accounts, room.members);
+      const creator = members[0];
+      const invitees = members.slice(1);
+      const creatorClient = createMatrixQaClient({
+        accessToken: creator.account.accessToken,
         baseUrl: params.baseUrl,
         fetchImpl: params.fetchImpl,
-        roomId,
       });
-    }
-    rooms.push({
-      key: room.key,
-      kind: room.kind,
-      memberRoles: members.map((entry) => entry.role),
-      memberUserIds: members.map((entry) => entry.account.userId),
-      name: room.name,
-      requireMention: resolveProvisionedRoomRequireMention(room),
-      roomId,
-    });
-  }
+      const roomId = await creatorClient.createPrivateRoom({
+        inviteUserIds: invitees.map((entry) => entry.account.userId),
+        isDirect: room.kind === "dm",
+        name: room.name,
+      });
+      await Promise.all(
+        invitees.map((invitee) =>
+          joinRoomWithRetry({
+            accessToken: invitee.account.accessToken,
+            baseUrl: params.baseUrl,
+            fetchImpl: params.fetchImpl,
+            roomId,
+          }),
+        ),
+      );
+      return {
+        key: room.key,
+        kind: room.kind,
+        memberRoles: members.map((entry) => entry.role),
+        memberUserIds: members.map((entry) => entry.account.userId),
+        name: room.name,
+        requireMention: resolveProvisionedRoomRequireMention(room),
+        roomId,
+      };
+    }),
+  );
 
   const defaultRoom = findMatrixQaProvisionedRoom(
     {
@@ -569,24 +572,26 @@ export async function provisionMatrixQaRoom(params: {
     baseUrl: params.baseUrl,
     fetchImpl: params.fetchImpl,
   });
-  const driver = await anonClient.registerWithToken({
-    deviceName: "OpenClaw Matrix QA Driver",
-    localpart: params.driverLocalpart,
-    password: `driver-${randomUUID()}`,
-    registrationToken: params.registrationToken,
-  });
-  const sut = await anonClient.registerWithToken({
-    deviceName: "OpenClaw Matrix QA SUT",
-    localpart: params.sutLocalpart,
-    password: `sut-${randomUUID()}`,
-    registrationToken: params.registrationToken,
-  });
-  const observer = await anonClient.registerWithToken({
-    deviceName: "OpenClaw Matrix QA Observer",
-    localpart: params.observerLocalpart,
-    password: `observer-${randomUUID()}`,
-    registrationToken: params.registrationToken,
-  });
+  const [driver, sut, observer] = await Promise.all([
+    anonClient.registerWithToken({
+      deviceName: "OpenClaw Matrix QA Driver",
+      localpart: params.driverLocalpart,
+      password: `driver-${randomUUID()}`,
+      registrationToken: params.registrationToken,
+    }),
+    anonClient.registerWithToken({
+      deviceName: "OpenClaw Matrix QA SUT",
+      localpart: params.sutLocalpart,
+      password: `sut-${randomUUID()}`,
+      registrationToken: params.registrationToken,
+    }),
+    anonClient.registerWithToken({
+      deviceName: "OpenClaw Matrix QA Observer",
+      localpart: params.observerLocalpart,
+      password: `observer-${randomUUID()}`,
+      registrationToken: params.registrationToken,
+    }),
+  ]);
   const topology = await provisionMatrixQaTopology({
     accounts: {
       driver,