Diagnose Gnippets E2E HTTP failures

esokullu · esokullu · commit b1230c97d256 · 2026-07-21T13:25:24.000+03:00
diff --git a/.github/workflows/cloud-e2e.yml b/.github/workflows/cloud-e2e.yml
@@ -18,6 +18,10 @@ on:
         description: Parallel incognito browsers
         required: true
         default: "2"
+      scenario:
+        description: Optional single scenario ID
+        required: false
+        type: string
 
 permissions:
   contents: read
@@ -44,7 +48,15 @@ jobs:
           GNIPPETS_BASE_URL: ${{ vars.GNIPPETS_BASE_URL }}
           GNIPPETS_E2E_CONTROL_TOKEN: ${{ secrets.GNIPPETS_E2E_CONTROL_TOKEN }}
           CAPSOLVER_API_KEY: ${{ secrets.CAPSOLVER_API_KEY }}
-        run: node ci/run.mjs --pack "${{ inputs.pack }}" --concurrency "${{ inputs.concurrency }}"
+          E2E_PACK: ${{ inputs.pack }}
+          E2E_CONCURRENCY: ${{ inputs.concurrency }}
+          E2E_SCENARIO: ${{ inputs.scenario }}
+        run: |
+          args=(--pack "$E2E_PACK" --concurrency "$E2E_CONCURRENCY")
+          if [[ -n "$E2E_SCENARIO" ]]; then
+            args+=(--scenario "$E2E_SCENARIO")
+          fi
+          node ci/run.mjs "${args[@]}"
 
       - name: Upload traces, recordings, and rubrics
         if: always()
diff --git a/ci/lib/webbrain-client.mjs b/ci/lib/webbrain-client.mjs
@@ -150,8 +150,38 @@ export class GnippetsE2EClient {
       },
       body: body === undefined ? undefined : JSON.stringify(body),
     });
-    const value = await response.json().catch(() => ({}));
-    if (!response.ok) throw new Error(value.error || `Gnippets E2E returned HTTP ${response.status}.`);
+    const text = await response.text();
+    let value;
+    try { value = text ? JSON.parse(text) : {}; } catch { value = {}; }
+    if (!response.ok) {
+      const header = (name) => String(response.headers?.get?.(name) || 'unknown')
+        .replace(/\s+/g, ' ')
+        .slice(0, 160);
+      let preview = text;
+      if (this.controlToken) preview = preview.split(this.controlToken).join('[redacted]');
+      preview = preview
+        .replace(/Bearer\s+[^\s<]+/gi, 'Bearer [redacted]')
+        .replace(/\s+/g, ' ')
+        .trim()
+        .slice(0, 240);
+      const diagnostics = {
+        server: header('server'),
+        cf_ray: header('cf-ray'),
+        cf_mitigated: header('cf-mitigated'),
+        content_type: header('content-type'),
+        body_preview: preview,
+      };
+      const summary = Object.entries(diagnostics)
+        .filter(([, diagnostic]) => diagnostic && diagnostic !== 'unknown')
+        .map(([name, diagnostic]) => `${name}=${diagnostic}`)
+        .join('; ');
+      const error = new Error(
+        value.error || `Gnippets E2E returned HTTP ${response.status}${summary ? ` (${summary})` : ''}.`,
+      );
+      error.status = response.status;
+      error.body = diagnostics;
+      throw error;
+    }
     return value;
   }
 
diff --git a/ci/test.mjs b/ci/test.mjs
@@ -4,6 +4,7 @@ import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { gradeScenario, inferStuckAt, renderSummary } from './lib/grader.mjs';
 import { buildSessionSettings, resolveCloudRunId, suiteShouldFail } from './lib/suite.mjs';
+import { GnippetsE2EClient } from './lib/webbrain-client.mjs';
 
 const root = path.dirname(fileURLToPath(import.meta.url));
 const scenarios = JSON.parse(await fs.readFile(path.join(root, 'catalog', 'scenarios.json'), 'utf8'));
@@ -29,6 +30,41 @@ assert.deepEqual(
   { enabled: true, key: 'captcha-key' },
 );
 
+const diagnosticSecret = 'diagnostic-secret-that-must-not-leak';
+const diagnosticClient = new GnippetsE2EClient({
+  baseUrl: 'https://gnippets.example',
+  controlToken: diagnosticSecret,
+  fetchImpl: async () => ({
+    ok: false,
+    status: 403,
+    headers: {
+      get(name) {
+        return {
+          server: 'cloudflare',
+          'cf-ray': 'fixture-ray-IST',
+          'cf-mitigated': 'challenge',
+          'content-type': 'text/html',
+        }[name.toLowerCase()] || null;
+      },
+    },
+    async text() {
+      return `<html><title>Attention Required</title><body>Bearer ${diagnosticSecret}</body></html>`;
+    },
+  }),
+});
+await assert.rejects(
+  diagnosticClient.createRun('fixture'),
+  (error) => {
+    assert.equal(error.status, 403);
+    assert.equal(error.body.server, 'cloudflare');
+    assert.equal(error.body.cf_mitigated, 'challenge');
+    assert.match(error.message, /fixture-ray-IST/);
+    assert.match(error.message, /Attention Required/);
+    assert.doesNotMatch(error.message, new RegExp(diagnosticSecret));
+    return true;
+  },
+);
+
 const mountainScenario = scenarios.find((scenario) => scenario.id === 'wikipedia-table-extraction');
 const invalidMountainHeights = gradeScenario({
   scenario: mountainScenario,
