test(cloudflare): Use .makeRequestAndWaitForEnvelope to wait for envelopes (#20208)

There was one flaky test, which got me a little deeper into the
`runner.ts` logic. This test was only passing when it was running /
finishing first. With the `shuffle` flag it was consistently failing,
this is why this is added in this PR as well.

Furthermore a random port will be created for each runner by setting
`--port 0`, this just makes sure that when running `wrangler dev` in
another tab, while running the tests, the local development has the
default `:8787` port.

`.makeRequestAndWaitForEnvelope` has been added that is making a request
and then waits for an envelope, with that we make sure that flush has
been called before going to the next request. The issue is that
miniflare might cancel ongoing `waitUntil` calls (where we flush)

Author: JPeer264

dev-packages/cloudflare-integration-tests/runner.ts

@@ -50,6 +50,12 @@ type StartResult = {
     path: string,
     options?: { headers?: Record<string, string>; data?: BodyInit; expectError?: boolean },
   ): Promise<T | undefined>;
+  makeRequestAndWaitForEnvelope<T>(
+    method: 'get' | 'post',
+    path: string,
+    expected: Expected | Expected[],
+    options?: { headers?: Record<string, string>; data?: BodyInit; expectError?: boolean },
+  ): Promise<T | undefined>;
 };
 
 /** Creates a test runner */
@@ -108,6 +114,7 @@ export function createRunner(...paths: string[]) {
       const expectedEnvelopeCount = expectedEnvelopes.length;
 
       let envelopeCount = 0;
+      const envelopeWaiters: { expected: Expected; resolve: () => void; reject: (e: unknown) => void }[] = [];
       const { resolve: setWorkerPort, promise: workerPortPromise } = deferredPromise<number>();
       let child: ReturnType<typeof spawn> | undefined;
       let childSubWorker: ReturnType<typeof spawn> | undefined;
@@ -120,6 +127,12 @@ export function createRunner(...paths: string[]) {
         }
       }
 
+      function waitForEnvelope(expected: Expected): Promise<void> {
+        return new Promise((resolveWaiter, rejectWaiter) => {
+          envelopeWaiters.push({ expected, resolve: resolveWaiter, reject: rejectWaiter });
+        });
+      }
+
       function assertEnvelopeMatches(expected: Expected, envelope: Envelope): void {
         if (typeof expected === 'function') {
           expected(envelope);
@@ -137,6 +150,18 @@ export function createRunner(...paths: string[]) {
           return;
         }
 
+        // Check per-request waiters first (FIFO order)
+        if (envelopeWaiters.length > 0) {
+          const waiter = envelopeWaiters.shift()!;
+          try {
+            assertEnvelopeMatches(waiter.expected, envelope);
+            waiter.resolve();
+          } catch (e) {
+            waiter.reject(e);
+          }
+          return;
+        }
+
         try {
           if (unordered) {
             // find any matching expected envelope
@@ -242,6 +267,10 @@ export function createRunner(...paths: string[]) {
               `SENTRY_DSN:http://public@localhost:${mockServerPort}/1337`,
               '--var',
               `SERVER_URL:${serverUrl}`,
+              '--port',
+              '0',
+              '--inspector-port',
+              '0',
               ...extraWranglerArgs,
             ],
             { stdio, signal },
@@ -304,6 +333,18 @@ export function createRunner(...paths: string[]) {
             return;
           }
         },
+        makeRequestAndWaitForEnvelope: async function <T>(
+          method: 'get' | 'post',
+          path: string,
+          expected: Expected | Expected[],
+          options: { headers?: Record<string, string>; data?: BodyInit; expectError?: boolean } = {},
+        ): Promise<T | undefined> {
+          const expectations = Array.isArray(expected) ? expected : [expected];
+          const envelopePromises = expectations.map(e => waitForEnvelope(e));
+          const result = await this.makeRequest<T>(method, path, options);
+          await Promise.all(envelopePromises);
+          return result;
+        },
       };
     },
   };

dev-packages/cloudflare-integration-tests/suites/public-api/startSpan-streamed/test.ts

@@ -197,7 +197,7 @@ it('sends a streamed span envelope with correct spans for a manually started spa
           },
           'url.port': {
             type: 'string',
-            value: '8787',
+            value: expect.stringMatching(/^\d{4,5}$/),
           },
           'url.scheme': {
             type: 'string',
@@ -221,7 +221,7 @@ it('sends a streamed span envelope with correct spans for a manually started spa
           },
           'http.request.header.cf_connecting_ip': {
             type: 'string',
-            value: '::1',
+            value: expect.stringMatching(/^(::1|127\.0\.0\.1)$/),
           },
           'http.request.header.host': {
             type: 'string',

dev-packages/cloudflare-integration-tests/suites/tracing/durableobject-spans/test.ts

@@ -25,15 +25,13 @@ it.skip('sends child spans on repeated Durable Object calls', async ({ signal })
 
     // All 5 child spans should be present
     expect(transactionEvent.spans).toHaveLength(5);
-    expect(transactionEvent.spans).toEqual(
-      expect.arrayContaining([
-        expect.objectContaining({ description: 'task-1', op: 'task' }),
-        expect.objectContaining({ description: 'task-2', op: 'task' }),
-        expect.objectContaining({ description: 'task-3', op: 'task' }),
-        expect.objectContaining({ description: 'task-4', op: 'task' }),
-        expect.objectContaining({ description: 'task-5', op: 'task' }),
-      ]),
-    );
+    expect(transactionEvent.spans).toEqual([
+      expect.objectContaining({ description: 'task-1', op: 'task' }),
+      expect.objectContaining({ description: 'task-2', op: 'task' }),
+      expect.objectContaining({ description: 'task-3', op: 'task' }),
+      expect.objectContaining({ description: 'task-4', op: 'task' }),
+      expect.objectContaining({ description: 'task-5', op: 'task' }),
+    ]);
 
     // All child spans share the root trace_id
     const rootTraceId = transactionEvent.contexts?.trace?.trace_id;
@@ -43,13 +41,12 @@ it.skip('sends child spans on repeated Durable Object calls', async ({ signal })
     }
   }
 
-  // Expect 5 transaction envelopes — one per call.
-  const runner = createRunner(__dirname).expectN(5, assertDoWorkEnvelope).start(signal);
+  const runner = createRunner(__dirname).start(signal);
 
-  await runner.makeRequest('get', '/');
-  await runner.makeRequest('get', '/');
-  await runner.makeRequest('get', '/');
-  await runner.makeRequest('get', '/');
-  await runner.makeRequest('get', '/');
-  await runner.completed();
+  // Each request waits for its envelope to be received and validated before proceeding.
+  await runner.makeRequestAndWaitForEnvelope('get', '/', assertDoWorkEnvelope);
+  await runner.makeRequestAndWaitForEnvelope('get', '/', assertDoWorkEnvelope);
+  await runner.makeRequestAndWaitForEnvelope('get', '/', assertDoWorkEnvelope);
+  await runner.makeRequestAndWaitForEnvelope('get', '/', assertDoWorkEnvelope);
+  await runner.makeRequestAndWaitForEnvelope('get', '/', assertDoWorkEnvelope);
 });

dev-packages/cloudflare-integration-tests/vite.config.mts

@@ -28,6 +28,9 @@ export default defineConfig({
         singleThread: true,
       },
     },
+    sequence: {
+      shuffle: true,
+    },
     reporters: process.env.DEBUG
       ? ['default', { summary: false }]
       : process.env.GITHUB_ACTIONS