chore: make smoketests faster via test.concurrent, no sharding, 800% workers (#773)

Co-authored-by: codeant-ai[bot] <151821869+codeant-ai[bot]@users.noreply.github.com>

Author: jason-rl

.github/workflows/sdk-coverage.yml

@@ -53,10 +53,10 @@ jobs:
 
       - name: Run smoke tests with coverage
         id: tests
-        continue-on-error: true
         run: |
           set -o pipefail
-          yarn test:objects-coverage 2>&1 | tee test-output.log
+          yarn test:objects-coverage --maxWorkers=800% --color 2>&1 | \
+            tee -i test-output.log
 
       - name: Upload coverage report
         uses: runloopai/upload-artifact@main
@@ -73,7 +73,7 @@ jobs:
           script: |
             const fs = require('fs');
             const testsPassed = '${{ steps.tests.outcome }}' === 'success';
-            const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+            const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
             const COMMENT_MARKER = '<!-- sdk-coverage-report -->';
             
             let lines = [COMMENT_MARKER];
@@ -197,8 +197,3 @@ jobs:
               });
             }
 
-      - name: Fail if tests failed
-        if: steps.tests.outcome != 'success'
-        run: |
-          echo "Smoke tests failed. See PR comment for details."
-          exit 1

.github/workflows/smoke-tests.yml

@@ -27,10 +27,6 @@ jobs:
   smoke-tests:
     runs-on: ubuntu-latest
     timeout-minutes: 60
-    strategy:
-      fail-fast: false
-      matrix:
-        shard: [1, 2, 3, 4]
     steps:
       - name: Checkout
         uses: runloopai/checkout@main
@@ -41,6 +37,7 @@ jobs:
         uses: runloopai/setup-node@main
         with:
           node-version: '20'
+          cache: yarn
 
       - name: Install dependencies
         run: yarn --frozen-lockfile
@@ -67,74 +64,5 @@ jobs:
       - name: Verify generated example artifacts
         run: yarn check:examples-md
 
-      - name: Run smoke tests (Shard ${{ matrix.shard }}/4)
-        id: tests
-        shell: python
-        run: |
-          from subprocess import Popen, PIPE
-          import sys
-
-          proc = Popen(["yarn", "test:smoke", "--shard=${{ matrix.shard }}/4", "--color"], stderr=PIPE)
-          zstd = Popen(["zstd", "-q", "-5", "-o", "smoke-tests-shard-${{ matrix.shard }}.zst"], stdin=PIPE)
-          tee = Popen(["tee", "-i", "/dev/stderr"], stdin=proc.stderr, stdout=zstd.stdin)
-          proc.stderr.close()
-          sys.exit(proc.wait())
-
-      - name: Upload test output
-        if: always()
-        uses: runloopai/upload-artifact@main
-        with:
-          path: smoke-tests-shard-${{ matrix.shard }}.zst
-          if-no-files-found: error
-          retention-days: 1
-          archive: false
-
-  smoke-tests-results:
-    runs-on: ubuntu-slim
-    needs: smoke-tests
-    if: always()
-    steps:
-      - name: Download all test output artifacts
-        uses: runloopai/download-artifact@main
-        with:
-          pattern: smoke-tests-shard-*.zst
-          path: all-output
-
-      - name: Decompress and display logs
-        run: |
-          # Decompress all files
-          zstd -q --rm -d all-output/*/*.zst
-
-          BOLD=$'\033[1m'
-          RESET=$'\033[22m'
-          # Extract and display failed tests from each shard
-          for output_file in all-output/*/*; do
-            shard_name=$(basename "$output_file")
-            # Check if this shard has failures
-            if grep -q "FAIL\|✕" "$output_file" 2>/dev/null; then
-              echo "$BOLD$shard_name had test failures$RESET"
-              # Filter output to show only failures:
-              # - Skip PASS suites entirely (from PASS to next FAIL or "Test Suites:")
-              # - Within FAIL suites, skip individual passing tests (✓) but show failures (● and ✕)
-              awk '
-                { line=$0; gsub(/\x1b\[[0-9;]*m/, "") }
-                /^[[:space:]]*PASS[[:space:]]/ { in_pass_suite=1; in_fail_suite=0; next }
-                /^[[:space:]]*FAIL[[:space:]]/ { in_pass_suite=0; in_fail_suite=1; print line; next }
-                /^Test Suites:/ { in_pass_suite=0; in_fail_suite=0; print line; next }
-                in_pass_suite { next }
-                in_fail_suite && /^[[:space:]]*✓/ { next }
-                in_fail_suite { print line; next }
-                { print line }
-              ' "$output_file"
-            fi
-          done
-
-          # Aggregate all output in a collapsed group
-          echo "::group::Full output from all shards"
-          for output_file in all-output/*/*; do
-            shard_name=$(basename "$output_file")
-            echo "$BOLD=== $shard_name ===$RESET"
-            cat "$output_file"
-            echo ""
-          done
-          echo "::endgroup::"
+      - name: Run smoke tests
+        run: yarn test:smoke --maxWorkers=800% --color

tests/smoketests/blueprints.test.ts

@@ -86,7 +86,7 @@ describe('smoketest: blueprints', () => {
   (process.env['RUN_SMOKETESTS'] ? describe : describe.skip)('blueprint secrets', () => {
     const secretsBlueprintName = uniqueName('bp-secrets');
 
-    test(
+    test.concurrent(
       'create blueprint with secret in Dockerfile and await build',
       async () => {
         let bpt: BlueprintView | undefined;

tests/smoketests/devboxes.test.ts

@@ -8,7 +8,7 @@ describe('smoketest: devboxes', () => {
    * Test V2 tunnel functionality.
    */
   describe('devbox tunnels', () => {
-    test(
+    test.concurrent(
       'create devbox with tunnel in create params',
       async () => {
         let devbox: DevboxView | undefined;
@@ -39,7 +39,7 @@ describe('smoketest: devboxes', () => {
       SHORT_TIMEOUT,
     );
 
-    test(
+    test.concurrent(
       'create devbox with authenticated tunnel in create params (deprecated polling path)',
       async () => {
         let devbox: DevboxView | undefined;
@@ -71,7 +71,7 @@ describe('smoketest: devboxes', () => {
       SHORT_TIMEOUT,
     );
 
-    test(
+    test.concurrent(
       'create devbox then enable tunnel',
       async () => {
         let devbox: DevboxView | undefined;
@@ -110,7 +110,7 @@ describe('smoketest: devboxes', () => {
       SHORT_TIMEOUT,
     );
 
-    test(
+    test.concurrent(
       'create devbox then enable authenticated tunnel',
       async () => {
         let devbox: DevboxView | undefined;
@@ -158,7 +158,7 @@ describe('smoketest: devboxes', () => {
       }
     });
 
-    test(
+    test.concurrent(
       'create devbox',
       async () => {
         let devbox: DevboxView | undefined;
@@ -211,7 +211,7 @@ describe('smoketest: devboxes', () => {
     });
   });
 
-  test(
+  test.concurrent(
     'createAndAwaitRunning long set up',
     async () => {
       // createAndAwaitRunning should poll until devbox is running
@@ -229,7 +229,7 @@ describe('smoketest: devboxes', () => {
     SHORT_TIMEOUT * 4,
   );
 
-  test(
+  test.concurrent(
     'createAndAwaitRunning timeout (deprecated polling path)',
     async () => {
       await expect(

tests/smoketests/examples/examples.test.ts

@@ -7,7 +7,7 @@ describe('smoketest: examples', () => {
   for (const entry of exampleRegistry) {
     const hasRequiredEnv = entry.requiredEnv.every((name) => Boolean(process.env[name]));
     const runLiveExample = hasLiveFlag && hasRequiredEnv;
-    const itLive = runLiveExample ? test : test.skip;
+    const itLive = runLiveExample ? test.concurrent : test.concurrent.skip;
 
     itLive(
       `${entry.slug} example runs with successful checks and cleanup`,

tests/smoketests/object-oriented/agent.test.ts

@@ -5,7 +5,7 @@ const runloop = makeClientSDK();
 
 describe('smoketest: object-oriented agent', () => {
   describe('agent lifecycle', () => {
-    test(
+    test.concurrent(
       'create agent basic',
       async () => {
         const name = uniqueName('sdk-agent-test-basic');
@@ -37,7 +37,7 @@ describe('smoketest: object-oriented agent', () => {
       SHORT_TIMEOUT,
     );
 
-    test(
+    test.concurrent(
       'get agent info',
       async () => {
         const name = uniqueName('sdk-agent-test-info');
@@ -68,7 +68,7 @@ describe('smoketest: object-oriented agent', () => {
   });
 
   describe('agent listing', () => {
-    test(
+    test.concurrent(
       'list agents',
       async () => {
         const agents = await runloop.agent.list({ limit: 10 });
@@ -80,7 +80,7 @@ describe('smoketest: object-oriented agent', () => {
       SHORT_TIMEOUT,
     );
 
-    test(
+    test.concurrent(
       'get agent by ID',
       async () => {
         // Create an agent
@@ -110,7 +110,7 @@ describe('smoketest: object-oriented agent', () => {
       SHORT_TIMEOUT,
     );
 
-    test(
+    test.concurrent(
       'list multiple agents',
       async () => {
         const sourceConfig = {
@@ -159,7 +159,7 @@ describe('smoketest: object-oriented agent', () => {
   });
 
   describe('agent creation variations', () => {
-    test(
+    test.concurrent(
       'agent with source npm',
       async () => {
         const name = uniqueName('sdk-agent-test-npm');
@@ -188,7 +188,7 @@ describe('smoketest: object-oriented agent', () => {
       SHORT_TIMEOUT,
     );
 
-    test(
+    test.concurrent(
       'agent with source git',
       async () => {
         const name = uniqueName('sdk-agent-test-git');
@@ -220,7 +220,7 @@ describe('smoketest: object-oriented agent', () => {
   });
 
   describe('agent convenience methods', () => {
-    test(
+    test.concurrent(
       'createFromNpm',
       async () => {
         const name = uniqueName('sdk-agent-from-npm');
@@ -243,7 +243,7 @@ describe('smoketest: object-oriented agent', () => {
       SHORT_TIMEOUT,
     );
 
-    test(
+    test.concurrent(
       'createFromPip',
       async () => {
         const name = uniqueName('sdk-agent-from-pip');
@@ -266,7 +266,7 @@ describe('smoketest: object-oriented agent', () => {
       SHORT_TIMEOUT,
     );
 
-    test(
+    test.concurrent(
       'createFromGit',
       async () => {
         const name = uniqueName('sdk-agent-from-git');
@@ -290,7 +290,7 @@ describe('smoketest: object-oriented agent', () => {
       SHORT_TIMEOUT,
     );
 
-    test(
+    test.concurrent(
       'createFromObject with storage object',
       async () => {
         let storageObject: StorageObject | undefined;
@@ -332,7 +332,7 @@ describe('smoketest: object-oriented agent', () => {
   });
 
   describe('agent with devbox mounting', () => {
-    test(
+    test.concurrent(
       'mount agent from storage object into devbox',
       async () => {
         let storageObject: StorageObject | undefined;