CCOR-13194 - Switch integration tests to run against sm v4 server and sdkdev for v5 (#127)

* testing integration tests against alternate servers

* adjust notes around integration testing. add another test server - now we have all 5 to see what is the least flaky

* make http bin url dynamic b/c different clusters have it on different host name, make timeouts more sane in one test, add some missing task cleanup logic in tests

* add new constants file that can grab from env before setting constant

* add some retry on things that seem to be particularly painfully flaky

* add missing file

* additional retry and diagnostic within parts of the integration testing.

* add missing helper util file

* cancel in progress

* rather aggressive retry fixes for test flakiness

* add missing file

* further retry and timeout tuning

* reduce pr integration test runs to sm-v4 and sdkdev-v5

* trying to sort out inscrutible and inexplicable issues. without a whole lot of luck

* helper script to make running test shards locally easy

* require v4 integration tests to pass

* account for difference in how asyncComplete http task status works when http task is system task (v4 - no api orch) or worker task (v5 with api orch) in the way a test detects waiting http task

* exclude integration-tests/** from code coverage requirement

* restrict the asyncComplete test to v5 so it doesn't introduce the only weird v4-vs-v5-branching-inside-one-test in the entire test suite

Author: chrishagglund-ship-it

.env.example

@@ -6,4 +6,7 @@ CONDUCTOR_AUTH_SECRET=""
 CONDUCTOR_MAX_HTTP2_CONNECTIONS=
 
 CONDUCTOR_TLS_INSECURE=
-CONDUCTOR_DISABLE_HTTP2=
+CONDUCTOR_DISABLE_HTTP2=
+
+# Hostname of the httpbin service reachable from the Conductor server (default: httpbin-server)
+HTTPBIN_SERVICE_HOSTNAME=

.github/workflows/pull_request.yml

@@ -7,6 +7,10 @@ on:
   pull_request:
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   linter:
     runs-on: ubuntu-latest
@@ -79,19 +83,17 @@ jobs:
           name: codecov-unit-node-${{ matrix.node-version }}
           fail_ci_if_error: false
 
-  # Integration tests (v5): lower max-parallel reduces 502/503 from the shared Conductor server
-  # but makes CI slower without eliminating flakes entirely — feel free to experiment.
-  # Sharding (--shard i/N) splits the suite so each job runs ~1/N of tests.
-  integration-tests:
+  # Integration tests (v5 sdkdev): mirrors integration-tests but targets the sdkdev environment.
+  integration-tests-v5-sdkdev:
     runs-on: ubuntu-latest
     timeout-minutes: 25
     strategy:
       fail-fast: false
-      max-parallel: 2
+      max-parallel: 3
       matrix:
         node-version: [20, 22, 24]
         shard: [1, 2, 3]
-    name: Node.js v${{ matrix.node-version }} - integration v5 (shard ${{ matrix.shard }}/3)
+    name: Node.js v${{ matrix.node-version }} - integration v5 sdkdev (shard ${{ matrix.shard }}/3)
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -111,45 +113,45 @@ jobs:
       - name: Install Dependencies
         if: steps.cache.outputs.cache-hit != 'true'
         run: npm ci
-      - name: Run integration tests (v5) shard ${{ matrix.shard }}/3
+      - name: Run integration tests (v5 sdkdev) shard ${{ matrix.shard }}/3
         run: npm run test:integration:v5 -- --ci --coverage --runInBand --testTimeout=120000 --shard=${{ matrix.shard }}/3 --reporters=default --reporters=github-actions --reporters=jest-junit
         env:
           ORKES_BACKEND_VERSION: "5"
-          CONDUCTOR_SERVER_URL: ${{ vars.SERVER_URL }}
-          CONDUCTOR_AUTH_KEY: ${{ secrets.AUTH_KEY }}
-          CONDUCTOR_AUTH_SECRET: ${{ secrets.AUTH_SECRET }}
+          CONDUCTOR_SERVER_URL: ${{ vars.SDKDEV_V5_SERVER_URL }}
+          CONDUCTOR_AUTH_KEY: ${{ vars.SDKDEV_V5_AUTH_KEY }}
+          CONDUCTOR_AUTH_SECRET: ${{ secrets.SDKDEV_V5_AUTH_SECRET }}
           CONDUCTOR_REQUEST_TIMEOUT_MS: "120000"
           CONDUCTOR_RETRY_SERVER_ERRORS: "true"
-          JEST_JUNIT_OUTPUT_NAME: integration-v5-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}-test-results.xml
+          HTTPBIN_SERVICE_HOSTNAME: "httpbin"
+          JEST_JUNIT_OUTPUT_NAME: integration-v5-sdkdev-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}-test-results.xml
       - name: Publish Test Results
         uses: dorny/test-reporter@v2
         if: ${{ !cancelled() }}
         with:
-          name: integration v5 Node ${{ matrix.node-version }} shard ${{ matrix.shard }}/3
-          path: reports/integration-v5-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}-test-results.xml
+          name: integration v5 sdkdev Node ${{ matrix.node-version }} shard ${{ matrix.shard }}/3
+          path: reports/integration-v5-sdkdev-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}-test-results.xml
           reporter: jest-junit
       - name: Upload coverage to Codecov
         if: always()
         uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./coverage/lcov.info
-          flags: integration-v5
-          name: codecov-integration-v5-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}
+          flags: integration-v5-sdkdev
+          name: codecov-integration-v5-sdkdev-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}
           fail_ci_if_error: false
 
-  # Integration tests (v4): same sharding as v5. v4 fails in CI (passes locally); do not block PRs.
-  integration-tests-v4:
+  # Integration tests (v4 sm): mirrors integration-tests-v4 but targets the sm environment.
+  integration-tests-v4-sm:
     runs-on: ubuntu-latest
     timeout-minutes: 25
-    continue-on-error: true
     strategy:
       fail-fast: false
       max-parallel: 1
       matrix:
         node-version: [20, 22, 24]
         shard: [1, 2, 3]
-    name: Node.js v${{ matrix.node-version }} - integration v4 (shard ${{ matrix.shard }}/3)
+    name: Node.js v${{ matrix.node-version }} - integration v4 sm (shard ${{ matrix.shard }}/3)
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -169,29 +171,29 @@ jobs:
       - name: Install Dependencies
         if: steps.cache.outputs.cache-hit != 'true'
         run: npm ci
-      - name: Run integration tests (v4) shard ${{ matrix.shard }}/3
+      - name: Run integration tests (v4 sm) shard ${{ matrix.shard }}/3
         run: npm run test:integration:v4 -- --ci --coverage --runInBand --testTimeout=120000 --shard=${{ matrix.shard }}/3 --reporters=default --reporters=github-actions --reporters=jest-junit
         env:
           ORKES_BACKEND_VERSION: "4"
-          CONDUCTOR_SERVER_URL: ${{ vars.SERVER_URL_V4 }}
-          CONDUCTOR_AUTH_KEY: ${{ secrets.AUTH_KEY_V4 }}
-          CONDUCTOR_AUTH_SECRET: ${{ secrets.AUTH_SECRET_V4 }}
+          CONDUCTOR_SERVER_URL: ${{ vars.SM_V4_SERVER_URL }}
+          CONDUCTOR_AUTH_KEY: ${{ vars.SM_V4_AUTH_KEY }}
+          CONDUCTOR_AUTH_SECRET: ${{ secrets.SM_V4_AUTH_SECRET }}
           CONDUCTOR_REQUEST_TIMEOUT_MS: "120000"
           CONDUCTOR_RETRY_SERVER_ERRORS: "true"
-          JEST_JUNIT_OUTPUT_NAME: integration-v4-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}-test-results.xml
+          JEST_JUNIT_OUTPUT_NAME: integration-v4-sm-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}-test-results.xml
       - name: Publish Test Results
         uses: dorny/test-reporter@v2
         if: ${{ !cancelled() }}
         with:
-          name: integration v4 Node ${{ matrix.node-version }} shard ${{ matrix.shard }}/3
-          path: reports/integration-v4-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}-test-results.xml
+          name: integration v4 sm Node ${{ matrix.node-version }} shard ${{ matrix.shard }}/3
+          path: reports/integration-v4-sm-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}-test-results.xml
           reporter: jest-junit
       - name: Upload coverage to Codecov
         if: always()
         uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./coverage/lcov.info
-          flags: integration-v4
-          name: codecov-integration-v4-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}
+          flags: integration-v4-sm
+          name: codecov-integration-v4-sm-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}
           fail_ci_if_error: false

SDK_DEVELOPMENT.md

@@ -416,19 +416,89 @@ npm run test:unit       # Unit tests only
 
 ### Integration tests (E2E)
 
-Against a real Conductor server. Location: `src/integration-tests/`.
+Against a real Orkes Conductor (enterprise) server. Location: `src/integration-tests/`.
 
 ```bash
 # Full suite
-CONDUCTOR_SERVER_URL=http://localhost:8080 \
-CONDUCTOR_AUTH_KEY=key CONDUCTOR_AUTH_SECRET=secret \
-ORKES_BACKEND_VERSION=5 \
-npm run test:integration:orkes-v5
+npm run test:integration:v5
 
 # Single file
-npx jest --force-exit --testPathPatterns="AuthorizationClient"
+npx jest --force-exit --detectOpenHandles --testPathPatterns="AuthorizationClient"
 ```
 
+Server credentials are read from environment variables. You can export them in your
+shell or put them in a `.env` file in the repo root (loaded automatically by
+`jest.config.mjs` via `dotenv`). To use a `.env` file, copy the example:
+
+```bash
+cp .env.example .env
+```
+
+Required variables plus two that CI sets for flake resilience:
+
+```bash
+CONDUCTOR_SERVER_URL="https://your-cluster.orkesconductor.io/api"
+CONDUCTOR_AUTH_KEY="your-key"
+CONDUCTOR_AUTH_SECRET="your-secret"
+CONDUCTOR_REQUEST_TIMEOUT_MS=120000
+CONDUCTOR_RETRY_SERVER_ERRORS=true
+```
+
+### Replicating CI matrix shards locally
+
+CI runs integration tests as a matrix of **3 Node versions (20, 22, 24) x 3 shards**,
+using Jest's built-in `--shard=i/N` flag. Each shard runs ~1/3 of the test files
+deterministically, so shard 1/3 always contains the same files for a given test suite.
+
+To reproduce a specific CI cell locally you need two things: the right Node version
+and the right shard flag.
+
+#### Node version switching with fnm
+
+Install [fnm](https://github.com/Schniz/fnm) (Fast Node Manager) if you don't have it.
+If you currently have Node installed directly via Homebrew, remove it first so the two
+don't conflict:
+
+```bash
+brew remove node                    # remove Homebrew-managed Node if present
+brew install fnm
+eval "$(fnm env --use-on-cd)"      # add this line to ~/.zshrc for persistence
+```
+
+Then install the versions CI uses:
+
+```bash
+fnm install 20 && fnm install 22 && fnm install 24
+```
+
+#### Running a shard
+
+Example: reproduce **"Node.js v22 — integration v5 (shard 1/3)"**:
+
+```bash
+fnm use 22
+node --version                     # confirm v22.x.x
+
+npm run test:integration:v5 -- --runInBand --testTimeout=120000 --shard=1/3
+```
+
+Run all three shards sequentially to cover the full suite:
+
+```bash
+for shard in 1 2 3; do
+  npm run test:integration:v5 -- --runInBand --testTimeout=120000 --shard=$shard/3
+done
+```
+
+Switch back to your default Node when done:
+
+```bash
+fnm use default                    # or: fnm use system
+node --version                     # confirm you're back to your usual version
+```
+
+The same pattern works for v4 — swap `test:integration:v5` with `test:integration:v4`.
+
 ### Integration test template
 
 ```typescript

jest.config.mjs

@@ -10,6 +10,7 @@ export default {
     "!src/**/generated/**",
     "!src/**/spec/**",
     "!src/**/*.test.{ts,tsx}",
+    "!src/integration-tests/**",
     "!src/**/index.ts",
     "!src/**/types.ts",
     "!src/**/*.types.ts",

scripts/run-integration-v4.sh

@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+#
+# Run one cell of the v4 integration-test matrix, mirroring the
+# `integration-tests-v4-sm` job in .github/workflows/pull_request.yml.
+#
+# Required (shell env or .env at repo root):
+#   CONDUCTOR_SERVER_URL, CONDUCTOR_AUTH_KEY, CONDUCTOR_AUTH_SECRET
+#
+# Usage:
+#   scripts/run-integration-v4.sh [-n|--node 20|22|24] [-s|--shard 1|2|3]
+#                                 [-t|--test <path|pattern>] [-c|--coverage] [-- jest args]
+# Examples:
+#   scripts/run-integration-v4.sh                       # node 24, shard 1/3
+#   scripts/run-integration-v4.sh --node 22 --shard 2   # node 22, shard 2/3
+#   scripts/run-integration-v4.sh --coverage            # include the coverage report (off by default)
+#   # Run a single test file (--test disables sharding so the file isn't filtered out):
+#   scripts/run-integration-v4.sh --test src/integration-tests/WorkflowExecutor.test.ts
+#   scripts/run-integration-v4.sh --node 20 --test WorkflowExecutor
+#   scripts/run-integration-v4.sh -- --testPathPatterns="WorkflowExecutor"
+set -euo pipefail
+
+NODE_VERSION=24
+SHARD=1
+TOTAL_SHARDS=3
+TEST_PATTERN=""
+COVERAGE=0
+
+# Print the leading comment block (everything after the shebang up to the first
+# non-comment line) as help text, so it stays in sync with the header above.
+usage() { awk 'NR>1 && /^#/ {sub(/^# ?/, ""); print; next} NR>1 {exit}' "$0"; }
+
+extra=()
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -n|--node)  NODE_VERSION="${2:?--node needs a value}"; shift 2 ;;
+    -s|--shard) SHARD="${2:?--shard needs a value}"; shift 2 ;;
+    -t|--test)  TEST_PATTERN="${2:?--test needs a path or pattern}"; shift 2 ;;
+    -c|--coverage) COVERAGE=1; shift ;;
+    -h|--help)  usage; exit 0 ;;
+    --)         shift; extra=("$@"); break ;;
+    *) echo "Unknown argument: $1" >&2; usage; exit 1 ;;
+  esac
+done
+
+case "$NODE_VERSION" in 20|22|24) ;; *) echo "Error: --node must be 20, 22, or 24 (got '$NODE_VERSION')" >&2; exit 1 ;; esac
+case "$SHARD" in 1|2|3) ;; *) echo "Error: --shard must be 1, 2, or 3 (got '$SHARD')" >&2; exit 1 ;; esac
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+cd "${REPO_ROOT}"
+
+if [[ -f .env ]]; then
+  set -a
+  # shellcheck disable=SC1091
+  source .env
+  set +a
+fi
+
+missing=()
+for var in CONDUCTOR_SERVER_URL CONDUCTOR_AUTH_KEY CONDUCTOR_AUTH_SECRET; do
+  if [[ -z "${!var:-}" ]]; then
+    missing+=("$var")
+  fi
+done
+if (( ${#missing[@]} > 0 )); then
+  echo "Error: missing required environment variable(s): ${missing[*]}" >&2
+  echo "Set them in your shell or in a .env file at the repo root (see .env.example)." >&2
+  exit 1
+fi
+
+export ORKES_BACKEND_VERSION=4
+export CONDUCTOR_REQUEST_TIMEOUT_MS="${CONDUCTOR_REQUEST_TIMEOUT_MS:-120000}"
+export CONDUCTOR_RETRY_SERVER_ERRORS="${CONDUCTOR_RETRY_SERVER_ERRORS:-true}"
+
+test_cmd=(npm run test:integration:v4 -- --ci --runInBand --testTimeout=120000)
+if [[ "${COVERAGE}" == "1" ]]; then
+  test_cmd+=(--coverage)
+fi
+if [[ -n "${TEST_PATTERN}" ]]; then
+  # Targeting a specific test: don't shard, or the file may be filtered out.
+  test_cmd+=("--testPathPatterns=${TEST_PATTERN}")
+else
+  test_cmd+=("--shard=${SHARD}/${TOTAL_SHARDS}")
+fi
+if (( ${#extra[@]} > 0 )); then
+  test_cmd+=("${extra[@]}")
+fi
+
+if [[ -n "${TEST_PATTERN}" ]]; then
+  echo "Running v4 integration tests | node ${NODE_VERSION} | test '${TEST_PATTERN}' | server ${CONDUCTOR_SERVER_URL}"
+else
+  echo "Running v4 integration tests | node ${NODE_VERSION} | shard ${SHARD}/${TOTAL_SHARDS} | server ${CONDUCTOR_SERVER_URL}"
+fi
+
+if command -v fnm >/dev/null 2>&1; then
+  fnm exec --using="${NODE_VERSION}" -- "${test_cmd[@]}"
+else
+  current_major="$(node -v 2>/dev/null | sed -E 's/^v([0-9]+).*/\1/')"
+  if [[ "${current_major}" != "${NODE_VERSION}" ]]; then
+    echo "Error: requested Node ${NODE_VERSION} but current Node is v${current_major:-unknown} and fnm is not installed." >&2
+    echo "Install fnm (see SDK_DEVELOPMENT.md) or switch your Node version manually before running." >&2
+    exit 1
+  fi
+  "${test_cmd[@]}"
+fi