diff --git a/.github/workflows/e2e-image.yml b/.github/workflows/e2e-image.yml new file mode 100644 index 0000000000..c21d2d9670 --- /dev/null +++ b/.github/workflows/e2e-image.yml @@ -0,0 +1,83 @@ +name: e2e-image +on: + push: + branches: + - main + paths: + - "sdks/go/**" + - "sdks/python/**" + - "sdks/typescript/**" + - "sdks/ruby/**" + +jobs: + build-push-e2e-go: + name: hatchet-e2e-go + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Login to GHCR + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + - name: Build and push + run: | + DOCKER_BUILDKIT=1 docker build -f ./build/package/e2e-go.dockerfile \ + -t ghcr.io/hatchet-dev/hatchet/hatchet-e2e-go:${{ github.sha }} \ + -t ghcr.io/hatchet-dev/hatchet/hatchet-e2e-go:main \ + --platform linux/amd64 \ + . + docker push ghcr.io/hatchet-dev/hatchet/hatchet-e2e-go:${{ github.sha }} + docker push ghcr.io/hatchet-dev/hatchet/hatchet-e2e-go:main + + build-push-e2e-python: + name: hatchet-e2e-python + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Login to GHCR + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + - name: Build and push + run: | + DOCKER_BUILDKIT=1 docker build -f ./build/package/e2e-python.dockerfile \ + -t ghcr.io/hatchet-dev/hatchet/hatchet-e2e-python:${{ github.sha }} \ + -t ghcr.io/hatchet-dev/hatchet/hatchet-e2e-python:main \ + --platform linux/amd64 \ + . + docker push ghcr.io/hatchet-dev/hatchet/hatchet-e2e-python:${{ github.sha }} + docker push ghcr.io/hatchet-dev/hatchet/hatchet-e2e-python:main + + build-push-e2e-typescript: + name: hatchet-e2e-typescript + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Login to GHCR + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + - name: Build and push + run: | + DOCKER_BUILDKIT=1 docker build -f ./build/package/e2e-typescript.dockerfile \ + -t ghcr.io/hatchet-dev/hatchet/hatchet-e2e-typescript:${{ github.sha }} \ + -t ghcr.io/hatchet-dev/hatchet/hatchet-e2e-typescript:main \ + --platform linux/amd64 \ + . + docker push ghcr.io/hatchet-dev/hatchet/hatchet-e2e-typescript:${{ github.sha }} + docker push ghcr.io/hatchet-dev/hatchet/hatchet-e2e-typescript:main + + build-push-e2e-ruby: + name: hatchet-e2e-ruby + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Login to GHCR + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + - name: Build and push + run: | + DOCKER_BUILDKIT=1 docker build -f ./build/package/e2e-ruby.dockerfile \ + -t ghcr.io/hatchet-dev/hatchet/hatchet-e2e-ruby:${{ github.sha }} \ + -t ghcr.io/hatchet-dev/hatchet/hatchet-e2e-ruby:main \ + --platform linux/amd64 \ + . + docker push ghcr.io/hatchet-dev/hatchet/hatchet-e2e-ruby:${{ github.sha }} + docker push ghcr.io/hatchet-dev/hatchet/hatchet-e2e-ruby:main diff --git a/.github/workflows/sdk-python.yml b/.github/workflows/sdk-python.yml index d56fd3fd0b..91ee63b418 100644 --- a/.github/workflows/sdk-python.yml +++ b/.github/workflows/sdk-python.yml @@ -147,7 +147,7 @@ jobs: - name: Run pytest run: | echo "Using HATCHET_CLIENT_NAMESPACE: $HATCHET_CLIENT_NAMESPACE" - + export HATCHET_CLIENT_SERVER_URL=http://localhost:8080 poetry run pytest -s -vvv --maxfail=5 --capture=no --retries 3 --retry-delay 2 -n 8 - name: Test with wheel diff --git a/build/package/e2e-go.dockerfile b/build/package/e2e-go.dockerfile new file mode 100644 index 0000000000..4168b8993f --- /dev/null +++ b/build/package/e2e-go.dockerfile @@ -0,0 +1,31 @@ +# Base Go environment +# ------------------- +FROM golang:1.25-alpine as base +WORKDIR /hatchet + +COPY go.mod go.sum ./ + +RUN go mod download + +COPY /pkg ./pkg +COPY /internal ./internal +COPY /api ./api +COPY /sdks/go ./sdks/go + +# Go build environment +# -------------------- +FROM base AS build-go + +RUN go test -c -tags e2e -v -o ./bin/e2e-test ./sdks/go/e2e/ + +# Deployment environment +# ---------------------- +FROM alpine AS deployment + +WORKDIR /hatchet + +RUN apk update && apk add --no-cache ca-certificates tzdata + +COPY --from=build-go /hatchet/bin/e2e-test /hatchet/ + +CMD ["/hatchet/e2e-test", "-test.v", "-test.timeout=10m"] diff --git a/build/package/e2e-python.dockerfile b/build/package/e2e-python.dockerfile new file mode 100644 index 0000000000..59fc26989b --- /dev/null +++ b/build/package/e2e-python.dockerfile @@ -0,0 +1,13 @@ +# Base Python environment +# ----------------------- +FROM python:3.13-slim AS deployment + +WORKDIR /hatchet/sdks/python + +RUN pip install --no-cache-dir poetry==2.3.0 + +COPY sdks/python/ . + +RUN poetry install --no-interaction --all-extras + +CMD ["poetry", "run", "pytest", "-s", "-vvv", "-x", "--capture=no"] diff --git a/build/package/e2e-ruby-entrypoint.sh b/build/package/e2e-ruby-entrypoint.sh new file mode 100644 index 0000000000..6940cb936a --- /dev/null +++ b/build/package/e2e-ruby-entrypoint.sh @@ -0,0 +1,44 @@ +#!/bin/sh + +WORKER_PID="" +FINAL_EXIT=0 + +cleanup() { + if [ -n "$WORKER_PID" ]; then + kill -9 "$WORKER_PID" 2>/dev/null || true + wait "$WORKER_PID" 2>/dev/null || true + fi +} +trap cleanup EXIT INT TERM + +echo "=== Running integration tests ===" +cd /hatchet/sdks/ruby/src +timeout 300 bundle exec rspec spec/integration/ --format documentation --tag integration +INTEGRATION_EXIT=$? +[ $INTEGRATION_EXIT -ne 0 ] && FINAL_EXIT=$INTEGRATION_EXIT + +echo "=== Starting example worker ===" +cd /hatchet/sdks/ruby/examples +HATCHET_CLIENT_WORKER_HEALTHCHECK_ENABLED=true \ +HATCHET_CLIENT_WORKER_HEALTHCHECK_PORT=8001 \ +bundle exec ruby worker.rb & +WORKER_PID=$! + +for i in $(seq 1 60); do + if curl -sf http://localhost:8001/health > /dev/null 2>&1; then + echo "Worker healthy after ${i}s" + break + fi + if [ "$i" -eq 60 ]; then + echo "Worker failed to start within 60s" + exit 1 + fi + sleep 1 +done + +echo "=== Running e2e tests ===" +timeout 1200 bundle exec rspec -f d --fail-fast +E2E_EXIT=$? +[ $E2E_EXIT -ne 0 ] && FINAL_EXIT=$E2E_EXIT + +exit $FINAL_EXIT diff --git a/build/package/e2e-ruby.dockerfile b/build/package/e2e-ruby.dockerfile new file mode 100644 index 0000000000..46bdeea098 --- /dev/null +++ b/build/package/e2e-ruby.dockerfile @@ -0,0 +1,18 @@ +# Base Ruby environment +# --------------------- +FROM ruby:3.2 AS deployment + +WORKDIR /hatchet/sdks/ruby + +RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* + +COPY sdks/ruby/src/ src/ +RUN cd src && bundle lock --add-platform x86_64-linux && bundle install + +COPY sdks/ruby/examples/ examples/ +RUN cd examples && bundle lock --add-platform x86_64-linux && bundle install + +COPY build/package/e2e-ruby-entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +CMD ["/entrypoint.sh"] diff --git a/build/package/e2e-typescript.dockerfile b/build/package/e2e-typescript.dockerfile new file mode 100644 index 0000000000..5450d58c8a --- /dev/null +++ b/build/package/e2e-typescript.dockerfile @@ -0,0 +1,13 @@ +# Base Node environment +# --------------------- +FROM node:20-alpine AS deployment + +WORKDIR /hatchet/sdks/typescript + +RUN corepack enable && corepack prepare pnpm@10.16.1 --activate + +COPY sdks/typescript/ . + +RUN pnpm install --frozen-lockfile + +CMD ["pnpm", "run", "test:e2e"] diff --git a/examples/python/durable_eviction/worker.py b/examples/python/durable_eviction/worker.py index 2b63ac1da5..2e9d2d2998 100644 --- a/examples/python/durable_eviction/worker.py +++ b/examples/python/durable_eviction/worker.py @@ -149,7 +149,7 @@ async def capacity_evictable_sleep( ) async def non_evictable_sleep(input: EmptyModel, ctx: DurableContext) -> dict[str, Any]: """Has eviction disabled -- should never be evicted.""" - await ctx.aio_sleep_for(timedelta(seconds=10)) + await ctx.aio_sleep_for(timedelta(seconds=30)) return {"status": "completed"} diff --git a/pkg/worker/context.go b/pkg/worker/context.go index 27706d0db3..d68f9e842f 100644 --- a/pkg/worker/context.go +++ b/pkg/worker/context.go @@ -20,6 +20,7 @@ import ( "github.com/hatchet-dev/hatchet/pkg/client" "github.com/hatchet-dev/hatchet/pkg/client/create" "github.com/hatchet-dev/hatchet/pkg/client/types" + clientconfig "github.com/hatchet-dev/hatchet/pkg/config/client" "github.com/hatchet-dev/hatchet/pkg/worker/condition" ) @@ -1061,6 +1062,8 @@ func (d *durableHatchetContext) SleepFor(duration time.Duration) (*SingleWaitRes // WaitForEvent implements the DurableHatchetContext.WaitForEvent method. func (d *durableHatchetContext) WaitForEvent(eventKey, expression string) (*SingleWaitResult, error) { + namespace := d.c.Namespace() + eventKey = clientconfig.ApplyNamespace(eventKey, &namespace) wr, err := d.waitFor(condition.UserEventCondition(eventKey, expression), "wait_for_event", eventKey) if err != nil { diff --git a/sdks/go/e2e/durable_test.go b/sdks/go/e2e/durable_test.go index 2aade8e5cd..4268d97218 100644 --- a/sdks/go/e2e/durable_test.go +++ b/sdks/go/e2e/durable_test.go @@ -44,7 +44,9 @@ func TestDurableWorkflow(t *testing.T) { id := uniqueID() - time.Sleep(time.Duration(sleepTime+10) * time.Second) + // Wait for the run to start, then let the internal SleepFor(sleepTime) finish before pushing the event. + pollUntilRunStatus(t, ctx, sharedClient, ref.RunId, string(rest.V1TaskStatusRUNNING)) + time.Sleep(time.Duration(sleepTime+3) * time.Second) err = sharedClient.Events().Push(ctx, eventKey, AwaitedEvent{ID: id}) require.NoError(t, err) @@ -68,15 +70,21 @@ func TestDurableSleepCancelReplay(t *testing.T) { ref, err := testWaitForSleepTwice.RunNoWait(ctx, EmptyInput{}) require.NoError(t, err) - time.Sleep(time.Duration(sleepTime/2) * time.Second) + pollUntilRunStatus(t, ctx, sharedClient, ref.RunId, string(rest.V1TaskStatusRUNNING)) _, err = sharedClient.Runs().Cancel(ctx, rest.V1CancelTaskRequest{ ExternalIds: toUUIDs(ref.RunId), }) require.NoError(t, err) - // Wait for cancellation - time.Sleep(2 * time.Second) + // Wait for cancellation to propagate before replaying. + pollUntil(t, ctx, func() (bool, error) { + status, err := sharedClient.Runs().GetStatus(ctx, ref.RunId) + if err != nil { + return false, err + } + return *status == rest.V1TaskStatusCANCELLED, nil + }) replayStart := time.Now() _, err = sharedClient.Runs().Replay(ctx, rest.V1ReplayTaskRequest{ @@ -92,8 +100,8 @@ func TestDurableSleepCancelReplay(t *testing.T) { err = result.TaskOutput("wait-for-sleep-twice").Into(&output) require.NoError(t, err) - assert.Less(t, output["runtime"], float64(sleepTime)) - assert.LessOrEqual(t, replayElapsed, float64(sleepTime)) + assert.Less(t, output["runtime"], float64(sleepTime)+timingTolerance) + assert.LessOrEqual(t, replayElapsed, float64(sleepTime)+timingTolerance) } func TestDurableChildSpawn(t *testing.T) { @@ -122,6 +130,7 @@ func TestDurableChildBulkSpawn(t *testing.T) { require.NoError(t, err) outputs, ok := m["child_outputs"].([]any) require.True(t, ok, "expected child_outputs to be an array") + assert.GreaterOrEqual(t, len(outputs), n-1) assert.LessOrEqual(t, len(outputs), n) @@ -145,7 +154,9 @@ func TestDurableSleepEventSpawnReplay(t *testing.T) { ref, err := testDurableSleepEventSpawn.RunNoWait(ctx, EmptyInput{}) require.NoError(t, err) - time.Sleep(time.Duration(sleepTime+5) * time.Second) + // Wait for the run to start, then let the internal SleepFor(sleepTime) finish before pushing the event. + pollUntilRunStatus(t, ctx, sharedClient, ref.RunId, string(rest.V1TaskStatusRUNNING)) + time.Sleep(time.Duration(sleepTime+3) * time.Second) err = sharedClient.Events().Push(ctx, eventKey, map[string]string{"test": "test"}) require.NoError(t, err) @@ -173,7 +184,7 @@ func TestDurableSleepEventSpawnReplay(t *testing.T) { replayChild, ok := rm["child_output"].(map[string]any) require.True(t, ok) assert.Equal(t, "hello from child 1", replayChild["message"]) - assert.Less(t, replayElapsed, float64(sleepTime)) + assert.Less(t, replayElapsed, float64(sleepTime)+timingTolerance) } func TestDurableCompletedReplay(t *testing.T) { @@ -207,8 +218,8 @@ func TestDurableCompletedReplay(t *testing.T) { var replayOutput map[string]float64 err = replayResult.TaskOutput("wait-for-sleep-twice").Into(&replayOutput) require.NoError(t, err) - assert.Less(t, replayOutput["runtime"], float64(sleepTime)) - assert.Less(t, elapsed, float64(sleepTime)) + assert.Less(t, replayOutput["runtime"], float64(sleepTime)+timingTolerance) + assert.Less(t, elapsed, float64(sleepTime)+timingTolerance) } func TestDurableSpawnDAG(t *testing.T) { @@ -306,7 +317,7 @@ func TestDurableReplayReset(t *testing.T) { durations := []float64{resetOutput.Sleep1Duration, resetOutput.Sleep2Duration, resetOutput.Sleep3Duration} for i, d := range durations { if int64(i+1) < nodeID { - assert.Less(t, d, float64(replayResetSleepTime)) + assert.Less(t, d, float64(replayResetSleepTime)+timingTolerance) } else { assert.GreaterOrEqual(t, d, float64(replayResetSleepTime)) } @@ -369,7 +380,7 @@ func TestDurableBranchingOffBranch(t *testing.T) { err = resetResult2.TaskOutput("durable-replay-reset").Into(&resetOutput2) require.NoError(t, err) - assert.Less(t, resetOutput2.Sleep1Duration, float64(replayResetSleepTime)) + assert.Less(t, resetOutput2.Sleep1Duration, float64(replayResetSleepTime)+timingTolerance) assert.GreaterOrEqual(t, resetOutput2.Sleep2Duration, float64(replayResetSleepTime)) assert.GreaterOrEqual(t, resetOutput2.Sleep3Duration, float64(replayResetSleepTime)) assert.GreaterOrEqual(t, resetElapsed2, float64(2*replayResetSleepTime)) @@ -407,7 +418,7 @@ func TestDurableMemoizationViaReplay(t *testing.T) { require.NoError(t, err) assert.GreaterOrEqual(t, duration1, float64(sleepTime)) - assert.Less(t, duration2, 1.0) + assert.Less(t, duration2, 1.1) assert.Equal(t, output1.Message, output2.Message) } diff --git a/sdks/go/e2e/e2e_test.go b/sdks/go/e2e/e2e_test.go index b02d37ebb3..bb96abb31f 100644 --- a/sdks/go/e2e/e2e_test.go +++ b/sdks/go/e2e/e2e_test.go @@ -16,9 +16,9 @@ import ( ) const ( - defaultTimeout = 5 * time.Minute - pollInterval = 200 * time.Millisecond - maxPolls = 150 + defaultTimeout = 5 * time.Minute + pollInterval = 200 * time.Millisecond + timingTolerance = 1.0 // seconds of slack for scheduling/measurement overhead in timing assertions ) var ( @@ -62,13 +62,13 @@ func uniqueID() string { return uuid.New().String() } -// pollUntil polls fn every pollInterval until it returns true or maxPolls is reached. +// pollUntil polls fn every pollInterval until it returns true or the context is done. func pollUntil(t *testing.T, ctx context.Context, fn func() (bool, error)) { t.Helper() - for i := 0; i < maxPolls; i++ { + for { done, err := fn() if err != nil { - t.Logf("poll error (attempt %d): %v", i, err) + t.Logf("poll error: %v", err) } if done { return @@ -79,7 +79,6 @@ func pollUntil(t *testing.T, ctx context.Context, fn func() (bool, error)) { case <-time.After(pollInterval): } } - t.Fatalf("polling timed out after %d attempts", maxPolls) } // pollUntilRunStatus polls run details until any task reaches the given status. diff --git a/sdks/go/e2e/worker.go b/sdks/go/e2e/worker.go index 2b8118cffc..4af79b52d5 100644 --- a/sdks/go/e2e/worker.go +++ b/sdks/go/e2e/worker.go @@ -595,8 +595,8 @@ func startTestWorker(client *hatchet.Client) (*hatchet.Worker, func() error, err return nil, nil, fmt.Errorf("failed to start worker: %w", err) } - // Give the worker a moment to register - time.Sleep(2 * time.Second) + // Give the worker a moment to register with the server + time.Sleep(5 * time.Second) return worker, cleanup, nil } diff --git a/sdks/go/workflow.go b/sdks/go/workflow.go index 4517591cd3..9cb8687a3e 100644 --- a/sdks/go/workflow.go +++ b/sdks/go/workflow.go @@ -749,7 +749,7 @@ func (w *Workflow) RunMany(ctx context.Context, inputs []RunManyOpt) ([]Workflow var wg sync.WaitGroup var errs []error var errsMutex sync.Mutex - + var workflowRefsMutex sync.Mutex wg.Add(len(inputs)) for _, input := range inputs { @@ -763,8 +763,9 @@ func (w *Workflow) RunMany(ctx context.Context, inputs []RunManyOpt) ([]Workflow errsMutex.Unlock() return } - + workflowRefsMutex.Lock() workflowRefs = append(workflowRefs, *workflowRef) + workflowRefsMutex.Unlock() }() } diff --git a/sdks/python/examples/bulk_operations/test_bulk_replay.py b/sdks/python/examples/bulk_operations/test_bulk_replay.py index ef85735d95..f1a8abacf7 100644 --- a/sdks/python/examples/bulk_operations/test_bulk_replay.py +++ b/sdks/python/examples/bulk_operations/test_bulk_replay.py @@ -3,6 +3,8 @@ from uuid import uuid4 import pytest +import tenacity +from tenacity import stop_after_attempt, wait_exponential from examples.bulk_operations.worker import ( bulk_replay_test_1, @@ -10,6 +12,8 @@ bulk_replay_test_3, ) from hatchet_sdk import BulkCancelReplayOpts, Hatchet, RunFilter +from hatchet_sdk.clients.rest.models.v1_task_summary_list import V1TaskSummaryList +from hatchet_sdk.clients.rest.models.v1_task_summary import V1TaskSummary from hatchet_sdk.clients.rest.models.v1_task_status import V1TaskStatus @@ -17,7 +21,7 @@ async def test_bulk_replay(hatchet: Hatchet) -> None: test_run_id = str(uuid4()) n = 100 - + test_start = datetime.now(tz=timezone.utc) with pytest.raises(Exception): await bulk_replay_test_1.aio_run_many( [ @@ -65,20 +69,40 @@ async def test_bulk_replay(hatchet: Hatchet) -> None: opts=BulkCancelReplayOpts( filters=RunFilter( workflow_ids=workflow_ids, - since=datetime.now(tz=timezone.utc) - timedelta(minutes=2), + since=test_start, additional_metadata={"test_run_id": test_run_id}, ) ) ) - await asyncio.sleep(10) + await asyncio.sleep(20) - runs = await hatchet.runs.aio_list( - workflow_ids=workflow_ids, - since=datetime.now(tz=timezone.utc) - timedelta(minutes=2), - additional_metadata={"test_run_id": test_run_id}, - limit=1000, + @tenacity.retry( + stop=stop_after_attempt(10), wait=wait_exponential(multiplier=1, min=4, max=10) ) + async def get_runs() -> V1TaskSummaryList: + runs = await hatchet.runs.aio_list( + workflow_ids=workflow_ids, + since=test_start, + additional_metadata={"test_run_id": test_run_id}, + limit=1000, + ) + + def predicate(r: V1TaskSummary) -> bool: + return ( + r.status == V1TaskStatus.COMPLETED # type: ignore[return-value] + and r.retry_count + and r.retry_count >= 1 + and r.attempt + and r.attempt >= 2 + ) + + for r in runs.rows: + if not predicate(r): + raise Exception + return runs + + runs = await get_runs() assert len(runs.rows) == n + 1 + (n // 2 - 1) + (n // 2 - 2) diff --git a/sdks/python/examples/cancellation/test_cancellation.py b/sdks/python/examples/cancellation/test_cancellation.py index 237bd840b2..6eff4a9bfa 100644 --- a/sdks/python/examples/cancellation/test_cancellation.py +++ b/sdks/python/examples/cancellation/test_cancellation.py @@ -10,7 +10,7 @@ async def test_cancellation(hatchet: Hatchet) -> None: ref = await cancellation_workflow.aio_run(wait_for_result=False) - for _ in range(30): + for _ in range(120): run = await hatchet.runs.aio_get_details(ref.workflow_run_id) if run.status in [RunStatus.RUNNING, RunStatus.QUEUED]: diff --git a/sdks/python/examples/concurrency_workflow_level/test_workflow_level_concurrency.py b/sdks/python/examples/concurrency_workflow_level/test_workflow_level_concurrency.py index aeb1d0a350..4ebbbe051a 100644 --- a/sdks/python/examples/concurrency_workflow_level/test_workflow_level_concurrency.py +++ b/sdks/python/examples/concurrency_workflow_level/test_workflow_level_concurrency.py @@ -74,6 +74,7 @@ async def test_workflow_level_concurrency(hatchet: Hatchet) -> None: wait_for_result=False, ) + await asyncio.sleep(10) await asyncio.gather(*[r.aio_result() for r in run_refs]) workflows = ( diff --git a/sdks/python/examples/conditions/test_conditions.py b/sdks/python/examples/conditions/test_conditions.py index 7e51deeb92..b3205e25b5 100644 --- a/sdks/python/examples/conditions/test_conditions.py +++ b/sdks/python/examples/conditions/test_conditions.py @@ -3,6 +3,7 @@ import pytest from examples.conditions.worker import task_condition_workflow +from examples.test_utils import wait_for_running_status from hatchet_sdk import Hatchet @@ -10,10 +11,12 @@ async def test_waits(hatchet: Hatchet) -> None: ref = task_condition_workflow.run(wait_for_result=False) + await wait_for_running_status(hatchet, ref.workflow_run_id) await asyncio.sleep(15) hatchet.event.push("skip_on_event:skip", {}) hatchet.event.push("wait_for_event:start", {}) + await asyncio.sleep(5) result = await ref.aio_result() diff --git a/sdks/python/examples/durable/test_durable.py b/sdks/python/examples/durable/test_durable.py index 9f39d1c020..34665f77ca 100644 --- a/sdks/python/examples/durable/test_durable.py +++ b/sdks/python/examples/durable/test_durable.py @@ -31,6 +31,10 @@ ) from hatchet_sdk import Hatchet +from examples.test_utils import wait_for_running_status + +TIMING_TOLERANCE = 1.0 + requires_durable_eviction = pytest.mark.usefixtures("_skip_unless_durable_eviction") @@ -39,7 +43,8 @@ async def test_durable_workflow(hatchet: Hatchet) -> None: ref = await durable_workflow.aio_run(wait_for_result=False) id = str(uuid4()) - await asyncio.sleep(SLEEP_TIME + 10) + await wait_for_running_status(hatchet, ref.workflow_run_id) + await asyncio.sleep(SLEEP_TIME + 3) event = await hatchet.event.aio_push( EVENT_KEY, AwaitedEvent(id=id).model_dump(mode="json") @@ -81,8 +86,7 @@ async def test_durable_workflow(hatchet: Hatchet) -> None: async def test_durable_sleep_cancel_replay(hatchet: Hatchet) -> None: first_sleep = await wait_for_sleep_twice.aio_run(wait_for_result=False) - await asyncio.sleep(SLEEP_TIME / 2) - + await wait_for_running_status(hatchet, first_sleep.workflow_run_id) await hatchet.runs.aio_cancel(first_sleep.workflow_run_id) await first_sleep.aio_result() @@ -95,8 +99,8 @@ async def test_durable_sleep_cancel_replay(hatchet: Hatchet) -> None: second_sleep_result = await first_sleep.aio_result() replay_elapsed = time.time() - replay_start - assert second_sleep_result["runtime"] < SLEEP_TIME - assert replay_elapsed <= SLEEP_TIME + assert second_sleep_result["runtime"] < SLEEP_TIME + TIMING_TOLERANCE + assert replay_elapsed <= SLEEP_TIME + TIMING_TOLERANCE @pytest.mark.asyncio(loop_scope="session") @@ -122,7 +126,8 @@ async def test_durable_sleep_event_spawn_replay(hatchet: Hatchet) -> None: start = time.time() ref = await durable_sleep_event_spawn.aio_run(wait_for_result=False) - await asyncio.sleep(SLEEP_TIME + 5) + await wait_for_running_status(hatchet, ref.workflow_run_id) + await asyncio.sleep(SLEEP_TIME + 3) hatchet.event.push(EVENT_KEY, {"test": "test"}) result = await ref.aio_result() @@ -137,7 +142,7 @@ async def test_durable_sleep_event_spawn_replay(hatchet: Hatchet) -> None: replay_elapsed = time.time() - replay_start assert replayed_result["child_output"] == {"message": "hello from child 1"} - assert replay_elapsed < SLEEP_TIME + assert replay_elapsed < SLEEP_TIME + TIMING_TOLERANCE @requires_durable_eviction @@ -157,8 +162,8 @@ async def test_durable_completed_replay(hatchet: Hatchet) -> None: replayed_result = await ref.aio_result() elapsed = time.time() - start - assert replayed_result["runtime"] < SLEEP_TIME - assert elapsed < SLEEP_TIME + assert replayed_result["runtime"] < SLEEP_TIME + TIMING_TOLERANCE + assert elapsed < SLEEP_TIME + TIMING_TOLERANCE @pytest.mark.asyncio(loop_scope="session") @@ -222,7 +227,7 @@ async def test_durable_replay_reset(hatchet: Hatchet, node_id: int) -> None: for i, duration in enumerate(durations, start=1): if i < node_id: - assert duration < REPLAY_RESET_SLEEP_TIME + assert duration < REPLAY_RESET_SLEEP_TIME + TIMING_TOLERANCE else: assert duration >= REPLAY_RESET_SLEEP_TIME @@ -272,7 +277,7 @@ async def test_durable_branching_off_branch(hatchet: Hatchet) -> None: reset_result = await ref.aio_result() reset_elapsed = time.time() - start - assert reset_result.sleep_1_duration < REPLAY_RESET_SLEEP_TIME + assert reset_result.sleep_1_duration < REPLAY_RESET_SLEEP_TIME + TIMING_TOLERANCE assert reset_result.sleep_2_duration >= REPLAY_RESET_SLEEP_TIME assert reset_result.sleep_3_duration >= REPLAY_RESET_SLEEP_TIME @@ -329,7 +334,7 @@ async def test_event_lookback_before_wait(hatchet: Hatchet) -> None: result = await wait_for_event_lookback.aio_run(EventLookbackInput(user_id=user_id)) assert ( - result.elapsed < 1 + result.elapsed < 1 + TIMING_TOLERANCE ), "Event lookback should find the event that was pushed before the wait started, so should be basically instantaneous" assert result.event.order == "first" @@ -343,7 +348,7 @@ async def test_or_group_event_lookback_before_wait(hatchet: Hatchet) -> None: result = await wait_for_or_event_lookback.aio_run(EventLookbackInput(scope=scope)) - assert result.elapsed < SLEEP_TIME + assert result.elapsed < SLEEP_TIME + TIMING_TOLERANCE @pytest.mark.asyncio(loop_scope="session") @@ -367,7 +372,7 @@ async def test_two_event_waits_second_pushed_first(hatchet: Hatchet) -> None: result = await ref.aio_result() - assert result.elapsed < SLEEP_TIME + assert result.elapsed < SLEEP_TIME + TIMING_TOLERANCE assert result.event1.order == "first" assert result.event2.order == "second" diff --git a/sdks/python/examples/durable_eviction/test_durable_eviction.py b/sdks/python/examples/durable_eviction/test_durable_eviction.py index a944c391f2..8f2fa42538 100644 --- a/sdks/python/examples/durable_eviction/test_durable_eviction.py +++ b/sdks/python/examples/durable_eviction/test_durable_eviction.py @@ -94,7 +94,7 @@ async def test_non_evictable_task_not_evicted(hatchet: Hatchet) -> None: ref = await non_evictable_sleep.aio_run(wait_for_result=False) await _poll_until_status(hatchet, ref.workflow_run_id, V1TaskStatus.RUNNING) - await asyncio.sleep(7) # Past EVICTION_TTL (5s), task still sleeping (10s total) + await asyncio.sleep(20) # Past EVICTION_TTL (5s), task still sleeping (10s total) details = await hatchet.runs.aio_get_details(ref.workflow_run_id) assert not _has_evicted_task( details diff --git a/sdks/python/examples/durable_eviction/worker.py b/sdks/python/examples/durable_eviction/worker.py index 705e50aaf5..8cab7947d8 100644 --- a/sdks/python/examples/durable_eviction/worker.py +++ b/sdks/python/examples/durable_eviction/worker.py @@ -151,7 +151,7 @@ async def capacity_evictable_sleep( ) async def non_evictable_sleep(input: EmptyModel, ctx: DurableContext) -> dict[str, Any]: """Has eviction disabled -- should never be evicted.""" - await ctx.aio_sleep_for(timedelta(seconds=10)) + await ctx.aio_sleep_for(timedelta(seconds=30)) return {"status": "completed"} diff --git a/sdks/python/examples/on_failure/test_on_failure.py b/sdks/python/examples/on_failure/test_on_failure.py index 30cb2c08ca..fc53be40a9 100644 --- a/sdks/python/examples/on_failure/test_on_failure.py +++ b/sdks/python/examples/on_failure/test_on_failure.py @@ -1,9 +1,12 @@ import asyncio import pytest +import tenacity +from tenacity import stop_after_attempt, wait_exponential from examples.on_failure.worker import on_failure_wf from hatchet_sdk import Hatchet +from hatchet_sdk.clients.rest.models.v1_workflow_run_details import V1WorkflowRunDetails from hatchet_sdk.clients.rest.models.v1_task_status import V1TaskStatus @@ -17,10 +20,19 @@ async def test_run_timeout(hatchet: Hatchet) -> None: except Exception as e: assert "step1 failed" in str(e) - await asyncio.sleep(5) # Wait for the on_failure job to finish - - details = await hatchet.runs.aio_get(run.workflow_run_id) - + @tenacity.retry( + stop=stop_after_attempt(5), wait=wait_exponential(multiplier=1, min=4, max=10) + ) + async def get_runs() -> V1WorkflowRunDetails: + details = await hatchet.runs.aio_get(run.workflow_run_id) + if len(details.tasks) == 2 and all( + t.status in [V1TaskStatus.COMPLETED, V1TaskStatus.FAILED] + for t in details.tasks + ): + return details + raise Exception() + + details = await get_runs() assert len(details.tasks) == 2 assert sum(t.status == V1TaskStatus.COMPLETED for t in details.tasks) == 1 assert sum(t.status == V1TaskStatus.FAILED for t in details.tasks) == 1 diff --git a/sdks/python/examples/run_details/test_run_detail_getter.py b/sdks/python/examples/run_details/test_run_detail_getter.py index 3350e7042c..87f60a0288 100644 --- a/sdks/python/examples/run_details/test_run_detail_getter.py +++ b/sdks/python/examples/run_details/test_run_detail_getter.py @@ -1,9 +1,9 @@ -import asyncio from uuid import uuid4 import pytest from examples.run_details.worker import MockInput, run_detail_test_workflow +from examples.test_utils import wait_for_running_status from hatchet_sdk import Hatchet, RunStatus, V1TaskStatus @@ -18,9 +18,8 @@ async def test_run(hatchet: Hatchet) -> None: wait_for_result=False, ) - await asyncio.sleep(2) - - details = hatchet.runs.get_details(ref.workflow_run_id) + await wait_for_running_status(hatchet, ref.workflow_run_id) + details = await hatchet.runs.aio_get_details(ref.workflow_run_id) assert details.status == RunStatus.RUNNING assert details.input == mock_input.model_dump() diff --git a/sdks/python/examples/test_utils.py b/sdks/python/examples/test_utils.py new file mode 100644 index 0000000000..6585ecb34b --- /dev/null +++ b/sdks/python/examples/test_utils.py @@ -0,0 +1,76 @@ +import asyncio +from datetime import datetime + +import tenacity +from tenacity import stop_after_attempt, wait_exponential + +from hatchet_sdk import Hatchet, RunStatus +from hatchet_sdk.clients.rest.models.v1_event import V1Event +from hatchet_sdk.clients.rest.models.v1_task_summary import V1TaskSummary +from hatchet_sdk.clients.rest.models.v1_task_status import V1TaskStatus + + +async def wait_for_running_status( + hatchet: Hatchet, run_id: str, timeout: float = 60.0 +) -> None: + """Poll until the workflow run reaches RUNNING status or timeout is exceeded.""" + interval = 0.5 + max_iters = int(timeout / interval) + for _ in range(max_iters): + run = await hatchet.runs.aio_get_details(run_id) + if run.status == RunStatus.RUNNING: + return + await asyncio.sleep(interval) + + +async def wait_for_event( + hatchet: Hatchet, + webhook_name: str, + test_start: datetime, +) -> V1Event | None: + await asyncio.sleep(5) + + @tenacity.retry( + stop=stop_after_attempt(5), wait=wait_exponential(multiplier=1, min=4, max=10) + ) + async def get_events() -> V1Event | None: + events = await hatchet.event.aio_list(since=test_start) + if not events.rows: + raise Exception() + filtered_event = next( + ( + event + for event in events.rows + if event.triggering_webhook_name == webhook_name + ), + None, + ) + if not filtered_event: + raise Exception() + return filtered_event + + try: + return await get_events() + except tenacity.RetryError: + return None + + +async def wait_for_workflow_run( + hatchet: Hatchet, event_id: str, test_start: datetime +) -> V1TaskSummary | None: + @tenacity.retry( + stop=stop_after_attempt(5), wait=wait_exponential(multiplier=1, min=4, max=10) + ) + async def get_runs() -> V1TaskSummary: + runs = await hatchet.runs.aio_list( + since=test_start, + additional_metadata={ + "hatchet__event_id": event_id, + }, + ) + for row in runs.rows: + if row.status == V1TaskStatus.COMPLETED: + return row + raise Exception() + + return await get_runs() diff --git a/sdks/python/examples/webhook_with_scope/test_webhooks_with_scope.py b/sdks/python/examples/webhook_with_scope/test_webhooks_with_scope.py index 8542adad00..e8190beeef 100644 --- a/sdks/python/examples/webhook_with_scope/test_webhooks_with_scope.py +++ b/sdks/python/examples/webhook_with_scope/test_webhooks_with_scope.py @@ -1,21 +1,20 @@ -import asyncio from collections.abc import AsyncGenerator from contextlib import asynccontextmanager from datetime import datetime, timezone -from typing import Any +from typing import Any, AsyncGenerator from uuid import uuid4 import aiohttp import pytest +from aiohttp import ClientResponse +from examples.test_utils import wait_for_event, wait_for_workflow_run from examples.webhook_with_scope.worker import ( WebhookInputWithScope, WebhookInputWithStaticPayload, ) from hatchet_sdk import Hatchet -from hatchet_sdk.clients.rest.models.v1_event import V1Event from hatchet_sdk.clients.rest.models.v1_task_status import V1TaskStatus -from hatchet_sdk.clients.rest.models.v1_task_summary import V1TaskSummary from hatchet_sdk.clients.rest.models.v1_webhook import V1Webhook from hatchet_sdk.clients.rest.models.v1_webhook_basic_auth import V1WebhookBasicAuth from hatchet_sdk.clients.rest.models.v1_webhook_source_name import V1WebhookSourceName @@ -51,56 +50,18 @@ def test_start() -> datetime: return datetime.now(timezone.utc) +@asynccontextmanager async def send_webhook_request( url: str, body: dict[str, Any], username: str = TEST_BASIC_USERNAME, password: str = TEST_BASIC_PASSWORD, -) -> aiohttp.ClientResponse: +) -> AsyncGenerator[ClientResponse, Any]: auth = aiohttp.BasicAuth(username, password) async with aiohttp.ClientSession() as session: - return await session.post(url, json=body, auth=auth) - - -async def wait_for_event( - hatchet: Hatchet, - webhook_name: str, - test_start: datetime, -) -> V1Event | None: - await asyncio.sleep(5) - - events = await hatchet.event.aio_list(since=test_start) - - if events.rows is None: - return None - - return next( - ( - event - for event in events.rows - if event.triggering_webhook_name == webhook_name - ), - None, - ) - - -async def wait_for_workflow_run( - hatchet: Hatchet, event_id: str, test_start: datetime -) -> V1TaskSummary | None: - await asyncio.sleep(5) - - runs = await hatchet.runs.aio_list( - since=test_start, - additional_metadata={ - "hatchet__event_id": event_id, - }, - ) - - if len(runs.rows) == 0: - return None - - return runs.rows[0] + async with session.post(url, json=body, auth=auth) as response: + yield response @asynccontextmanager @@ -201,8 +162,8 @@ async def webhook_with_scope_and_static( hatchet.webhooks.delete(incoming_webhook.name) -def url(tenant_id: str, webhook_name: str) -> str: - return f"http://localhost:8080/api/v1/stable/tenants/{tenant_id}/webhooks/{webhook_name}" +def url(hatchet: Hatchet, webhook_name: str) -> str: + return f"{hatchet.config.server_url}/api/v1/stable/tenants/{hatchet.tenant_id}/webhooks/{webhook_name}" async def assert_has_runs( @@ -256,8 +217,8 @@ async def test_scope_expression_from_payload( ) as incoming_webhook: assert incoming_webhook.scope_expression == "input.scope" - async with await send_webhook_request( - url(hatchet.tenant_id, incoming_webhook.name), + async with send_webhook_request( + url(hatchet, incoming_webhook.name), webhook_body_with_scope.model_dump(), ) as response: assert response.status == 200 @@ -290,7 +251,7 @@ async def test_scope_expression_from_headers( auth = aiohttp.BasicAuth(TEST_BASIC_USERNAME, TEST_BASIC_PASSWORD) async with aiohttp.ClientSession() as session: async with await session.post( - url(hatchet.tenant_id, incoming_webhook.name), + url(hatchet, incoming_webhook.name), json=webhook_body.model_dump(), auth=auth, headers={"X-Custom-Scope": "header-scope-value"}, @@ -320,8 +281,8 @@ async def test_scope_expression_concatenation( test_run_id, scope_expression="'prefix:' + input.type + ':' + input.scope", ) as incoming_webhook: - async with await send_webhook_request( - url(hatchet.tenant_id, incoming_webhook.name), + async with send_webhook_request( + url(hatchet, incoming_webhook.name), webhook_body_with_scope.model_dump(), ) as response: assert response.status == 200 @@ -355,8 +316,8 @@ async def test_static_payload_adds_fields( ) as incoming_webhook: assert incoming_webhook.static_payload == static_payload - async with await send_webhook_request( - url(hatchet.tenant_id, incoming_webhook.name), + async with send_webhook_request( + url(hatchet, incoming_webhook.name), webhook_body_for_static.model_dump(), ) as response: assert response.status == 200 @@ -396,8 +357,8 @@ async def test_static_payload_overrides_existing_fields( test_run_id, static_payload=static_payload, ) as incoming_webhook: - async with await send_webhook_request( - url(hatchet.tenant_id, incoming_webhook.name), + async with send_webhook_request( + url(hatchet, incoming_webhook.name), incoming_body, ) as response: assert response.status == 200 @@ -436,8 +397,8 @@ async def test_scope_expression_uses_static_payload_values( scope_expression="input.customer_id", static_payload=static_payload, ) as incoming_webhook: - async with await send_webhook_request( - url(hatchet.tenant_id, incoming_webhook.name), + async with send_webhook_request( + url(hatchet, incoming_webhook.name), incoming_body, ) as response: assert response.status == 200 @@ -471,8 +432,8 @@ async def test_webhook_update_scope_expression( assert updated.scope_expression == "input.scope" assert updated.event_key_expression == incoming_webhook.event_key_expression - async with await send_webhook_request( - url(hatchet.tenant_id, incoming_webhook.name), + async with send_webhook_request( + url(hatchet, incoming_webhook.name), webhook_body_with_scope.model_dump(), ) as response: assert response.status == 200 diff --git a/sdks/python/examples/webhooks/test_webhooks.py b/sdks/python/examples/webhooks/test_webhooks.py index af70a9597b..91ed7f08ec 100644 --- a/sdks/python/examples/webhooks/test_webhooks.py +++ b/sdks/python/examples/webhooks/test_webhooks.py @@ -1,4 +1,3 @@ -import asyncio import base64 import hashlib import hmac @@ -12,11 +11,10 @@ import aiohttp import pytest +from examples.test_utils import wait_for_event, wait_for_workflow_run from examples.webhooks.worker import WebhookInput from hatchet_sdk import Hatchet -from hatchet_sdk.clients.rest.models.v1_event import V1Event from hatchet_sdk.clients.rest.models.v1_task_status import V1TaskStatus -from hatchet_sdk.clients.rest.models.v1_task_summary import V1TaskSummary from hatchet_sdk.clients.rest.models.v1_webhook import V1Webhook from hatchet_sdk.clients.rest.models.v1_webhook_api_key_auth import V1WebhookAPIKeyAuth from hatchet_sdk.clients.rest.models.v1_webhook_basic_auth import V1WebhookBasicAuth @@ -81,13 +79,14 @@ def create_hmac_signature( raise ValueError(f"Unsupported encoding: {encoding}") +@asynccontextmanager async def send_webhook_request( url: str, body: WebhookInput, auth_type: str, auth_data: dict[str, Any] | None = None, headers: dict[str, str] | None = None, -) -> aiohttp.ClientResponse: +) -> AsyncGenerator[aiohttp.ClientResponse, None]: request_headers = headers or {} auth = None @@ -106,49 +105,10 @@ async def send_webhook_request( request_headers[auth_data["header_name"]] = signature async with aiohttp.ClientSession() as session: - return await session.post( + async with session.post( url, json=body.model_dump(), auth=auth, headers=request_headers - ) - - -async def wait_for_event( - hatchet: Hatchet, - webhook_name: str, - test_start: datetime, -) -> V1Event | None: - await asyncio.sleep(5) - - events = await hatchet.event.aio_list(since=test_start) - - if events.rows is None: - return None - - return next( - ( - event - for event in events.rows - if event.triggering_webhook_name == webhook_name - ), - None, - ) - - -async def wait_for_workflow_run( - hatchet: Hatchet, event_id: str, test_start: datetime -) -> V1TaskSummary | None: - await asyncio.sleep(5) - - runs = await hatchet.runs.aio_list( - since=test_start, - additional_metadata={ - "hatchet__event_id": event_id, - }, - ) - - if len(runs.rows) == 0: - return None - - return runs.rows[0] + ) as response: + yield response @asynccontextmanager @@ -227,8 +187,8 @@ async def hmac_webhook( hatchet.webhooks.delete(incoming_webhook.name) -def url(tenant_id: str, webhook_name: str) -> str: - return f"http://localhost:8080/api/v1/stable/tenants/{tenant_id}/webhooks/{webhook_name}" +def url(hatchet: Hatchet, webhook_name: str) -> str: + return f"{hatchet.config.server_url}/api/v1/stable/tenants/{hatchet.tenant_id}/webhooks/{webhook_name}" async def assert_has_runs( @@ -277,8 +237,8 @@ async def test_basic_auth_success( webhook_body: WebhookInput, ) -> None: async with basic_auth_webhook(hatchet, test_run_id) as incoming_webhook: - async with await send_webhook_request( - url(hatchet.tenant_id, incoming_webhook.name), + async with send_webhook_request( + url(hatchet, incoming_webhook.name), webhook_body, "BASIC", {"username": TEST_BASIC_USERNAME, "password": TEST_BASIC_PASSWORD}, @@ -316,8 +276,8 @@ async def test_basic_auth_failure( ) -> None: """Test basic authentication failures.""" async with basic_auth_webhook(hatchet, test_run_id) as incoming_webhook: - async with await send_webhook_request( - url(hatchet.tenant_id, incoming_webhook.name), + async with send_webhook_request( + url(hatchet, incoming_webhook.name), webhook_body, "BASIC", {"username": username, "password": password}, @@ -339,8 +299,8 @@ async def test_basic_auth_missing_credentials( webhook_body: WebhookInput, ) -> None: async with basic_auth_webhook(hatchet, test_run_id) as incoming_webhook: - async with await send_webhook_request( - url(hatchet.tenant_id, incoming_webhook.name), webhook_body, "NONE" + async with send_webhook_request( + url(hatchet, incoming_webhook.name), webhook_body, "NONE" ) as response: assert response.status == 403 @@ -359,8 +319,8 @@ async def test_api_key_success( webhook_body: WebhookInput, ) -> None: async with api_key_webhook(hatchet, test_run_id) as incoming_webhook: - async with await send_webhook_request( - url(hatchet.tenant_id, incoming_webhook.name), + async with send_webhook_request( + url(hatchet, incoming_webhook.name), webhook_body, "API_KEY", {"header_name": TEST_API_KEY_HEADER, "api_key": TEST_API_KEY_VALUE}, @@ -395,8 +355,8 @@ async def test_api_key_failure( api_key: str, ) -> None: async with api_key_webhook(hatchet, test_run_id) as incoming_webhook: - async with await send_webhook_request( - url(hatchet.tenant_id, incoming_webhook.name), + async with send_webhook_request( + url(hatchet, incoming_webhook.name), webhook_body, "API_KEY", {"header_name": TEST_API_KEY_HEADER, "api_key": api_key}, @@ -418,8 +378,8 @@ async def test_api_key_missing_header( webhook_body: WebhookInput, ) -> None: async with api_key_webhook(hatchet, test_run_id) as incoming_webhook: - async with await send_webhook_request( - url(hatchet.tenant_id, incoming_webhook.name), webhook_body, "NONE" + async with send_webhook_request( + url(hatchet, incoming_webhook.name), webhook_body, "NONE" ) as response: assert response.status == 403 @@ -438,8 +398,8 @@ async def test_hmac_success( webhook_body: WebhookInput, ) -> None: async with hmac_webhook(hatchet, test_run_id) as incoming_webhook: - async with await send_webhook_request( - url(hatchet.tenant_id, incoming_webhook.name), + async with send_webhook_request( + url(hatchet, incoming_webhook.name), webhook_body, "HMAC", { @@ -483,8 +443,8 @@ async def test_hmac_different_algorithms_and_encodings( async with hmac_webhook( hatchet, test_run_id, algorithm=algorithm, encoding=encoding ) as incoming_webhook: - async with await send_webhook_request( - url(hatchet.tenant_id, incoming_webhook.name), + async with send_webhook_request( + url(hatchet, incoming_webhook.name), webhook_body, "HMAC", { @@ -524,8 +484,8 @@ async def test_hmac_signature_failure( secret: str, ) -> None: async with hmac_webhook(hatchet, test_run_id) as incoming_webhook: - async with await send_webhook_request( - url(hatchet.tenant_id, incoming_webhook.name), + async with send_webhook_request( + url(hatchet, incoming_webhook.name), webhook_body, "HMAC", { @@ -552,8 +512,8 @@ async def test_hmac_missing_signature_header( webhook_body: WebhookInput, ) -> None: async with hmac_webhook(hatchet, test_run_id) as incoming_webhook: - async with await send_webhook_request( - url(hatchet.tenant_id, incoming_webhook.name), webhook_body, "NONE" + async with send_webhook_request( + url(hatchet, incoming_webhook.name), webhook_body, "NONE" ) as response: assert response.status == 403 @@ -582,8 +542,8 @@ async def test_different_source_types( async with basic_auth_webhook( hatchet, test_run_id, source_name=source_name ) as incoming_webhook: - async with await send_webhook_request( - url(hatchet.tenant_id, incoming_webhook.name), + async with send_webhook_request( + url(hatchet, incoming_webhook.name), webhook_body, "BASIC", {"username": TEST_BASIC_USERNAME, "password": TEST_BASIC_PASSWORD}, diff --git a/sdks/python/tests/test_rest_api.py b/sdks/python/tests/test_rest_api.py index 21847f3de2..f78ec39ac3 100644 --- a/sdks/python/tests/test_rest_api.py +++ b/sdks/python/tests/test_rest_api.py @@ -1,6 +1,8 @@ import asyncio import pytest +import tenacity +from tenacity import stop_after_attempt, wait_exponential from examples.dag.worker import dag_workflow from hatchet_sdk import Hatchet @@ -10,15 +12,23 @@ async def test_list_runs(hatchet: Hatchet) -> None: dag_result = await dag_workflow.aio_run() - runs = await hatchet.runs.aio_list( - only_tasks=True, - workflow_ids=[dag_workflow.id], - include_payloads=True, - limit=100, + @tenacity.retry( + stop=stop_after_attempt(10), wait=wait_exponential(multiplier=1, min=4, max=10) ) - - for v in dag_result.values(): - assert v in [r.output for r in runs.rows] + async def validate_runs() -> bool: + runs = await hatchet.runs.aio_list( + only_tasks=True, + workflow_ids=[dag_workflow.id], + include_payloads=True, + limit=100, + ) + + for v in dag_result.values(): + if v not in [r.output for r in runs.rows]: + raise Exception() + return True + + assert await validate_runs() == True @pytest.mark.asyncio(loop_scope="session") diff --git a/sdks/ruby/examples/bulk_operations/test_bulk_replay_spec.rb b/sdks/ruby/examples/bulk_operations/test_bulk_replay_spec.rb index f095909a2a..fda76e0895 100644 --- a/sdks/ruby/examples/bulk_operations/test_bulk_replay_spec.rb +++ b/sdks/ruby/examples/bulk_operations/test_bulk_replay_spec.rb @@ -59,7 +59,7 @@ total_expected = (n + 1) + (n / 2 - 1) + (n / 2 - 2) # Poll until all runs are completed instead of a fixed sleep - 30.times do + 60.times do runs = HATCHET.runs.list( workflow_ids: workflow_ids, additional_metadata: { "test_run_id" => test_run_id }, diff --git a/sdks/ruby/examples/cancellation/test_cancellation_spec.rb b/sdks/ruby/examples/cancellation/test_cancellation_spec.rb index de3bb161b3..13051c217d 100644 --- a/sdks/ruby/examples/cancellation/test_cancellation_spec.rb +++ b/sdks/ruby/examples/cancellation/test_cancellation_spec.rb @@ -7,9 +7,6 @@ it "cancels a workflow run" do ref = CANCELLATION_WORKFLOW.run_no_wait - # Wait for the cancellation to happen - sleep 10 - # Poll until the run reaches a terminal state run = HATCHET.runs.poll(ref.workflow_run_id, interval: 1.0, timeout: 60) diff --git a/sdks/ruby/examples/conditions/test_conditions_spec.rb b/sdks/ruby/examples/conditions/test_conditions_spec.rb index 33e0e272f8..16cee56880 100644 --- a/sdks/ruby/examples/conditions/test_conditions_spec.rb +++ b/sdks/ruby/examples/conditions/test_conditions_spec.rb @@ -7,8 +7,8 @@ it "runs the condition workflow with event triggers" do ref = TASK_CONDITION_WORKFLOW.run_no_wait - # Wait for the sleep conditions, then push events - sleep 2 + wait_for_running_status(HATCHET, ref.workflow_run_id) + sleep 5 HATCHET.events.create(key: "wait_for_event:start", data: {}) diff --git a/sdks/ruby/examples/durable/test_durable_spec.rb b/sdks/ruby/examples/durable/test_durable_spec.rb index f2d1ab5f15..70c2da8695 100644 --- a/sdks/ruby/examples/durable/test_durable_spec.rb +++ b/sdks/ruby/examples/durable/test_durable_spec.rb @@ -7,8 +7,8 @@ it "completes a durable sleep then waits for event" do ref = DURABLE_WORKFLOW.run_no_wait - # Wait for the sleep to complete - sleep(DURABLE_SLEEP_TIME + 2) + wait_for_running_status(HATCHET, ref.workflow_run_id) + sleep(DURABLE_SLEEP_TIME + 3) # Push the event to unblock the durable task HATCHET.events.create(key: DURABLE_EVENT_KEY, data: { "test" => true }) diff --git a/sdks/ruby/examples/non_retryable/test_no_retry_spec.rb b/sdks/ruby/examples/non_retryable/test_no_retry_spec.rb index f47917550b..27bc4a1ca7 100644 --- a/sdks/ruby/examples/non_retryable/test_no_retry_spec.rb +++ b/sdks/ruby/examples/non_retryable/test_no_retry_spec.rb @@ -9,12 +9,17 @@ expect { ref.result }.to raise_error(Hatchet::FailedRunError) - # Poll until all task events have been recorded (replaces fixed sleep 3) + # Poll until all task events have been recorded. + # Rescue 404 errors: the run record may not be visible immediately after result raises. run_details = nil - 30.times do - run_details = HATCHET.runs.get_details(ref.workflow_run_id) - failed_events = run_details.task_events.select { |e| e.event_type == "FAILED" } - break if failed_events.length >= 3 + 60.times do + begin + run_details = HATCHET.runs.get_details(ref.workflow_run_id) + failed_events = run_details.task_events.select { |e| e.event_type == "FAILED" } + break if failed_events.length >= 3 + rescue HatchetSdkRest::ApiError => e + raise unless e.code == 404 + end sleep 0.5 end diff --git a/sdks/ruby/examples/on_failure/test_on_failure_spec.rb b/sdks/ruby/examples/on_failure/test_on_failure_spec.rb index daaf925033..4a7e12cd46 100644 --- a/sdks/ruby/examples/on_failure/test_on_failure_spec.rb +++ b/sdks/ruby/examples/on_failure/test_on_failure_spec.rb @@ -11,9 +11,13 @@ # Poll until both tasks are in a terminal state (replaces fixed sleep 5) details = nil - 30.times do - details = HATCHET.runs.get_details(ref.workflow_run_id) - break if details.tasks.length >= 2 && details.tasks.all? { |t| %w[COMPLETED FAILED].include?(t.status) } + 120.times do + begin + details = HATCHET.runs.get_details(ref.workflow_run_id) + break if details.tasks.length >= 2 && details.tasks.all? { |t| %w[COMPLETED FAILED].include?(t.status) } + rescue HatchetSdkRest::ApiError => e + raise unless e.code == 404 + end sleep 0.5 end diff --git a/sdks/ruby/examples/spec_helper.rb b/sdks/ruby/examples/spec_helper.rb index d67386a332..8044099cc9 100644 --- a/sdks/ruby/examples/spec_helper.rb +++ b/sdks/ruby/examples/spec_helper.rb @@ -27,3 +27,18 @@ def hatchet RSpec.configuration.hatchet_client end + +# Poll until the workflow run reaches RUNNING status or timeout is exceeded. +# Retries on 404s since the run record may not be immediately visible. +def wait_for_running_status(client, run_id, timeout: 60, interval: 0.5) + max_iters = (timeout / interval).to_i + max_iters.times do + begin + details = client.runs.get_details(run_id) + return if details.run&.status == "RUNNING" + rescue HatchetSdkRest::ApiError => e + raise unless e.code == 404 + end + sleep interval + end +end diff --git a/sdks/typescript/src/legacy/examples/simple-worker.e2e.ts b/sdks/typescript/src/legacy/examples/simple-worker.e2e.ts index e26e571b74..ba227cd56f 100644 --- a/sdks/typescript/src/legacy/examples/simple-worker.e2e.ts +++ b/sdks/typescript/src/legacy/examples/simple-worker.e2e.ts @@ -1,6 +1,8 @@ import { Workflow, Worker } from '../..'; import sleep from '../../util/sleep'; import Hatchet from '../../sdk'; +import {poll} from "@hatchet/v1/examples/__e2e__/harness"; +import {V1TaskEventType, V1TaskStatus} from "@clients/rest/generated/data-contracts"; describe('e2e', () => { let hatchet: Hatchet; @@ -64,10 +66,18 @@ describe('e2e', () => { test: 'test', }); - await sleep(10000); - console.log('invoked', invoked); - + await poll( + async () => { + return invoked; + }, + { + timeoutMs: 60_000, + intervalMs: 400, + label: 'nonRetryableWorkflow terminal with events', + shouldStop: (d) => d >= 2, + } + ); expect(invoked).toEqual(2); await worker.stop(); diff --git a/sdks/typescript/src/v1/examples/bulk_operations/bulk_replay.e2e.ts b/sdks/typescript/src/v1/examples/bulk_operations/bulk_replay.e2e.ts index 68fd1ba830..fcd1f4bc24 100644 --- a/sdks/typescript/src/v1/examples/bulk_operations/bulk_replay.e2e.ts +++ b/sdks/typescript/src/v1/examples/bulk_operations/bulk_replay.e2e.ts @@ -1,6 +1,6 @@ import { applyNamespace } from '@hatchet/util/apply-namespace'; import { makeE2EClient, poll, makeTestScope } from '../__e2e__/harness'; -import { V1TaskStatus } from '../../../clients/rest/generated/data-contracts'; +import { V1TaskStatus, V1TaskSummaryList } from '../../../clients/rest/generated/data-contracts'; import { bulkReplayTest1, bulkReplayTest2, bulkReplayTest3 } from './workflow'; describe('bulk-replay-e2e', () => { @@ -8,7 +8,7 @@ describe('bulk-replay-e2e', () => { it('bulk replays matching runs and increments retry count', async () => { const testRunId = makeTestScope('bulk_replay'); - const n = 20; + const n = 4; const meta = { test_run_id: testRunId }; const since = new Date(Date.now() - 5 * 60 * 1000); @@ -19,20 +19,25 @@ describe('bulk-replay-e2e', () => { await bulkReplayTest2.runNoWait(inputs(n / 2 - 1), { additionalMetadata: meta }); await bulkReplayTest3.runNoWait(inputs(n / 2 - 2), { additionalMetadata: meta }); - const workflowNames = [bulkReplayTest1.name, bulkReplayTest2.name, bulkReplayTest3.name]; + const workflowNames = [ + applyNamespace(bulkReplayTest1.name, hatchet.config.namespace), + applyNamespace(bulkReplayTest2.name, hatchet.config.namespace), + applyNamespace(bulkReplayTest3.name, hatchet.config.namespace), + ]; const expectedTotal = n + 1 + (n / 2 - 1) + (n / 2 - 2); const initialRuns = await poll( - async () => - hatchet.runs.list({ + async () => { + return await hatchet.runs.list({ since, limit: 1000, workflowNames, additionalMetadata: meta, onlyTasks: true, - }), + }); + }, { - timeoutMs: 120_000, + timeoutMs: 600_000, intervalMs: 200, label: 'initial bulk runs completion', shouldStop: (runs) => @@ -54,7 +59,17 @@ describe('bulk-replay-e2e', () => { additionalMetadata: meta, }, }); - + function shouldStop(runs: V1TaskSummaryList) { + console.info(runs); + return ( + (runs.rows || []).length === expectedTotal && + (runs.rows || []).every( + (r: any) => + r.status === V1TaskStatus.COMPLETED && + (r.retryCount ?? 0) > (initialRetryCounts.get(r.metadata.id) ?? Number.MAX_SAFE_INTEGER) + ) + ); + } const replayedRuns = await poll( async () => hatchet.runs.list({ @@ -68,14 +83,7 @@ describe('bulk-replay-e2e', () => { timeoutMs: 120_000, intervalMs: 200, label: 'bulk replay retry counts visible', - shouldStop: (runs) => - (runs.rows || []).length === expectedTotal && - (runs.rows || []).every( - (r: any) => - r.status === V1TaskStatus.COMPLETED && - (r.retryCount ?? 0) > - (initialRetryCounts.get(r.metadata.id) ?? Number.MAX_SAFE_INTEGER) - ), + shouldStop: shouldStop, } ); @@ -90,5 +98,5 @@ describe('bulk-replay-e2e', () => { expect(byName(bulkReplayTest1.name)).toHaveLength(n + 1); expect(byName(bulkReplayTest2.name)).toHaveLength(n / 2 - 1); expect(byName(bulkReplayTest3.name)).toHaveLength(n / 2 - 2); - }, 240_000); + }, 1_320_000); }); diff --git a/sdks/typescript/src/v1/examples/durable/durable.e2e.ts b/sdks/typescript/src/v1/examples/durable/durable.e2e.ts index f74ac40b7f..f2ba5fb048 100644 --- a/sdks/typescript/src/v1/examples/durable/durable.e2e.ts +++ b/sdks/typescript/src/v1/examples/durable/durable.e2e.ts @@ -1,5 +1,11 @@ import sleep from '@hatchet/util/sleep'; -import { makeE2EClient, checkDurableEvictionSupport, makeTestScope } from '../__e2e__/harness'; +import { V1TaskStatus } from '@hatchet/clients/rest/generated/data-contracts'; +import { + makeE2EClient, + checkDurableEvictionSupport, + makeTestScope, + poll, +} from '../__e2e__/harness'; import { durableWorkflow, EVENT_KEY, @@ -18,6 +24,8 @@ import { waitForTwoEventsSecondPushedFirst, } from './workflow'; +const TIMING_TOLERANCE_SECONDS = 1; + describe('durable-e2e', () => { const hatchet = makeE2EClient(); let evictionSupported = false; @@ -26,6 +34,27 @@ describe('durable-e2e', () => { evictionSupported = await checkDurableEvictionSupport(hatchet); }); + async function pollUntilRunning(runId: string) { + return poll( + async () => { + try { + return await hatchet.runs.get(runId); + } catch (e: any) { + if (e?.response?.status === 404) return undefined; + throw e; + } + }, + { + timeoutMs: 60_000, + intervalMs: 500, + shouldStop: (details: any) => + details != null && + (details?.tasks || []).some((t: any) => t.status === V1TaskStatus.RUNNING), + label: 'status=RUNNING', + } + ); + } + function requireEviction() { if (!evictionSupported) { console.log('Skipping: engine does not support durable eviction'); @@ -78,7 +107,8 @@ describe('durable-e2e', () => { if (requireEviction()) return; const ref = await waitForSleepTwice.runNoWait({}); - await sleep((SLEEP_TIME_SECONDS * 1000) / 2); + const runId = await ref.getWorkflowRunId(); + await pollUntilRunning(runId); await ref.cancel(); await ref.output.catch(() => undefined); @@ -133,7 +163,7 @@ describe('durable-e2e', () => { const replayElapsed = (Date.now() - replayStart) / 1000; expect(replayed.child_output).toEqual({ message: 'hello from child 1' }); - expect(replayElapsed).toBeLessThan(SLEEP_TIME_SECONDS); + expect(replayElapsed).toBeLessThan(SLEEP_TIME_SECONDS + TIMING_TOLERANCE_SECONDS); }, 300_000); it('durable completed replay', async () => { @@ -152,8 +182,8 @@ describe('durable-e2e', () => { const replayed = await ref.output; const replayElapsed = (Date.now() - replayStart) / 1000; - expect(replayed.runtime).toBeLessThan(SLEEP_TIME_SECONDS); - expect(replayElapsed).toBeLessThan(SLEEP_TIME_SECONDS); + expect(replayed.runtime).toBeLessThan(SLEEP_TIME_SECONDS + TIMING_TOLERANCE_SECONDS); + expect(replayElapsed).toBeLessThan(SLEEP_TIME_SECONDS + TIMING_TOLERANCE_SECONDS); }, 300_000); it('durable spawn DAG', async () => { @@ -234,7 +264,7 @@ describe('durable-e2e', () => { const result = await waitForEventLookback.run({ userId }); - expect(result.elapsed).toBeLessThan(5); + expect(result.elapsed).toBeLessThan(5 + TIMING_TOLERANCE_SECONDS); expect(result.event).toMatchObject({ order: 'first' }); }, 30_000); @@ -246,7 +276,7 @@ describe('durable-e2e', () => { const result = await waitForOrEventLookback.run({ scope }); - expect(result.elapsed).toBeLessThan(SLEEP_TIME_SECONDS); + expect(result.elapsed).toBeLessThan(SLEEP_TIME_SECONDS + TIMING_TOLERANCE_SECONDS); }, 60_000); it('two event waits: second event pushed before workflow starts', async () => { @@ -262,7 +292,7 @@ describe('durable-e2e', () => { const result = await ref.output; - expect(result.elapsed).toBeLessThan(SLEEP_TIME_SECONDS); + expect(result.elapsed).toBeLessThan(SLEEP_TIME_SECONDS + TIMING_TOLERANCE_SECONDS); expect(result.event1).toMatchObject({ order: 'first' }); expect(result.event2).toMatchObject({ order: 'second' }); }, 60_000); diff --git a/sdks/typescript/src/v1/examples/durable_event/durable_event.e2e.ts b/sdks/typescript/src/v1/examples/durable_event/durable_event.e2e.ts index 62907afa29..85c1611595 100644 --- a/sdks/typescript/src/v1/examples/durable_event/durable_event.e2e.ts +++ b/sdks/typescript/src/v1/examples/durable_event/durable_event.e2e.ts @@ -15,7 +15,7 @@ describe('durable-event-e2e', () => { const eventPusher = (async () => { await sleep(2000); - for (let i = 0; i < 30 && !finished; i += 1) { + for (let i = 0; i < 300 && !finished; i += 1) { await hatchet.events.push(EVENT_KEY, { userId: '1234' }); await sleep(200); } @@ -37,7 +37,7 @@ describe('durable-event-e2e', () => { const eventPusher = (async () => { await sleep(2000); - for (let i = 0; i < 30 && !finished; i += 1) { + for (let i = 0; i < 300 && !finished; i += 1) { await hatchet.events.push(EVENT_KEY, { userId: '1234' }); await sleep(200); } diff --git a/sdks/typescript/src/v1/examples/events/event.e2e.ts b/sdks/typescript/src/v1/examples/events/event.e2e.ts index 5ffbf9ade3..e7c305bbd4 100644 --- a/sdks/typescript/src/v1/examples/events/event.e2e.ts +++ b/sdks/typescript/src/v1/examples/events/event.e2e.ts @@ -16,7 +16,9 @@ describe('events-e2e', () => { const finalExpression = expression || `input.should_skip == false && payload.test_run_id == '${testRunId}'`; - const workflowId = (await hatchet.workflows.get(lower.name)).metadata.id; + const workflowId = ( + await hatchet.workflows.get(applyNamespace(lower.name, hatchet.config.namespace)) + ).metadata.id; const filter = await hatchet.filters.create({ workflowId, @@ -25,6 +27,15 @@ describe('events-e2e', () => { payload: { test_run_id: testRunId, ...payload }, }); + // Verify the filter is reachable before returning so callers can immediately push events. + // Without this, events pushed right after filter creation may miss the filter in the engine + // if it hasn't finished propagating. + for (let i = 0; i < 50; i += 1) { + const filters = (await hatchet.filters.list({ scopes: [testRunId] })).rows || []; + if (filters.some((f) => f.metadata.id === filter.metadata.id)) break; + await sleep(100); + } + return async () => { await hatchet.filters.delete(filter.metadata.id); }; @@ -32,93 +43,54 @@ describe('events-e2e', () => { // Helper function to wait for events to process and fetch runs async function waitForEventsToProcess(events: Event[]): Promise> { - const eventIds = new Set(events.map((e) => e.eventId)); - - // Poll until all events are persisted (replaces fixed sleep - events can have propagation delay) - let persisted = (await hatchet.events.list({ limit: 100 })).rows || []; - for (let i = 0; i < 50; i += 1) { - const persistedIdsSoFar = new Set(persisted.map((e) => e.metadata.id)); - if (Array.from(eventIds).every((id) => persistedIdsSoFar.has(id))) { - break; + const eventIds = events.map((e) => e.eventId); + + // Use eventIds for direct lookup — limit: 100 can miss events in a busy staging environment + for (let i = 0; i < 150; i += 1) { + const persisted = (await hatchet.events.list({ eventIds })).rows || []; + const persistedIds = new Set(persisted.map((e) => e.metadata.id)); + if (eventIds.every((id) => persistedIds.has(id))) break; + if (i === 149) { + expect(eventIds.every((id) => persistedIds.has(id))).toBeTruthy(); } await sleep(100); - persisted = (await hatchet.events.list({ limit: 100 })).rows || []; } - const persistedIds = new Set(persisted.map((e) => e.metadata.id)); - expect(Array.from(eventIds).every((id) => persistedIds.has(id))).toBeTruthy(); - - let attempts = 0; - const maxAttempts = 60; // 100ms × 60 ≈ 6s for runs to appear and complete const eventToRuns: Record = {}; - while (true) { - console.log('Waiting for event runs to complete...'); - if (attempts > maxAttempts) { - console.log('Timed out waiting for event runs to complete.'); - return {}; - } - - attempts += 1; - - // For each event, fetch its runs - const runsPromises = events.map(async (event) => { - const runsResp = await hatchet.runs.list({ - triggeringEventExternalId: event.eventId, - }); - const rawRuns = runsResp.rows || []; - - // Only consider runs that are in a terminal state (match Python fetch_runs_for_event) - const runs = - rawRuns.length > 0 && - rawRuns.every( - (r) => r.status === 'COMPLETED' || r.status === 'FAILED' || r.status === 'CANCELLED' - ) - ? rawRuns - : []; - - // Extract metadata from event - const meta = event.additionalMetadata ? JSON.parse(event.additionalMetadata) : {}; - - const payload = event.payload ? JSON.parse(event.payload) : {}; - - return { - event: { - id: event.eventId, - payload, - meta, - shouldHaveRuns: Boolean(meta.should_have_runs), - testRunId: meta.test_run_id, - }, - runs, - }; - }); - - const eventRuns = await Promise.all(runsPromises); + const pollDeadlineMs = Date.now() + 50_000; + + while (Date.now() < pollDeadlineMs) { + const runsResults = await Promise.all( + events.map(async (event) => { + // Query by hatchet__event_id metadata rather than triggeringEventExternalId. + // Filter-triggered runs (scope-routed events) are not linked via + // triggeringEventExternalId but do get hatchet__event_id set in their metadata, + // matching what verifyEventRuns already uses to validate results. + const runsResp = await hatchet.runs.list({ + additionalMetadata: { hatchet__event_id: event.eventId }, + }); + return { eventId: event.eventId, rawRuns: runsResp.rows || [] }; + }) + ); - // If all events have no runs yet, wait and retry - if (eventRuns.every(({ runs }) => runs.length === 0)) { + if (!runsResults) { await sleep(100); - continue; } - // Store runs by event ID - for (const { event, runs } of eventRuns) { - eventToRuns[event.id] = runs; - } - - // Check if any runs are still in progress - const anyInProgress = Object.values(eventToRuns).some((runs) => - runs.some((run) => run.status === 'QUEUED' || run.status === 'RUNNING') - ); + const anyHaveRuns = runsResults.some(({ rawRuns }) => rawRuns.length > 0); - if (anyInProgress) { + if (!anyHaveRuns) { await sleep(100); - continue; } + // All runs are terminal (or absent). Store results for all events. + for (const { eventId, rawRuns } of runsResults) { + eventToRuns[eventId] = rawRuns; + } + console.info(eventToRuns); break; } @@ -245,7 +217,7 @@ describe('events-e2e', () => { ); } }); - }, 30000); + }, 180_000); function generateBulkEvents() { return [ @@ -325,7 +297,7 @@ describe('events-e2e', () => { } finally { await cleanup(); } - }, 30000); + }, 180_000); it('should filter events by payload expression not matching', async () => { const cleanup = await setupEventFilter( @@ -354,7 +326,7 @@ describe('events-e2e', () => { } finally { await cleanup(); } - }, 20000); + }, 120_000); it('should filter events by payload expression matching', async () => { const cleanup = await setupEventFilter( @@ -384,5 +356,5 @@ describe('events-e2e', () => { } finally { await cleanup(); } - }, 20000); + }, 90_000); }); diff --git a/sdks/typescript/src/v1/examples/non_retryable/non_retryable.e2e.ts b/sdks/typescript/src/v1/examples/non_retryable/non_retryable.e2e.ts index f2ccd28477..4806bd2a96 100644 --- a/sdks/typescript/src/v1/examples/non_retryable/non_retryable.e2e.ts +++ b/sdks/typescript/src/v1/examples/non_retryable/non_retryable.e2e.ts @@ -21,10 +21,16 @@ describe('non-retryable-e2e', () => { }, { timeoutMs: 60_000, - intervalMs: 100, - label: 'nonRetryableWorkflow terminal', + intervalMs: 400, + label: 'nonRetryableWorkflow terminal with events', shouldStop: (d) => - ![V1TaskStatus.QUEUED, V1TaskStatus.RUNNING].includes(d.run.status as any), + ![V1TaskStatus.QUEUED, V1TaskStatus.RUNNING].includes(d.run.status as any) && + d.taskEvents.some( + (e: { eventType: V1TaskEventType }) => e.eventType === V1TaskEventType.RETRYING + ) && + d.taskEvents.filter( + (e: { eventType: V1TaskEventType }) => e.eventType === V1TaskEventType.FAILED + ).length >= 3, } ); diff --git a/sdks/typescript/src/v1/examples/on_event/event.e2e.ts b/sdks/typescript/src/v1/examples/on_event/event.e2e.ts index 5ffbf9ade3..e3aa2d042b 100644 --- a/sdks/typescript/src/v1/examples/on_event/event.e2e.ts +++ b/sdks/typescript/src/v1/examples/on_event/event.e2e.ts @@ -16,7 +16,9 @@ describe('events-e2e', () => { const finalExpression = expression || `input.should_skip == false && payload.test_run_id == '${testRunId}'`; - const workflowId = (await hatchet.workflows.get(lower.name)).metadata.id; + const workflowId = ( + await hatchet.workflows.get(applyNamespace(lower.name, hatchet.config.namespace)) + ).metadata.id; const filter = await hatchet.filters.create({ workflowId, @@ -25,6 +27,15 @@ describe('events-e2e', () => { payload: { test_run_id: testRunId, ...payload }, }); + // Verify the filter is reachable before returning so callers can immediately push events. + // Without this, events pushed right after filter creation may miss the filter in the engine + // if it hasn't finished propagating. + for (let i = 0; i < 50; i += 1) { + const filters = (await hatchet.filters.list({ scopes: [testRunId] })).rows || []; + if (filters.some((f) => f.metadata.id === filter.metadata.id)) break; + await sleep(100); + } + return async () => { await hatchet.filters.delete(filter.metadata.id); }; @@ -32,93 +43,64 @@ describe('events-e2e', () => { // Helper function to wait for events to process and fetch runs async function waitForEventsToProcess(events: Event[]): Promise> { - const eventIds = new Set(events.map((e) => e.eventId)); - - // Poll until all events are persisted (replaces fixed sleep - events can have propagation delay) - let persisted = (await hatchet.events.list({ limit: 100 })).rows || []; - for (let i = 0; i < 50; i += 1) { - const persistedIdsSoFar = new Set(persisted.map((e) => e.metadata.id)); - if (Array.from(eventIds).every((id) => persistedIdsSoFar.has(id))) { - break; + const eventIds = events.map((e) => e.eventId); + + // Use eventIds for direct lookup — limit: 100 can miss events in a busy staging environment + for (let i = 0; i < 150; i += 1) { + const persisted = (await hatchet.events.list({ eventIds })).rows || []; + const persistedIds = new Set(persisted.map((e) => e.metadata.id)); + if (eventIds.every((id) => persistedIds.has(id))) break; + if (i === 149) { + expect(eventIds.every((id) => persistedIds.has(id))).toBeTruthy(); } await sleep(100); - persisted = (await hatchet.events.list({ limit: 100 })).rows || []; } - const persistedIds = new Set(persisted.map((e) => e.metadata.id)); - expect(Array.from(eventIds).every((id) => persistedIds.has(id))).toBeTruthy(); - - let attempts = 0; - const maxAttempts = 60; // 100ms × 60 ≈ 6s for runs to appear and complete const eventToRuns: Record = {}; + // Use a wall-clock deadline so the function exits predictably regardless of API latency. + // Iteration counts (e.g. maxAttempts * intervalMs) undercount real time because each + // hatchet.runs.list() call itself takes ~100-200ms on top of the sleep interval. + const pollDeadlineMs = Date.now() + 50_000; + + while (Date.now() < pollDeadlineMs) { + const runsResults = await Promise.all( + events.map(async (event) => { + // Query by hatchet__event_id metadata rather than triggeringEventExternalId. + // Filter-triggered runs (scope-routed events) are not linked via + // triggeringEventExternalId but do get hatchet__event_id set in their metadata, + // matching what verifyEventRuns already uses to validate results. + const runsResp = await hatchet.runs.list({ + additionalMetadata: { hatchet__event_id: event.eventId }, + }); + return { eventId: event.eventId, rawRuns: runsResp.rows || [] }; + }) + ); - while (true) { - console.log('Waiting for event runs to complete...'); - if (attempts > maxAttempts) { - console.log('Timed out waiting for event runs to complete.'); - return {}; - } - - attempts += 1; - - // For each event, fetch its runs - const runsPromises = events.map(async (event) => { - const runsResp = await hatchet.runs.list({ - triggeringEventExternalId: event.eventId, - }); - const rawRuns = runsResp.rows || []; - - // Only consider runs that are in a terminal state (match Python fetch_runs_for_event) - const runs = - rawRuns.length > 0 && - rawRuns.every( - (r) => r.status === 'COMPLETED' || r.status === 'FAILED' || r.status === 'CANCELLED' - ) - ? rawRuns - : []; - - // Extract metadata from event - const meta = event.additionalMetadata ? JSON.parse(event.additionalMetadata) : {}; - - const payload = event.payload ? JSON.parse(event.payload) : {}; - - return { - event: { - id: event.eventId, - payload, - meta, - shouldHaveRuns: Boolean(meta.should_have_runs), - testRunId: meta.test_run_id, - }, - runs, - }; - }); - - const eventRuns = await Promise.all(runsPromises); + // Bug fix: check rawRuns directly for in-progress status. + // Original code stored runs=[] for in-progress events (because not all were terminal), + // then anyInProgress checked the stored empty array and incorrectly returned false, + // causing the loop to break prematurely with incomplete results. + const hasInProgress = runsResults.some(({ rawRuns }) => + rawRuns.some((r) => r.status === 'QUEUED' || r.status === 'RUNNING') + ); - // If all events have no runs yet, wait and retry - if (eventRuns.every(({ runs }) => runs.length === 0)) { + if (hasInProgress) { await sleep(100); - continue; } - // Store runs by event ID - for (const { event, runs } of eventRuns) { - eventToRuns[event.id] = runs; - } + const anyHaveRuns = runsResults.some(({ rawRuns }) => rawRuns.length > 0); - // Check if any runs are still in progress - const anyInProgress = Object.values(eventToRuns).some((runs) => - runs.some((run) => run.status === 'QUEUED' || run.status === 'RUNNING') - ); - - if (anyInProgress) { + if (!anyHaveRuns) { await sleep(100); - continue; } + // All runs are terminal (or absent). Store results for all events. + for (const { eventId, rawRuns } of runsResults) { + eventToRuns[eventId] = rawRuns; + } + break; } @@ -245,7 +227,7 @@ describe('events-e2e', () => { ); } }); - }, 30000); + }, 180_000); function generateBulkEvents() { return [ @@ -325,7 +307,7 @@ describe('events-e2e', () => { } finally { await cleanup(); } - }, 30000); + }, 180_000); it('should filter events by payload expression not matching', async () => { const cleanup = await setupEventFilter( @@ -354,7 +336,7 @@ describe('events-e2e', () => { } finally { await cleanup(); } - }, 20000); + }, 90_000); it('should filter events by payload expression matching', async () => { const cleanup = await setupEventFilter( @@ -384,5 +366,5 @@ describe('events-e2e', () => { } finally { await cleanup(); } - }, 20000); + }, 90_000); }); diff --git a/sdks/typescript/src/v1/examples/runtime_affinity/runtime-affinity.e2e.ts b/sdks/typescript/src/v1/examples/runtime_affinity/runtime-affinity.e2e.ts index 95b397fd2f..3bf3563654 100644 --- a/sdks/typescript/src/v1/examples/runtime_affinity/runtime-affinity.e2e.ts +++ b/sdks/typescript/src/v1/examples/runtime_affinity/runtime-affinity.e2e.ts @@ -1,9 +1,9 @@ -import sleep from '@hatchet/util/sleep'; import { WorkerList } from '@hatchet/clients/rest/generated/data-contracts'; -import { checkDurableEvictionSupport, stopWorker } from '../__e2e__/harness'; +import { checkDurableEvictionSupport, poll, stopWorker } from '../__e2e__/harness'; import { Worker } from '../../client/worker/worker'; import { hatchet } from '../hatchet-client'; import { affinityExampleTask } from './workflow'; +import {applyNamespace} from "@util/apply-namespace"; const labels = ['foo', 'bar'] as const; @@ -39,14 +39,19 @@ describe('runtime-affinity-e2e', () => { }); workerB.start().catch((err) => console.error('[affinity-test] workerB start error:', err)); await workerB.waitUntilReady(10_000); - - await sleep(5_000); - - const allWorkers: WorkerList = await hatchet.workers.list(); - const activeWorkers = (allWorkers.rows || []).filter( - (w) => w.status === 'ACTIVE' && `${w.name}`.includes('runtime-affinity-worker') + let runtimeAffinityWorkerName = applyNamespace('runtime-affinity-worker', hatchet.config.namespace) + const workerResult = await poll(() => hatchet.workers.list(), { + timeoutMs: 120_000, + intervalMs: 500, + label: 'active runtime-affinity workers', + shouldStop: (result: WorkerList) => + (result.rows || []).filter( + (w) => w.status === 'ACTIVE' && `${w.name}`.includes(runtimeAffinityWorkerName) + ).length === 2, + }); + const activeWorkers = (workerResult.rows || []).filter( + (w) => w.status === 'ACTIVE' && `${w.name}`.includes(runtimeAffinityWorkerName) ); - expect(activeWorkers.length).toBe(2); const workerLabelToId: Record = {}; @@ -77,5 +82,5 @@ describe('runtime-affinity-e2e', () => { expect(res.affinity_t1.worker_id).toBe(workerLabelToId[targetWorker]); expect(res.affinity_t2.worker_id).toBe(workerLabelToId[targetWorker]); } - }, 120_000); + }, 180_000); });