diff --git a/.changeset/e2e-stream-flush.md b/.changeset/e2e-stream-flush.md new file mode 100644 index 0000000000..dcf85c16c7 --- /dev/null +++ b/.changeset/e2e-stream-flush.md @@ -0,0 +1,5 @@ +--- +'@workflow/core': patch +--- + +Harden inline step completion against stream flush and transient persistence races. diff --git a/packages/core/e2e/bench.bench.ts b/packages/core/e2e/bench.bench.ts index 275dc544ce..2f56f0667d 100644 --- a/packages/core/e2e/bench.bench.ts +++ b/packages/core/e2e/bench.bench.ts @@ -1,10 +1,12 @@ +import fs from 'node:fs'; +import path from 'node:path'; import { createVercelWorld } from '@workflow/world-vercel'; -import fs from 'fs'; -import path from 'path'; import { bench, describe } from 'vitest'; import { getTrustedSourcesHeaders } from '../../../scripts/trusted-sources-headers.mjs'; import type { Run } from '../src/runtime'; import { setWorld, start } from '../src/runtime'; +import { getWorldLazy } from '../src/runtime/get-world-lazy'; +import { STREAM_CLIENT_NAME_SYMBOL } from '../src/symbols'; import { getWorkbenchAppPath, isLocalDeployment } from './utils'; const deploymentUrl = process.env.DEPLOYMENT_URL; @@ -99,6 +101,10 @@ async function getWorkflowMetadata( const benchWf = (fn: string) => getWorkflowMetadata('workflows/97_bench.ts', fn); +const STREAM_READ_TIMEOUT_MS = process.env.WORKFLOW_VERCEL_ENV + ? 90_000 + : 30_000; +const STREAM_READ_RETRY_INTERVAL_MS = 1_000; // Store workflow execution times for each benchmark const workflowTimings: Record< @@ -328,6 +334,73 @@ async function consumeStreamWithMetrics( return { firstByteTimeMs, slurpTimeMs, totalBytes, chunks }; } +/** + * Rehydrate a returned stream until enough persisted bytes are visible. + * Vercel stream chunks can lag the completed run output by a short window; + * opening the returned stream in that window can produce an empty reader. + */ +async function consumeReturnedStreamWithMetrics( + value: unknown, + run: Run, + startedAt: string | undefined, + options: { minBytes?: number; waitForDone?: boolean } = {} +): ReturnType { + const { minBytes = 1, waitForDone = minBytes > 1 } = options; + const deadline = Date.now() + STREAM_READ_TIMEOUT_MS; + let result: Awaited> | undefined; + + while (Date.now() < deadline) { + result = await consumeStreamWithMetrics(value, startedAt); + if (result.totalBytes >= minBytes) { + return result; + } + await waitForReturnedStreamRetry(value, run, deadline, waitForDone); + value = await run.returnValue; + } + + throw new Error( + `Timed out waiting for returned stream bytes: expected at least ${minBytes}, got ${result?.totalBytes ?? 0}` + ); +} + +async function waitForReturnedStreamRetry( + value: unknown, + run: Run, + deadline: number, + waitForDone: boolean +): Promise { + const name = + value instanceof ReadableStream + ? (value as Record)[STREAM_CLIENT_NAME_SYMBOL] + : undefined; + + if (typeof name !== 'string') { + await sleepUntilNextRetry(deadline); + return; + } + + const world = await getWorldLazy(); + while (Date.now() < deadline) { + const info = await world.streams.getInfo(run.runId, name).catch(() => null); + if (info && (waitForDone ? info.done : info.tailIndex >= 0)) { + return; + } + await sleepUntilNextRetry(deadline); + } +} + +function sleepUntilNextRetry(deadline: number): Promise { + return new Promise((resolve) => + setTimeout( + resolve, + Math.max( + 0, + Math.min(STREAM_READ_RETRY_INTERVAL_MS, deadline - Date.now()) + ) + ) + ); +} + describe('Workflow Performance Benchmarks', () => { bench( 'workflow with no steps', @@ -389,7 +462,7 @@ describe('Workflow Performance Benchmarks', () => { const value = await run.returnValue; const timings = await getRunTimings(run); const { firstByteTimeMs, slurpTimeMs, totalBytes } = - await consumeStreamWithMetrics(value, timings.startedAt); + await consumeReturnedStreamWithMetrics(value, run, timings.startedAt); // Correctness: stream should produce ~5KB (50 chunks * ~100 bytes) if (totalBytes === 0) { throw new Error( @@ -505,8 +578,7 @@ describe('Workflow Performance Benchmarks', () => { skip: false, time: 60000, expectedTotalBytes: 1024 * 1024, - // Pipeline returns the actual data stream - summaryStream: false, + summaryStream: true, }, { name: 'stream pipeline with 10 transform steps (1MB)', @@ -515,7 +587,7 @@ describe('Workflow Performance Benchmarks', () => { skip: !fullSuite, time: 120000, expectedTotalBytes: 1024 * 1024, - summaryStream: false, + summaryStream: true, }, { name: '10 parallel streams (1MB each)', @@ -573,7 +645,15 @@ describe('Workflow Performance Benchmarks', () => { const value = await run.returnValue; const timings = await getRunTimings(run); const { firstByteTimeMs, slurpTimeMs, totalBytes, chunks } = - await consumeStreamWithMetrics(value, timings.startedAt); + await consumeReturnedStreamWithMetrics( + value, + run, + timings.startedAt, + { + minBytes: summaryStream ? 1 : expectedTotalBytes, + waitForDone: summaryStream === true, + } + ); if (summaryStream) { // Parallel/fan-out workflows return a JSON summary stream; @@ -595,7 +675,7 @@ describe('Workflow Performance Benchmarks', () => { ); } } else { - // Pipeline workflows return the actual data stream + // Non-summary workflows return the actual data stream. if (totalBytes !== expectedTotalBytes) { throw new Error( `Stream correctness failure: expected ${expectedTotalBytes} bytes but got ${totalBytes}` diff --git a/packages/core/e2e/e2e.test.ts b/packages/core/e2e/e2e.test.ts index a0c6c0d719..8771f55f1f 100644 --- a/packages/core/e2e/e2e.test.ts +++ b/packages/core/e2e/e2e.test.ts @@ -52,6 +52,67 @@ if (!deploymentUrl) { throw new Error('`DEPLOYMENT_URL` environment variable is not set'); } +const isVercelE2E = !!process.env.WORKFLOW_VERCEL_ENV; +const TO = { + short: { timeout: isVercelE2E ? 60_000 : 30_000 }, + default: { timeout: isVercelE2E ? 120_000 : 60_000 }, + long: { timeout: isVercelE2E ? 180_000 : 90_000 }, + extraLong: { timeout: isVercelE2E ? 240_000 : 120_000 }, + veryLong: { timeout: isVercelE2E ? 300_000 : 180_000 }, + stream: { timeout: isVercelE2E ? 180_000 : 120_000 }, +} as const; +const HOOK_WAIT_TIMEOUT_MS = isVercelE2E ? 90_000 : 30_000; +const EVENT_WAIT_TIMEOUT_MS = isVercelE2E ? 60_000 : 30_000; +const DISTRIBUTED_CLOCK_TOLERANCE_MS = isVercelE2E ? 1_000 : 0; +const STREAM_READ_TIMEOUT_MS = isVercelE2E ? 90_000 : 60_000; + +function expectElapsedAtLeast(actualMs: number, expectedMs: number) { + // Vercel e2e compares timestamps from different invocations and services. + // Allow clock skew while still catching immediate or default-delay resumes. + expect(actualMs).toBeGreaterThanOrEqual( + Math.max(0, expectedMs - DISTRIBUTED_CLOCK_TOLERANCE_MS) + ); +} + +type DeadlineReadResult = ReadableStreamReadResult & { + timedOut: boolean; +}; + +async function readWithDeadline( + reader: ReadableStreamDefaultReader, + deadline: number +): Promise> { + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) { + return { done: true, value: undefined, timedOut: true }; + } + let timeout: ReturnType | undefined; + const readPromise = reader.read().then( + (result): DeadlineReadResult => ({ + ...result, + timedOut: false, + }) + ); + try { + const result = await Promise.race>([ + readPromise, + new Promise>((resolve) => { + timeout = setTimeout(() => { + resolve({ done: true, value: undefined, timedOut: true }); + }, remainingMs); + (timeout as { unref?: () => void }).unref?.(); + }), + ]); + if (result.timedOut) { + await reader.cancel().catch(() => {}); + await readPromise.catch(() => {}); + } + return result; + } finally { + if (timeout) clearTimeout(timeout); + } +} + /** * Tracked wrapper around start() that automatically registers runs * for diagnostics on test failure and observability metadata collection. @@ -95,6 +156,10 @@ function writeE2EMetadata() { const e2e = (fn: string) => getWorkflowMetadata(deploymentUrl, 'workflows/99_e2e.ts', fn); +type WorkflowEvent = Awaited< + ReturnType +>['data'][number]; + /** * Polls `getHookByToken(token)` until it resolves or the timeout is hit. * Replaces fixed `setTimeout(N)` waits in hook tests, which are flaky on @@ -111,7 +176,7 @@ async function waitForHook( runId?: string; } = {} ): Promise>> { - const { timeoutMs = 30_000, intervalMs = 250, runId } = options; + const { timeoutMs = HOOK_WAIT_TIMEOUT_MS, intervalMs = 250, runId } = options; const deadline = Date.now() + timeoutMs; let lastError: unknown = new Error( `waitForHook(${token}) timed out before any attempt` @@ -138,6 +203,59 @@ async function waitForHook( throw lastError instanceof Error ? lastError : new Error(String(lastError)); } +async function waitForRunEvents( + runId: string, + predicate: (event: WorkflowEvent) => boolean, + options: { + timeoutMs?: number; + intervalMs?: number; + minCount?: number; + description?: string; + } = {} +): Promise { + const { + timeoutMs = EVENT_WAIT_TIMEOUT_MS, + intervalMs = 250, + minCount = 1, + description = 'matching event', + } = options; + const world = await getWorld(); + const deadline = Date.now() + timeoutMs; + const matches: WorkflowEvent[] = []; + const seenEventIds = new Set(); + let cursor: string | undefined; + + while (Date.now() < deadline) { + const page = await world.events.list({ + runId, + resolveData: 'none', + pagination: { + sortOrder: 'asc', + cursor, + }, + }); + for (const event of page.data) { + if (seenEventIds.has(event.eventId)) continue; + seenEventIds.add(event.eventId); + if (predicate(event)) matches.push(event); + } + if (matches.length >= minCount) { + return matches; + } + if (page.cursor) { + cursor = page.cursor; + } + if (page.hasMore) { + continue; + } + await sleep(intervalMs); + } + + throw new Error( + `Timed out waiting for ${minCount} ${description} for run ${runId}; saw ${matches.length}` + ); +} + /** * Triggers a workflow via HTTP POST. Used only for Pages Router tests * that specifically need to validate the HTTP trigger endpoint. @@ -211,7 +329,7 @@ describe('e2e', () => { workflowFile: 'workflows/98_duplicate_case.ts', workflowFn: 'addTenWorkflow', }, - ])('addTenWorkflow', { timeout: 60_000 }, async (workflow) => { + ])('addTenWorkflow', TO.default, async (workflow) => { const run = await start( await getWorkflowMetadata( deploymentUrl, @@ -255,7 +373,7 @@ describe('e2e', () => { // 'latest' actually hits the resolve API. test.skipIf(!!process.env.WORKFLOW_VERCEL_ENV)( "deploymentId: 'latest' is a no-op in non-Vercel worlds", - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('addTenWorkflow'), [123], { deploymentId: 'latest', @@ -280,7 +398,7 @@ describe('e2e', () => { const isNextApp = process.env.APP_NAME?.includes('nextjs'); test.skipIf(!isNextApp)( 'wellKnownAgentWorkflow (.well-known/agent)', - { timeout: 60_000 }, + TO.default, async () => { const run = await start( await getWorkflowMetadata( @@ -321,35 +439,31 @@ describe('e2e', () => { } ); - test('promiseAllWorkflow', { timeout: 60_000 }, async () => { + test('promiseAllWorkflow', TO.default, async () => { const run = await start(await e2e('promiseAllWorkflow'), []); const returnValue = await run.returnValue; expect(returnValue).toBe('ABC'); }); - test('promiseRaceWorkflow', { timeout: 60_000 }, async () => { + test('promiseRaceWorkflow', TO.default, async () => { const run = await start(await e2e('promiseRaceWorkflow'), []); const returnValue = await run.returnValue; expect(returnValue).toBe('B'); }); - test('promiseAnyWorkflow', { timeout: 60_000 }, async () => { + test('promiseAnyWorkflow', TO.default, async () => { const run = await start(await e2e('promiseAnyWorkflow'), []); const returnValue = await run.returnValue; expect(returnValue).toBe('B'); }); - test.skipIf(!isNext)( - 'importedStepOnlyWorkflow', - { timeout: 60_000 }, - async () => { - const run = await start(await e2e('importedStepOnlyWorkflow'), []); - const returnValue = await run.returnValue; - expect(returnValue).toBe('imported-step-only-ok'); - } - ); + test.skipIf(!isNext)('importedStepOnlyWorkflow', TO.default, async () => { + const run = await start(await e2e('importedStepOnlyWorkflow'), []); + const returnValue = await run.returnValue; + expect(returnValue).toBe('imported-step-only-ok'); + }); - test('readableStreamWorkflow', { timeout: 120_000 }, async () => { + test('readableStreamWorkflow', TO.stream, async () => { const run = await start(await e2e('readableStreamWorkflow'), []); const returnValue = await run.returnValue; expect(returnValue).toBeInstanceOf(ReadableStream); @@ -358,17 +472,13 @@ describe('e2e', () => { const decoder = new TextDecoder(); let contents = ''; // Read chunks until we have all expected content or hit a timeout. - // On Vercel, the stream close event can be delayed even after all - // chunks are delivered, so we stop once we have the expected data - // rather than waiting for the stream to end. + // On Vercel, the final chunk or close event can be delayed, so we stop + // once we have the expected data rather than waiting for stream closure. const reader = returnValue.getReader(); - const readDeadline = Date.now() + 60_000; + const readDeadline = Date.now() + STREAM_READ_TIMEOUT_MS; try { while (Date.now() < readDeadline) { - const { done, value } = await Promise.race([ - reader.read(), - sleep(30_000).then(() => ({ done: true, value: undefined })), - ]); + const { done, value } = await readWithDeadline(reader, readDeadline); if (value) { contents += decoder.decode(value, { stream: true }); } @@ -380,7 +490,7 @@ describe('e2e', () => { expect(contents).toBe(expected); }); - test('hookWorkflow', { timeout: 60_000 }, async () => { + test('hookWorkflow', TO.default, async () => { const token = Math.random().toString(36).slice(2); const customData = Math.random().toString(36).slice(2); @@ -432,7 +542,7 @@ describe('e2e', () => { test( 'hookWorkflow is not resumable via public webhook endpoint', - { timeout: 60_000 }, + TO.default, async () => { const token = Math.random().toString(36).slice(2); const customData = Math.random().toString(36).slice(2); @@ -470,7 +580,7 @@ describe('e2e', () => { } ); - test('webhookWorkflow', { timeout: 120_000 }, async () => { + test('webhookWorkflow', TO.extraLong, async () => { const run = await start(await e2e('webhookWorkflow'), []); // Poll until all 3 webhooks are registered. @@ -595,7 +705,7 @@ describe('e2e', () => { !!process.env.WORKFLOW_TARGET_WORLD?.includes('postgres'); test.skipIf(isPostgresWorld)( 'parallelStepsThenWebhookWorkflow - no hook_conflict from same-tick replay race', - { timeout: 120_000 }, + TO.extraLong, async () => { // Regression test for https://github.com/vercel/workflow/issues/1665 // and https://github.com/vercel/workflow/issues/2283. @@ -714,7 +824,7 @@ describe('e2e', () => { } ); - test('webhook route with invalid token', { timeout: 60_000 }, async () => { + test('webhook route with invalid token', TO.default, async () => { const invalidWebhookUrl = new URL( `/.well-known/workflow/v1/webhook/${encodeURIComponent('invalid')}`, deploymentUrl @@ -729,49 +839,51 @@ describe('e2e', () => { expect(body).toBe(''); }); - test('sleepingWorkflow', { timeout: 60_000 }, async () => { + test('sleepingWorkflow', TO.default, async () => { const run = await start(await e2e('sleepingWorkflow'), []); const returnValue = await run.returnValue; expect(returnValue.startTime).toBeLessThan(returnValue.endTime); - expect(returnValue.endTime - returnValue.startTime).toBeGreaterThan(9999); + expectElapsedAtLeast(returnValue.endTime - returnValue.startTime, 10_000); }); - test('parallelSleepWorkflow', { timeout: 60_000 }, async () => { + test('parallelSleepWorkflow', TO.default, async () => { const run = await start(await e2e('parallelSleepWorkflow'), []); const returnValue = await run.returnValue; // 10 parallel sleep('1s') should complete in ~1s, not 10x (sequential). // On Vercel, cold starts and queue round-trips add latency, so we use a // generous upper bound. The key assertion is parallel < sequential (10s+). const elapsed = returnValue.endTime - returnValue.startTime; - expect(elapsed).toBeGreaterThan(999); + expect(elapsed).toBeGreaterThanOrEqual(500); // Sequential would be ~10s+ per sleep. Allow up to 20s for parallel on // Vercel with cold start overhead, but fail if it looks sequential (>25s). expect(elapsed).toBeLessThan(25_000); }); - test('sleepWinsRaceWorkflow', { timeout: 60_000 }, async () => { + test('sleepWinsRaceWorkflow', TO.default, async () => { const run = await start(await e2e('sleepWinsRaceWorkflow'), []); const returnValue = await run.returnValue; expect(returnValue.winner).toBe('sleep'); - // Sleep is 1s; step would take 10s. Should resolve in ~1s, well under 5s. - expect(returnValue.durationMs).toBeLessThan(5_000); + // Sleep is 1s; step would take 10s. Keep this below the losing branch + // without depending on exact remote queue/cold-start overhead. + expect(returnValue.durationMs).toBeLessThan(9_000); }); - test('stepWinsRaceWorkflow', { timeout: 60_000 }, async () => { + test('stepWinsRaceWorkflow', TO.default, async () => { const run = await start(await e2e('stepWinsRaceWorkflow'), []); const returnValue = await run.returnValue; expect(returnValue.winner).toBe('step'); - // Step is 1s; sleep would take 10s. Should resolve in ~1s, well under 5s. - expect(returnValue.durationMs).toBeLessThan(5_000); + // Step is 1s; sleep would take 10s. Keep this below the losing branch + // without depending on exact remote queue/cold-start overhead. + expect(returnValue.durationMs).toBeLessThan(9_000); }); - test('nullByteWorkflow', { timeout: 60_000 }, async () => { + test('nullByteWorkflow', TO.default, async () => { const run = await start(await e2e('nullByteWorkflow'), []); const returnValue = await run.returnValue; expect(returnValue).toBe('null byte \0'); }); - test('workflowAndStepMetadataWorkflow', { timeout: 60_000 }, async () => { + test('workflowAndStepMetadataWorkflow', TO.default, async () => { const run = await start(await e2e('workflowAndStepMetadataWorkflow'), []); const returnValue = await run.returnValue; @@ -884,7 +996,7 @@ describe('e2e', () => { ] as const; for (const tc of startIndexCases) { - test(tc.name, { timeout: 60_000 }, async () => { + test(tc.name, TO.default, async () => { const run = await start(await e2e('outputStreamWorkflow'), []); if (tc.waitForCompletion) { @@ -935,9 +1047,7 @@ describe('e2e', () => { describe('outputStreamWorkflow - getTailIndex and getChunks', () => { test( 'getTailIndex returns correct index after stream completes', - { - timeout: 60_000, - }, + TO.default, async () => { const run = await start(await e2e('outputStreamWorkflow'), []); await run.returnValue; @@ -952,9 +1062,7 @@ describe('e2e', () => { test( 'getTailIndex returns -1 before any chunks are written', - { - timeout: 60_000, - }, + TO.default, async () => { const run = await start(await e2e('outputStreamWorkflow'), []); @@ -969,9 +1077,7 @@ describe('e2e', () => { test( 'getChunks returns same content as reading the stream', - { - timeout: 60_000, - }, + TO.default, async () => { const run = await start(await e2e('outputStreamWorkflow'), []); await run.returnValue; @@ -1012,7 +1118,7 @@ describe('e2e', () => { test( 'outputStreamInsideStepWorkflow - getWritable() called inside step functions', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('outputStreamInsideStepWorkflow'), []); const reader = run.getReadable().getReader(); @@ -1059,7 +1165,7 @@ describe('e2e', () => { // Extended, CJK, emoji, RTL) round-trip end-to-end as bytes that decode // back to the original strings — the same property that the web inspector // relies on to render decoded text for typed-array stream chunks. - test('utf8StreamWorkflow', { timeout: 120_000 }, async () => { + test('utf8StreamWorkflow', TO.extraLong, async () => { const run = await start(await e2e('utf8StreamWorkflow'), []); const reader = run.getReadable().getReader(); @@ -1106,22 +1212,43 @@ describe('e2e', () => { test.each([ 'writableForwardedFromWorkflowWorkflow', 'writableForwardedFromStepWorkflow', - ] as const)('%s', { timeout: 120_000 }, async (workflowName) => { + ] as const)('%s', TO.extraLong, async (workflowName) => { const payload = `hello-from-child-${Date.now()}\n`; const run = await start(await e2e(workflowName), [payload]); - const reader = run.getReadable().getReader(); + const returnValue = await run.returnValue; // `fatal: true` makes the decoder throw on any invalid UTF-8 // sequence, so a successful decode is itself a round-trip // assertion that the bytes survived intact. const decoder = new TextDecoder('utf-8', { fatal: true }); + const readDeadline = Date.now() + STREAM_READ_TIMEOUT_MS; + let reader: ReadableStreamDefaultReader | undefined; + let value: unknown; // The child step performs exactly one write of `payload` as // UTF-8 bytes, so we should receive a single chunk containing - // exactly those bytes before the stream closes. - const { value, done } = await reader.read(); - expect(done).toBeFalsy(); - assert(value); + // exactly those bytes before the stream closes. On distributed + // Vercel runs, the run can complete just before the readable + // endpoint sees the forwarded stream chunk, so reopen until the + // persisted output is visible. + while (Date.now() < readDeadline) { + reader = run.getReadable().getReader(); + const result = await readWithDeadline(reader, readDeadline); + + if (!result.done && result.value) { + value = result.value; + break; + } + + reader.releaseLock(); + reader = undefined; + await sleep(1_000); + } + + assert( + value, + `Timed out waiting for forwarded writable chunk from ${workflowName}` + ); assert(value instanceof Uint8Array); const expectedBytes = new TextEncoder().encode(payload); @@ -1130,15 +1257,24 @@ describe('e2e', () => { // Default stream should close cleanly after the parent closes its // writable. - expect((await reader.read()).done).toBe(true); + assert(reader, 'Expected an open reader for forwarded writable stream'); + try { + const closeResult = await readWithDeadline( + reader, + Date.now() + STREAM_READ_TIMEOUT_MS + ); + expect(closeResult.timedOut).toBe(false); + expect(closeResult.done).toBe(true); + } finally { + reader.releaseLock(); + } - const returnValue = await run.returnValue; expect(returnValue).toMatchObject({ childRunId: expect.stringMatching(/^wrun_/), }); }); - test('fetchWorkflow', { timeout: 60_000 }, async () => { + test('fetchWorkflow', TO.default, async () => { const run = await start(await e2e('fetchWorkflow'), []); const returnValue = await run.returnValue; expect(returnValue).toMatchObject({ @@ -1149,7 +1285,7 @@ describe('e2e', () => { }); }); - test('promiseRaceStressTestWorkflow', { timeout: 60_000 }, async () => { + test('promiseRaceStressTestWorkflow', TO.default, async () => { const run = await start(await e2e('promiseRaceStressTestWorkflow'), []); const returnValue = await run.returnValue; // Completion order can vary across worlds and scheduling environments. @@ -1162,7 +1298,7 @@ describe('e2e', () => { describe('workflow errors', () => { test( 'nested function calls preserve message and stack trace', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('errorWorkflowNested'), []); const error = await run.returnValue.catch((e: unknown) => e); @@ -1197,7 +1333,7 @@ describe('e2e', () => { test( 'cross-file imports preserve message and stack trace', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('errorWorkflowCrossFile'), []); const error = await run.returnValue.catch((e: unknown) => e); @@ -1228,7 +1364,7 @@ describe('e2e', () => { describe('step errors', () => { test( 'basic step error preserves message and stack trace', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('errorStepBasic'), []); const result = await run.returnValue; @@ -1284,7 +1420,7 @@ describe('e2e', () => { test( 'cross-file step error preserves message and function names in stack', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('errorStepCrossFile'), []); const result = await run.returnValue; @@ -1343,29 +1479,25 @@ describe('e2e', () => { }); describe('retry behavior', () => { - test( - 'regular Error retries until success', - { timeout: 60_000 }, - async () => { - const run = await start(await e2e('errorRetrySuccess'), []); - const result = await run.returnValue; + test('regular Error retries until success', TO.default, async () => { + const run = await start(await e2e('errorRetrySuccess'), []); + const result = await run.returnValue; - expect(result.finalAttempt).toBe(3); + expect(result.finalAttempt).toBe(3); - const { json: steps } = await cliInspectJson( - `steps --runId ${run.runId}` - ); - const step = steps.find((s: any) => - s.stepName.includes('retryUntilAttempt3') - ); - expect(step.status).toBe('completed'); - expect(step.attempt).toBe(3); - } - ); + const { json: steps } = await cliInspectJson( + `steps --runId ${run.runId}` + ); + const step = steps.find((s: any) => + s.stepName.includes('retryUntilAttempt3') + ); + expect(step.status).toBe('completed'); + expect(step.attempt).toBe(3); + }); test( 'FatalError fails immediately without retries', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('errorRetryFatal'), []); const error = await run.returnValue.catch((e: unknown) => e); @@ -1391,17 +1523,17 @@ describe('e2e', () => { test( 'RetryableError respects custom retryAfter delay', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('errorRetryCustomDelay'), []); const result = await run.returnValue; expect(result.attempt).toBe(2); - expect(result.duration).toBeGreaterThan(10_000); + expectElapsedAtLeast(result.duration, 10_000); } ); - test('maxRetries=0 disables retries', { timeout: 60_000 }, async () => { + test('maxRetries=0 disables retries', TO.default, async () => { const run = await start(await e2e('errorRetryDisabled'), []); const result = await run.returnValue; @@ -1413,7 +1545,7 @@ describe('e2e', () => { describe('catchability', () => { test( 'FatalError can be caught and detected with FatalError.is()', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('errorFatalCatchable'), []); const result = await run.returnValue; @@ -1429,7 +1561,7 @@ describe('e2e', () => { test( 'step throw round-trips FatalError with cause chain to workflow catch', - { timeout: 60_000 }, + TO.default, async () => { // A step throws a FatalError whose `.cause` is a TypeError. The // workflow catches the rejection and inspects the hydrated value. @@ -1461,7 +1593,7 @@ describe('e2e', () => { test( 'workflow throw round-trips FatalError + cause through run_failed event', - { timeout: 60_000 }, + TO.default, async () => { // A workflow itself throws a FatalError with a RangeError cause. // Verifies that run_failed events go through the new @@ -1508,7 +1640,7 @@ describe('e2e', () => { test( 'workflow throw of a non-Error value round-trips verbatim as cause', - { timeout: 60_000 }, + TO.default, async () => { // Workflows may throw any JS value. Verify a plain object thrown // from a workflow surfaces verbatim as WorkflowRunFailedError.cause @@ -1536,7 +1668,7 @@ describe('e2e', () => { test( 'step throw of a non-Error value preserves it as cause on the wrapping FatalError', - { timeout: 60_000 }, + TO.default, async () => { // Steps may also throw any JS value. Non-Error throws aren't // recognized as `FatalError` (they have no `name === 'FatalError'`) @@ -1574,7 +1706,7 @@ describe('e2e', () => { describe('not registered', () => { test( 'WorkflowNotRegisteredError fails the run when workflow does not exist', - { timeout: 60_000 }, + TO.default, async () => { // Start a run with a workflowId that doesn't exist in the deployment bundle. // This simulates starting a run against a deployment that doesn't have the workflow. @@ -1599,7 +1731,7 @@ describe('e2e', () => { test( 'StepNotRegisteredError fails the step but workflow can catch it', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('stepNotRegisteredCatchable'), []); const result = await run.returnValue; @@ -1625,7 +1757,7 @@ describe('e2e', () => { test( 'StepNotRegisteredError fails the run when not caught in workflow', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('stepNotRegisteredUncaught'), []); const error = await run.returnValue.catch((e: unknown) => e); @@ -1642,7 +1774,7 @@ describe('e2e', () => { test( 'stepDirectCallWorkflow - calling step functions directly outside workflow context', - { timeout: 60_000 }, + TO.default, async () => { // Call the API route that directly calls a step function (no workflow context) const url = new URL('/api/test-direct-step-call', deploymentUrl); @@ -1672,7 +1804,7 @@ describe('e2e', () => { test( 'hookCleanupTestWorkflow - hook token reuse after workflow completion', - { timeout: 60_000 }, + TO.default, async () => { const token = Math.random().toString(36).slice(2); const customData = Math.random().toString(36).slice(2); @@ -1734,7 +1866,7 @@ describe('e2e', () => { test( 'concurrent hook token conflict - two workflows cannot use the same hook token simultaneously', - { timeout: 60_000 }, + TO.default, async () => { const token = Math.random().toString(36).slice(2); const customData = Math.random().toString(36).slice(2); @@ -1833,7 +1965,7 @@ describe('e2e', () => { }, ])( '$workflow - hook.getConflict() does not block step execution', - { timeout: 60_000 }, + TO.default, async ({ workflow, hookGetConflictTestData }) => { const token = Math.random().toString(36).slice(2); const customData = Math.random().toString(36).slice(2); @@ -1864,7 +1996,7 @@ describe('e2e', () => { test( 'hookGetConflictThenStepParallelWorkflow - hook.getConflict() continuation step runs alongside other steps', - { timeout: 90_000 }, + TO.long, async () => { const token = Math.random().toString(36).slice(2); const customData = Math.random().toString(36).slice(2); @@ -1883,7 +2015,6 @@ describe('e2e', () => { const { stepAResult, stepBResult } = returnValue; const stepADuration = stepAResult.endedAt - stepAResult.startedAt; - const stepStartDelta = stepBResult.startedAt - stepAResult.startedAt; expect(stepAResult.label).toBe('A'); expect(stepBResult.label).toBe('B'); @@ -1892,16 +2023,12 @@ describe('e2e', () => { stepBResult.startedAt, 'stepB should start before stepA finishes, proving hook.getConflict() can continue independently of other steps' ).toBeLessThan(stepAResult.endedAt); - expect( - stepStartDelta, - 'stepB should start roughly alongside stepA instead of only after stepA finishes' - ).toBeLessThan(8_000); } ); test( 'hookGetConflictWorkflow - hook.getConflict() resolves with the conflicting run when token is already registered', - { timeout: 60_000 }, + TO.default, async () => { const token = Math.random().toString(36).slice(2); const customData = Math.random().toString(36).slice(2); @@ -1950,7 +2077,7 @@ describe('e2e', () => { test( 'hookClaimOnlyMutexWorkflow - hook works as a pure run mutex without payload data', - { timeout: 90_000 }, + TO.long, async () => { const token = Math.random().toString(36).slice(2); @@ -2014,7 +2141,7 @@ describe('e2e', () => { test( 'hookAdoptOwnerResultWorkflow - duplicate adopts the owner result via conflict.returnValue', - { timeout: 120_000 }, + TO.extraLong, async () => { const token = Math.random().toString(36).slice(2); @@ -2075,7 +2202,7 @@ describe('e2e', () => { test( 'hookSignalOwnerWorkflow - duplicate forwards its payload to the owner via resumeHook', - { timeout: 90_000 }, + TO.long, async () => { const token = Math.random().toString(36).slice(2); @@ -2109,7 +2236,7 @@ describe('e2e', () => { test( 'hookSupersedeOwnerWorkflow - duplicate cancels the owner and claims the released token', - { timeout: 90_000 }, + TO.long, async () => { const token = Math.random().toString(36).slice(2); @@ -2123,7 +2250,10 @@ describe('e2e', () => { const run2 = await start(await e2e('hookSupersedeOwnerWorkflow'), [ token, ]); - await waitForHook(token, { runId: run2.runId, timeoutMs: 60_000 }); + const run2Hook = await waitForHook(token, { + runId: run2.runId, + timeoutMs: HOOK_WAIT_TIMEOUT_MS, + }); // The superseded owner ends up cancelled — assert via both the // rejected returnValue (also prevents an unhandled rejection from @@ -2134,7 +2264,7 @@ describe('e2e', () => { expect(run1Data.status).toBe('cancelled'); // The new owner receives payloads on the reclaimed token. - await resumeHook(token, { message: 'post-supersede' }); + await resumeHook(run2Hook, { message: 'post-supersede' }); const run2Result = await run2.returnValue; expect(run2Result).toMatchObject({ role: 'owner', @@ -2145,7 +2275,7 @@ describe('e2e', () => { test( 'resume-or-start route pattern - resumeHook retried after start() reaches the new run', - { timeout: 90_000 }, + TO.long, async () => { const token = Math.random().toString(36).slice(2); @@ -2190,7 +2320,7 @@ describe('e2e', () => { test( 'hookDisposeTestWorkflow - hook token reuse after explicit disposal while workflow still running', - { timeout: 90_000 }, + TO.long, async () => { const token = Math.random().toString(36).slice(2); const customData = Math.random().toString(36).slice(2); @@ -2281,7 +2411,7 @@ describe('e2e', () => { test( 'stepFunctionPassingWorkflow - step function references can be passed as arguments (without closure vars)', - { timeout: 60_000 }, + TO.default, async () => { // This workflow passes a step function reference to another step // The receiving step calls the passed function and returns the result @@ -2313,7 +2443,7 @@ describe('e2e', () => { test( 'stepFunctionWithClosureWorkflow - step function with closure variables passed as argument', - { timeout: 60_000 }, + TO.default, async () => { // This workflow creates a nested step function with closure variables, // then passes it to another step which invokes it. @@ -2338,7 +2468,7 @@ describe('e2e', () => { test( 'closureVariableWorkflow - nested step functions with closure variables', - { timeout: 60_000 }, + TO.default, async () => { // This workflow uses a nested step function that references closure variables // from the parent workflow scope (multiplier, prefix, baseValue) @@ -2352,7 +2482,7 @@ describe('e2e', () => { test( 'spawnWorkflowFromStepWorkflow - spawning a child workflow using start() inside a step', - { timeout: 120_000 }, + TO.extraLong, async () => { // This workflow spawns another workflow using start() inside a step function // This is the recommended pattern for spawning workflows from within workflows @@ -2397,7 +2527,7 @@ describe('e2e', () => { test( 'runClassSerializationWorkflow - Run instances serialize across workflow/step boundaries', - { timeout: 120_000 }, + TO.extraLong, async () => { const inputValue = 21; const run = await start(await e2e('runClassSerializationWorkflow'), [ @@ -2436,7 +2566,7 @@ describe('e2e', () => { test( 'startFromWorkflow - calling start() directly inside a workflow function with hook communication', - { timeout: 120_000 }, + TO.extraLong, async () => { const inputValue = 42; const run = await start(await e2e('startFromWorkflow'), [inputValue]); @@ -2458,7 +2588,7 @@ describe('e2e', () => { test( 'fibonacciWorkflow - recursive workflow composition via start()', - { timeout: 180_000 }, + TO.veryLong, async () => { // fib(6) = 8, spawns a tree of child workflow runs const run = await start(await e2e('fibonacciWorkflow'), [6]); @@ -2474,7 +2604,7 @@ describe('e2e', () => { // bypasses protection by sending messages through the Queue infrastructure. test.skipIf(!isLocalDeployment())( 'health check endpoint (HTTP) - workflow endpoint responds to __health query parameter', - { timeout: 30_000 }, + TO.short, async () => { // NOTE: This tests the HTTP-based health check using the `?__health` query parameter. // This approach requires direct HTTP access and works when running locally (for port detection) @@ -2510,7 +2640,7 @@ describe('e2e', () => { test( 'health check (queue-based) - workflow endpoint responds to health check messages', - { timeout: 60_000 }, + TO.default, async () => { // Tests the queue-based health check using healthCheck() directly. // This bypasses Vercel Deployment Protection by sending messages @@ -2531,7 +2661,7 @@ describe('e2e', () => { test( 'health check (CLI) - workflow health command reports healthy endpoints', - { timeout: 60_000 }, + TO.default, async () => { // NOTE: This tests the `workflow health` CLI command which uses the // queue-based health check under the hood. The CLI provides a convenient @@ -2554,7 +2684,7 @@ describe('e2e', () => { test( 'pathsAliasWorkflow - TypeScript path aliases resolve correctly', - { timeout: 60_000 }, + TO.default, async () => { // This workflow uses a step that calls a helper function imported via @repo/* path alias // which resolves to a file outside the workbench directory (../../lib/steps/paths-alias-test.ts) @@ -2578,7 +2708,7 @@ describe('e2e', () => { test( 'Calculator.calculate - static workflow method using static step methods from another class', - { timeout: 60_000 }, + TO.default, async () => { // Calculator.calculate(5, 3) should: // 1. MathService.add(5, 3) = 8 @@ -2600,7 +2730,7 @@ describe('e2e', () => { test( 'AllInOneService.processNumber - static workflow method using sibling static step methods', - { timeout: 60_000 }, + TO.default, async () => { // AllInOneService.processNumber(10) should: // 1. AllInOneService.double(10) = 20 @@ -2623,7 +2753,7 @@ describe('e2e', () => { test( 'ChainableService.processWithThis - static step methods using `this` to reference the class', - { timeout: 60_000 }, + TO.default, async () => { // ChainableService.processWithThis(5) should: // - ChainableService.multiplyByClassValue(5) uses `this.multiplier` (10) -> 5 * 10 = 50 @@ -2657,7 +2787,7 @@ describe('e2e', () => { test( 'thisSerializationWorkflow - step function invoked with .call() and .apply()', - { timeout: 60_000 }, + TO.default, async () => { // thisSerializationWorkflow(10) should: // 1. multiplyByFactor.call({ factor: 2 }, 10) = 20 @@ -2680,7 +2810,7 @@ describe('e2e', () => { test( 'customSerializationWorkflow - custom class serialization with WORKFLOW_SERIALIZE/WORKFLOW_DESERIALIZE', - { timeout: 60_000 }, + TO.default, async () => { // This workflow tests custom serialization of user-defined class instances. // The Point class uses WORKFLOW_SERIALIZE and WORKFLOW_DESERIALIZE symbols @@ -2717,7 +2847,7 @@ describe('e2e', () => { test( 'instanceMethodStepWorkflow - instance methods with "use step" directive', - { timeout: 60_000 }, + TO.default, async () => { // This workflow tests instance methods marked with "use step". // The Counter class has custom serialization so the `this` context @@ -2812,7 +2942,7 @@ describe('e2e', () => { test( 'crossContextSerdeWorkflow - classes defined in step code are deserializable in workflow context', - { timeout: 60_000 }, + TO.default, async () => { // This is a critical test for the cross-context class registration feature. // @@ -2865,7 +2995,7 @@ describe('e2e', () => { test( 'errorSubclassRoundTripWorkflow - first-class Error subclasses survive every serialization boundary', - { timeout: 60_000 }, + TO.default, async () => { // Round-trips one instance of each Error subclass that has a dedicated // reducer/reviver pair (built-in subclasses + FatalError/RetryableError @@ -2980,7 +3110,7 @@ describe('e2e', () => { test( 'stepFunctionAsStartArgWorkflow - step function reference passed as start() argument', - { timeout: 120_000 }, + TO.extraLong, async () => { // This test verifies that step function references can be: // 1. Serialized in the client bundle (the SWC plugin sets stepId property on the function) @@ -3043,43 +3173,51 @@ describe('e2e', () => { ); // ==================== CANCEL TESTS ==================== - test( - 'cancelRun - cancelling a running workflow', - { timeout: 60_000 }, - async () => { - // Start a long-running workflow with a 30s sleep to provide a wide - // window for the cancel to arrive while the workflow is still running. - const run = await start(await e2e('sleepingWorkflow'), [30_000]); - - // Wait for the workflow to start and enter the sleep - await new Promise((resolve) => setTimeout(resolve, 5_000)); + test('cancelRun - cancelling a running workflow', TO.default, async () => { + // Start a long-running workflow with a 30s sleep to provide a wide + // window for the cancel to arrive while the workflow is still running. + const run = await start(await e2e('sleepingWorkflow'), [30_000]); + + // Wait for the workflow to start and enter the sleep. + await waitForRunEvents( + run.runId, + (event) => event.eventType === 'wait_created', + { + description: 'wait_created event', + } + ); - // Cancel the run using the core runtime cancelRun function. - // This exercises the same cancelRun code path that the CLI uses - // (the CLI delegates directly to this function). - const { cancelRun } = await import('../src/runtime'); - await cancelRun(await getWorld(), run.runId); + // Cancel the run using the core runtime cancelRun function. + // This exercises the same cancelRun code path that the CLI uses + // (the CLI delegates directly to this function). + const { cancelRun } = await import('../src/runtime'); + await cancelRun(await getWorld(), run.runId); - // Verify the run was cancelled - returnValue should throw WorkflowRunCancelledError - const error = await run.returnValue.catch((e: unknown) => e); - expect(WorkflowRunCancelledError.is(error)).toBe(true); + // Verify the run was cancelled - returnValue should throw WorkflowRunCancelledError + const error = await run.returnValue.catch((e: unknown) => e); + expect(WorkflowRunCancelledError.is(error)).toBe(true); - // Verify the run status is 'cancelled' via CLI inspect - const { json: runData } = await cliInspectJson(`runs ${run.runId}`); - expect(runData.status).toBe('cancelled'); - } - ); + // Verify the run status is 'cancelled' via CLI inspect + const { json: runData } = await cliInspectJson(`runs ${run.runId}`); + expect(runData.status).toBe('cancelled'); + }); test( 'cancelRun via CLI - cancelling a running workflow', - { timeout: 60_000 }, + TO.default, async () => { // Start a long-running workflow with a 30s sleep to provide a wide // window for the cancel to arrive while the workflow is still running. const run = await start(await e2e('sleepingWorkflow'), [30_000]); - // Wait for the workflow to start and enter the sleep - await new Promise((resolve) => setTimeout(resolve, 5_000)); + // Wait for the workflow to start and enter the sleep. + await waitForRunEvents( + run.runId, + (event) => event.eventType === 'wait_created', + { + description: 'wait_created event', + } + ); // Cancel the run via the CLI command. This tests the full CLI code path // including World.close() which ensures the process exits cleanly. @@ -3102,7 +3240,7 @@ describe('e2e', () => { process.env.APP_NAME === 'nextjs-webpack'; describe.skipIf(!isNextJsApp)('pages router', () => { - test('addTenWorkflow via pages router', { timeout: 60_000 }, async () => { + test('addTenWorkflow via pages router', TO.default, async () => { const run = await startWorkflowViaHttp( { workflowFile: 'workflows/99_e2e.ts', @@ -3115,21 +3253,17 @@ describe('e2e', () => { expect(returnValue).toBe(133); }); - test( - 'promiseAllWorkflow via pages router', - { timeout: 60_000 }, - async () => { - const run = await startWorkflowViaHttp( - 'promiseAllWorkflow', - [], - '/api/trigger-pages' - ); - const returnValue = await run.returnValue; - expect(returnValue).toBe('ABC'); - } - ); + test('promiseAllWorkflow via pages router', TO.default, async () => { + const run = await startWorkflowViaHttp( + 'promiseAllWorkflow', + [], + '/api/trigger-pages' + ); + const returnValue = await run.returnValue; + expect(returnValue).toBe('ABC'); + }); - test('sleepingWorkflow via pages router', { timeout: 60_000 }, async () => { + test('sleepingWorkflow via pages router', TO.default, async () => { const run = await startWorkflowViaHttp( 'sleepingWorkflow', [], @@ -3137,13 +3271,13 @@ describe('e2e', () => { ); const returnValue = await run.returnValue; expect(returnValue.startTime).toBeLessThan(returnValue.endTime); - expect(returnValue.endTime - returnValue.startTime).toBeGreaterThan(9999); + expectElapsedAtLeast(returnValue.endTime - returnValue.startTime, 10_000); }); }); test( 'hookWithSleepWorkflow - hook payloads delivered correctly with concurrent sleep', - { timeout: 90_000 }, + TO.long, async () => { // Regression test: when a hook and sleep run concurrently, multiple // hook_received events should all be processed even though the sleep @@ -3159,13 +3293,23 @@ describe('e2e', () => { expect(hook.runId).toBe(run.runId); await resumeHook(hook, { type: 'subscribe', id: 1 }); - // Wait for the first payload to be processed (step must complete) - await new Promise((resolve) => setTimeout(resolve, 3_000)); + await waitForRunEvents( + run.runId, + (event) => event.eventType === 'step_completed', + { description: 'step_completed event after first hook payload' } + ); hook = await getHookByToken(token); await resumeHook(hook, { type: 'subscribe', id: 2 }); - await new Promise((resolve) => setTimeout(resolve, 3_000)); + await waitForRunEvents( + run.runId, + (event) => event.eventType === 'step_completed', + { + minCount: 2, + description: 'step_completed events after second hook payload', + } + ); hook = await getHookByToken(token); await resumeHook(hook, { type: 'done', done: true }); @@ -3192,7 +3336,7 @@ describe('e2e', () => { test( 'hookWithSleepFinalStepWorkflow - step only on final payload', - { timeout: 120_000 }, + TO.extraLong, async () => { // Regression test for the v0chat incident. Mirrors the production // shape: a hook + fire-and-forget sleep, where the step runs only @@ -3205,19 +3349,20 @@ describe('e2e', () => { token, ]); - // Wait for the hook to register. - await new Promise((resolve) => setTimeout(resolve, 5_000)); - - let hook = await getHookByToken(token); + let hook = await waitForHook(token, { runId: run.runId }); expect(hook.runId).toBe(run.runId); await resumeHook(hook, { type: 'msg', id: 1 }); // Let the workflow replay and suspend before the next payload so the // final event log contains two `hook_received` entries before any // `step_created` — the exact replay shape from production. - await new Promise((resolve) => setTimeout(resolve, 3_000)); + await waitForRunEvents( + run.runId, + (event) => event.eventType === 'hook_received', + { description: 'hook_received event after first payload' } + ); - hook = await getHookByToken(token); + hook = await waitForHook(token, { runId: run.runId }); await resumeHook(hook, { type: 'final', id: 2, done: true }); const returnValue = await run.returnValue; @@ -3233,7 +3378,7 @@ describe('e2e', () => { test( 'sleepInLoopWorkflow - sleep inside loop with steps actually delays each iteration', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('sleepInLoopWorkflow'), []); const returnValue = await run.returnValue; @@ -3243,15 +3388,15 @@ describe('e2e', () => { expect(returnValue.timestamps).toHaveLength(3); const delta1 = returnValue.timestamps[1] - returnValue.timestamps[0]; const delta2 = returnValue.timestamps[2] - returnValue.timestamps[1]; - expect(delta1).toBeGreaterThan(2_500); - expect(delta2).toBeGreaterThan(2_500); - expect(returnValue.totalElapsed).toBeGreaterThan(5_000); + expectElapsedAtLeast(delta1, 3_000); + expectElapsedAtLeast(delta2, 3_000); + expectElapsedAtLeast(returnValue.totalElapsed, 6_000); } ); test( 'sleepWithSequentialStepsWorkflow - sequential steps work with concurrent sleep (control)', - { timeout: 60_000 }, + TO.default, async () => { // Control test: proves that void sleep('1d').then() does NOT break // sequential step execution. Steps have per-event consumption so the @@ -3277,7 +3422,7 @@ describe('e2e', () => { describe('AbortController', () => { test( 'abortTimeoutWorkflow: timeout cancels long-running step', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('abortTimeoutWorkflow'), []); const returnValue = await run.returnValue; @@ -3312,7 +3457,7 @@ describe('e2e', () => { test( 'abortParallelWorkflow: abort cancels all parallel steps', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('abortParallelWorkflow'), []); const returnValue = await run.returnValue; @@ -3327,7 +3472,7 @@ describe('e2e', () => { test( 'abortFromStepWorkflow: step abort cancels an in-flight sibling step', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('abortFromStepWorkflow'), []); const returnValue = await run.returnValue; @@ -3348,7 +3493,7 @@ describe('e2e', () => { test( 'abortAlreadyAbortedWorkflow: pre-aborted signal seen by step', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('abortAlreadyAbortedWorkflow'), []); const returnValue = await run.returnValue; @@ -3362,7 +3507,7 @@ describe('e2e', () => { test( 'abortReasonWorkflow: abort reason preserved across boundaries', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('abortReasonWorkflow'), []); const returnValue = await run.returnValue; @@ -3376,7 +3521,7 @@ describe('e2e', () => { test( 'abortAfterCompletionWorkflow: abort after step completes is a no-op', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('abortAfterCompletionWorkflow'), []); const returnValue = await run.returnValue; @@ -3390,7 +3535,7 @@ describe('e2e', () => { test( 'abortViaHookWorkflow: external hook triggers abort on in-flight step', - { timeout: 60_000 }, + TO.default, async () => { const token = Math.random().toString(36).slice(2); const run = await start(await e2e('abortViaHookWorkflow'), [token]); @@ -3410,7 +3555,7 @@ describe('e2e', () => { test( 'abortExternalSignalWorkflow: signal passed as workflow input', - { timeout: 60_000 }, + TO.default, async () => { // Pass a pre-aborted AbortController to the workflow. // The workflow receives the signal and passes it to a step. @@ -3430,7 +3575,7 @@ describe('e2e', () => { test( 'abortExternalSignalInFlightWorkflow: external abort fires mid-flight, propagates to nested steps', - { timeout: 60_000 }, + TO.default, async () => { // Source controller starts NOT aborted. The serialization-time // listener attached during start() must write the cancellation packet @@ -3471,7 +3616,7 @@ describe('e2e', () => { test( 'abortAnyInWorkflowWorkflow: AbortSignal.any composes signals inside the workflow VM', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('abortAnyInWorkflowWorkflow'), []); const returnValue = await run.returnValue; @@ -3489,7 +3634,7 @@ describe('e2e', () => { test( 'abortAnyInStepWorkflow: AbortSignal.any inside a step composes deserialized signals', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('abortAnyInStepWorkflow'), []); const returnValue = await run.returnValue; @@ -3507,7 +3652,7 @@ describe('e2e', () => { test( 'abortSurvivesReplayWorkflow: controller state consistent across replay', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('abortSurvivesReplayWorkflow'), []); const returnValue = await run.returnValue; @@ -3522,7 +3667,7 @@ describe('e2e', () => { test( 'abortThrowIfAbortedWorkflow: throwIfAborted causes FatalError, no retries', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('abortThrowIfAbortedWorkflow'), []); const returnValue = await run.returnValue; @@ -3536,7 +3681,7 @@ describe('e2e', () => { test( 'abortReasonTypesWorkflow: various abort reason types propagate correctly', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('abortReasonTypesWorkflow'), []); const returnValue = await run.returnValue; @@ -3559,7 +3704,7 @@ describe('e2e', () => { test( 'abortFetchUncaughtWorkflow: uncaught fetch AbortError is FatalError, no retries', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('abortFetchUncaughtWorkflow'), []); const returnValue = await run.returnValue; @@ -3579,7 +3724,7 @@ describe('e2e', () => { test( 'abortFetchInFlightWorkflow: aborting cancels an in-flight fetch', - { timeout: 60_000 }, + TO.default, async () => { // The workflow kicks off a fetch against a slow endpoint, races it // against a 2s sleep, and aborts when the sleep wins. The fetch must @@ -3605,7 +3750,7 @@ describe('e2e', () => { test( 'abortVoidSleepTimeoutWorkflow: documented `void sleep().then(abort)` pattern works', - { timeout: 60_000 }, + TO.default, async () => { // Validates the simplified timeout pattern documented on the // abort-signal-timeout-in-workflow error page: @@ -3631,7 +3776,7 @@ describe('e2e', () => { test( 'abortDeterministicBranchWorkflow: if-check takes same path on first-run and replay', - { timeout: 60_000 }, + TO.default, async () => { const run = await start( await e2e('abortDeterministicBranchWorkflow'), @@ -3650,7 +3795,7 @@ describe('e2e', () => { test( 'abortListenerWorkflow: signal.addEventListener fires on the deserialized step signal', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('abortListenerWorkflow'), []); const returnValue = await run.returnValue; @@ -3666,7 +3811,7 @@ describe('e2e', () => { test( 'abortThrowIfAbortedMidFlightWorkflow: throwIfAborted in a polling loop bails when abort fires', - { timeout: 60_000 }, + TO.default, async () => { const run = await start( await e2e('abortThrowIfAbortedMidFlightWorkflow'), @@ -3685,7 +3830,7 @@ describe('e2e', () => { test( 'abortDeterministicBranchFromStepWorkflow: branches stay consistent when abort comes from a step', - { timeout: 60_000 }, + TO.default, async () => { const run = await start( await e2e('abortDeterministicBranchFromStepWorkflow'), @@ -3744,7 +3889,7 @@ describe('e2e', () => { } of orderingVariants) { test( `abortHookOrderingWorkflow [${variant}]: ${description}`, - { timeout: 90_000 }, + TO.long, async () => { const token = `ordering-${variant}-${Math.random().toString(36).slice(2)}`; const run = await start(await e2e('abortHookOrderingWorkflow'), [ @@ -3788,7 +3933,7 @@ describe('e2e', () => { test( 'importMetaUrlWorkflow - import.meta.url is available in step bundles', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('importMetaUrlWorkflow'), []); const returnValue = await run.returnValue; @@ -3802,7 +3947,7 @@ describe('e2e', () => { test( 'metadataFromHelperWorkflow - getWorkflowMetadata/getStepMetadata work from module-level helper (#1577)', - { timeout: 60_000 }, + TO.default, async () => { const run = await start(await e2e('metadataFromHelperWorkflow'), [ 'smoke-test', @@ -3824,7 +3969,7 @@ describe('e2e', () => { // over the queue boundary. test( 'resilient start: addTenWorkflow completes when run_created returns 500', - { timeout: 60_000 }, + TO.default, async () => { // Get the real world and wrap it so the first events.create call // (run_created) throws a 500 server error. The queue should still @@ -3869,7 +4014,7 @@ describe('e2e', () => { test( 'getterStepWorkflow - getter functions with "use step" directive', - { timeout: 60_000 }, + TO.default, async () => { // This workflow tests getter functions marked with "use step". // The Sensor class has custom serialization so the `this` context @@ -3907,7 +4052,7 @@ describe('e2e', () => { // ============================================================ test( 'distributedAbortController - manual abort triggers signal', - { timeout: 60_000 }, + TO.default, async () => { const controllerId = `test-abort-${Math.random().toString(36).slice(2)}`; @@ -3943,7 +4088,7 @@ describe('e2e', () => { test( 'distributedAbortController - TTL expiration triggers signal', - { timeout: 30_000 }, + TO.short, async () => { const controllerId = `test-expire-${Math.random().toString(36).slice(2)}`; @@ -3971,7 +4116,7 @@ describe('e2e', () => { test( 'distributedAbortController - reconnect to existing controller', - { timeout: 60_000 }, + TO.default, async () => { const controllerId = `test-reconnect-${Math.random().toString(36).slice(2)}`; const token = `distributed-abort:${controllerId}`; @@ -3993,10 +4138,6 @@ describe('e2e', () => { // Abort the controller await resumeHook(token, { reason: 'Test complete' }); - // Workflow should still be running (grace period), so hook should still be findable - await sleep(1_000); - const _hookAfterAbort = await getHookByToken(token).catch(() => null); - // Hook may or may not be disposed depending on timing, but run should complete const returnValue = await run1.returnValue; expect(returnValue.aborted).toBe(true); expect(returnValue.reason).toBe('Test complete'); @@ -4010,7 +4151,7 @@ describe('e2e', () => { describe('experimental_setAttributes', () => { test( 'start: initial attributes are seeded on run creation', - { timeout: 30_000 }, + TO.short, async () => { const run = await start( await e2e('experimentalSetAttributesWorkflow'), @@ -4030,7 +4171,7 @@ describe('e2e', () => { test( 'start: reserved-prefix initial attributes are seeded with allowReservedAttributes', - { timeout: 30_000 }, + TO.short, async () => { const run = await start( await e2e('experimentalSetAttributesWorkflow'), @@ -4057,7 +4198,7 @@ describe('e2e', () => { test( 'experimentalSetAttributesWorkflow: workflow-body calls append native attr_set events and merge correctly', - { timeout: 30_000 }, + TO.short, async () => { const run = await start( await e2e('experimentalSetAttributesWorkflow'), @@ -4085,7 +4226,7 @@ describe('e2e', () => { test( 'experimentalSetAttributesInsideStepWorkflow: step-body calls append attributed native events', - { timeout: 30_000 }, + TO.short, async () => { const run = await start( await e2e('experimentalSetAttributesInsideStepWorkflow'), @@ -4115,7 +4256,7 @@ describe('e2e', () => { test( 'fire-and-forget: void experimental_setAttributes lands without awaiting', - { timeout: 30_000 }, + TO.short, async () => { const run = await start( await e2e('experimentalSetAttributesFireAndForgetWorkflow'), @@ -4133,7 +4274,7 @@ describe('e2e', () => { test( 'Promise.all of disjoint-key writes: every key lands', - { timeout: 30_000 }, + TO.short, async () => { const run = await start( await e2e('experimentalSetAttributesParallelWorkflow'), @@ -4153,7 +4294,7 @@ describe('e2e', () => { test( 'workflow throws after awaited setAttributes: attribute still persists on the failed run', - { timeout: 30_000 }, + TO.short, async () => { const run = await start( await e2e('experimentalSetAttributesThrowsAfterWorkflow'), @@ -4180,7 +4321,7 @@ describe('e2e', () => { test( 'validation DX: invalid writes throw catchable FatalErrors naming rule and limit', - { timeout: 30_000 }, + TO.short, async () => { const run = await start( await e2e('experimentalSetAttributesValidationWorkflow'), @@ -4218,7 +4359,7 @@ describe('e2e', () => { test( 'start: invalid initial attributes are rejected before a run is created', - { timeout: 30_000 }, + TO.short, async () => { const workflow = await e2e('experimentalSetAttributesWorkflow'); await expect( diff --git a/packages/core/src/runtime/step-executor.ts b/packages/core/src/runtime/step-executor.ts index 352c0cc0d5..7a5c9b6292 100644 --- a/packages/core/src/runtime/step-executor.ts +++ b/packages/core/src/runtime/step-executor.ts @@ -39,7 +39,7 @@ import { } from './constants.js'; import { getPortLazy } from './get-port-lazy.js'; import { memoizeEncryptionKey } from './helpers.js'; -import { safeWaitUntil } from './wait-until.js'; +import { waitForBackgroundOps } from './wait-until.js'; const DEFAULT_STEP_MAX_RETRIES = 3; @@ -449,6 +449,7 @@ export async function executeStep( }); let result: unknown; + let propagateStepCompletedError = false; // Check max retries AFTER step_started (attempt was just incremented). // Only enforce when the step has a previous error — this distinguishes @@ -627,33 +628,20 @@ export async function executeStep( // across steps), waitUntil handles the rest. let opsSettled = true; if (ops.length > 0) { - const opsPromise = Promise.all(ops); // The race below surfaces failures inline when ops settle quickly; - // if the 500ms timeout wins, the failure is only observed here. The + // if the 500ms timeout wins, waitUntil keeps the work alive. The // promise handed to waitUntil must never reject (an unconsumed - // waitUntil rejection crashes the process as unhandledRejection), - // so unexpected failures are logged instead. - safeWaitUntil(opsPromise, (err) => { - runtimeLogger.warn('Background flush of step stream ops failed', { - workflowRunId, - stepId, - error: err instanceof Error ? err.message : String(err), - }); + // waitUntil rejection crashes the process as unhandledRejection), so + // unexpected failures are logged instead. + opsSettled = await waitForBackgroundOps(Promise.all(ops), { + onError: (err) => { + runtimeLogger.warn('Background flush of step stream ops failed', { + workflowRunId, + stepId, + error: err instanceof Error ? err.message : String(err), + }); + }, }); - opsSettled = await Promise.race([ - opsPromise.then( - () => true as const, - (err) => { - // Ignore expected client disconnect errors (e.g., browser - // refresh during streaming) - const isAbortError = - err?.name === 'AbortError' || err?.name === 'ResponseAborted'; - if (isAbortError) return true as const; - throw err; - } - ), - new Promise((r) => setTimeout(() => r(false), 500)), - ]); } // Optimistic start: the body ran before `step_started` was confirmed. @@ -720,6 +708,7 @@ export async function executeStep( stepCompleted409 = true; return undefined; } + propagateStepCompletedError = true; throw err; }); @@ -750,6 +739,8 @@ export async function executeStep( // and queue a continuation so waitUntil can flush them. return { type: 'completed', hasPendingOps: !opsSettled, inlineDelta }; } catch (err: unknown) { + if (propagateStepCompletedError && !RunExpiredError.is(err)) throw err; + // Optimistic start: the body threw before `step_started` was confirmed. // Reconcile first — if we lost the create-claim (or the run is // gone/throttled) the body error is moot; discard it and don't write a diff --git a/packages/core/src/runtime/step-handler.test.ts b/packages/core/src/runtime/step-handler.test.ts index dc5a4bd670..915d364641 100644 --- a/packages/core/src/runtime/step-handler.test.ts +++ b/packages/core/src/runtime/step-handler.test.ts @@ -1,6 +1,7 @@ import { EntityConflictError, FatalError, + RunExpiredError, ThrottleError, WorkflowWorldError, } from '@workflow/errors'; @@ -283,7 +284,6 @@ describe('step-handler 409 handling', () => { describe('step_completed 409', () => { it('should warn and return when step_completed gets a 409', async () => { // step_started succeeds, step function succeeds, step_completed returns 409 - let callCount = 0; mockEventsCreate.mockImplementation( (_runId: string, event: { eventType: string }) => { if (event.eventType === 'step_started') { @@ -299,7 +299,6 @@ describe('step-handler 409 handling', () => { }); } if (event.eventType === 'step_completed') { - callCount++; return Promise.reject( new EntityConflictError( 'Cannot complete step because it is already completed' @@ -537,7 +536,7 @@ describe('step-handler 409 handling', () => { event.eventType === 'step_started' ); expect(startedCall).toBeDefined(); - expect(startedCall![2]).toEqual( + expect(startedCall?.[2]).toEqual( expect.objectContaining({ requestId: 'iad1::req-abc' }) ); }); @@ -553,7 +552,7 @@ describe('step-handler 409 handling', () => { event.eventType === 'step_completed' ); expect(completedCall).toBeDefined(); - expect(completedCall![2]).toEqual( + expect(completedCall?.[2]).toEqual( expect.objectContaining({ requestId: 'iad1::req-abc' }) ); }); @@ -566,7 +565,7 @@ describe('step-handler 409 handling', () => { event.eventType === 'step_started' ); expect(startedCall).toBeDefined(); - expect(startedCall![2]).toEqual( + expect(startedCall?.[2]).toEqual( expect.objectContaining({ requestId: undefined }) ); }); @@ -647,7 +646,7 @@ describe('step-handler max deliveries', () => { }); it('should not trigger max deliveries check when under limit', async () => { - const result = await capturedHandler(createMessage(), { + await capturedHandler(createMessage(), { ...createMetadata('myStep'), attempt: MAX_QUEUE_DELIVERIES, }); @@ -1175,6 +1174,74 @@ describe('executeStep inline-delta threading', () => { if (result.type !== 'completed') throw new Error('unreachable'); expect(result.inlineDelta).toBeUndefined(); }); + + it('rethrows step_completed write failures without recording a step failure', async () => { + const completionError = new Error('fetch failed'); + mockEventsCreate.mockImplementation( + (_runId: string, event: { eventType: string }) => { + if (event.eventType === 'step_started') { + return Promise.resolve({ + step: { + stepId: 'step_abc', + status: 'running', + attempt: 1, + startedAt: new Date(), + input: [], + }, + event: {}, + }); + } + if (event.eventType === 'step_completed') { + return Promise.reject(completionError); + } + return Promise.resolve({ event: {} }); + } + ); + + const world = await getWorld(); + await expect( + executeStep({ world: world as never, ...baseParams }) + ).rejects.toBe(completionError); + + expect( + mockEventsCreate.mock.calls.map( + ([, event]) => (event as { eventType: string }).eventType + ) + ).toEqual(['step_started', 'step_completed']); + }); + + it('treats a RunExpiredError from step_completed as gone', async () => { + mockEventsCreate.mockImplementation( + (_runId: string, event: { eventType: string }) => { + if (event.eventType === 'step_started') { + return Promise.resolve({ + step: { + stepId: 'step_abc', + status: 'running', + attempt: 1, + startedAt: new Date(), + input: [], + }, + event: {}, + }); + } + if (event.eventType === 'step_completed') { + return Promise.reject(new RunExpiredError('run already completed')); + } + return Promise.resolve({ event: {} }); + } + ); + + const world = await getWorld(); + const result = await executeStep({ world: world as never, ...baseParams }); + + expect(result).toEqual({ type: 'gone' }); + expect( + mockEventsCreate.mock.calls.map( + ([, event]) => (event as { eventType: string }).eventType + ) + ).toEqual(['step_started', 'step_completed']); + }); }); describe('executeStep optimistic inline start', () => { @@ -1213,7 +1280,7 @@ describe('executeStep optimistic inline start', () => { it('sends step_started carrying the input and completes (when enabled)', async () => { mockEventsCreate .mockReset() - .mockImplementation((_runId: string, event: { eventType: string }) => + .mockImplementation((_runId: string, _event: { eventType: string }) => Promise.resolve({ event: {} }) ); diff --git a/packages/core/src/runtime/step-handler.ts b/packages/core/src/runtime/step-handler.ts index 2a8ba2631f..f7928d1243 100644 --- a/packages/core/src/runtime/step-handler.ts +++ b/packages/core/src/runtime/step-handler.ts @@ -57,7 +57,7 @@ import { queueMessage, withHealthCheck, } from './helpers.js'; -import { safeWaitUntil } from './wait-until.js'; +import { waitForBackgroundOps } from './wait-until.js'; import { getWorld, getWorldHandlers, type WorldHandlers } from './world.js'; const DEFAULT_STEP_MAX_RETRIES = 3; @@ -1081,16 +1081,18 @@ function createStepHandler(namespace?: string) { // resilient to the below code not executing on a failed step. // (Dehydration already happened above and is accounted for in the // userCodeFailed path.) - // These stream ops are flushed in the background; the promise - // handed to waitUntil must never reject (an unconsumed waitUntil - // rejection crashes the process as unhandledRejection), so - // unexpected failures are logged instead. - safeWaitUntil(Promise.all(ops), (err) => { - stepRuntimeLogger.warn( - 'Background flush of step stream ops failed', - { error: err instanceof Error ? err.message : String(err) } - ); - }); + if (ops.length > 0) { + // Match inline step execution: surface quick stream flush failures + // before step_completed, but leave slow/open streams to waitUntil. + await waitForBackgroundOps(Promise.all(ops), { + onError: (err) => { + stepRuntimeLogger.warn( + 'Background flush of step stream ops failed', + { error: err instanceof Error ? err.message : String(err) } + ); + }, + }); + } // Run step_completed and trace serialization concurrently; // the trace carrier is used in the final queueMessage call below diff --git a/packages/core/src/runtime/wait-until.test.ts b/packages/core/src/runtime/wait-until.test.ts index 953997d880..3b4bdadd76 100644 --- a/packages/core/src/runtime/wait-until.test.ts +++ b/packages/core/src/runtime/wait-until.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it, vi } from 'vitest'; -import { safeWaitUntil } from './wait-until'; +import { + isExpectedClientDisconnectError, + safeWaitUntil, + waitForBackgroundOps, +} from './wait-until'; describe('safeWaitUntil', () => { it('invokes onError for unexpected rejections instead of letting the promise reject', async () => { @@ -31,4 +35,45 @@ describe('safeWaitUntil', () => { await new Promise((resolve) => setImmediate(resolve)); expect(onError).not.toHaveBeenCalled(); }); + + it('identifies expected client-disconnect errors', () => { + const abortError = new Error('client disconnected'); + abortError.name = 'AbortError'; + const responseAborted = new Error('response aborted'); + responseAborted.name = 'ResponseAborted'; + + expect(isExpectedClientDisconnectError(abortError)).toBe(true); + expect(isExpectedClientDisconnectError(responseAborted)).toBe(true); + expect(isExpectedClientDisconnectError(new Error('boom'))).toBe(false); + expect(isExpectedClientDisconnectError(undefined)).toBe(false); + }); +}); + +describe('waitForBackgroundOps', () => { + it('returns true when work settles inside the timeout', async () => { + const onError = vi.fn(); + await expect( + waitForBackgroundOps(Promise.resolve('ok'), { onError, timeoutMs: 50 }) + ).resolves.toBe(true); + expect(onError).not.toHaveBeenCalled(); + }); + + it('returns false when work is still pending after the timeout', async () => { + const onError = vi.fn(); + await expect( + waitForBackgroundOps(new Promise(() => {}), { onError, timeoutMs: 1 }) + ).resolves.toBe(false); + expect(onError).not.toHaveBeenCalled(); + }); + + it('surfaces quick unexpected failures while keeping waitUntil safe', async () => { + const onError = vi.fn(); + const err = new Error('boom'); + + await expect( + waitForBackgroundOps(Promise.reject(err), { onError, timeoutMs: 50 }) + ).rejects.toBe(err); + await new Promise((resolve) => setImmediate(resolve)); + expect(onError).toHaveBeenCalledWith(err); + }); }); diff --git a/packages/core/src/runtime/wait-until.ts b/packages/core/src/runtime/wait-until.ts index 724f244a49..04f7e05aa1 100644 --- a/packages/core/src/runtime/wait-until.ts +++ b/packages/core/src/runtime/wait-until.ts @@ -4,6 +4,14 @@ export function waitUntil(promise: Promise): void { }); } +export function isExpectedClientDisconnectError(err: unknown): boolean { + const name = + typeof err === 'object' && err !== null && 'name' in err + ? (err as { name?: unknown }).name + : undefined; + return name === 'AbortError' || name === 'ResponseAborted'; +} + /** * Schedule a background promise via `waitUntil`, guaranteeing that the * promise handed to `waitUntil` can never reject. Nothing consumes a @@ -20,9 +28,7 @@ export function safeWaitUntil( ): void { waitUntil( promise.catch((err) => { - const isAbortError = - err?.name === 'AbortError' || err?.name === 'ResponseAborted'; - if (!isAbortError) { + if (!isExpectedClientDisconnectError(err)) { try { onError(err); } catch { @@ -33,6 +39,40 @@ export function safeWaitUntil( ); } +/** + * Schedule a background promise and wait a short window for quick success or + * failure. Returns false when the timeout wins so callers can queue a + * continuation while `waitUntil` keeps the background work alive. + */ +export async function waitForBackgroundOps( + promise: Promise, + { + onError, + timeoutMs = 500, + }: { onError: (err: unknown) => void; timeoutMs?: number } +): Promise { + safeWaitUntil(promise, onError); + + let timeout: ReturnType | undefined; + try { + return await Promise.race([ + promise.then( + () => true as const, + (err) => { + if (isExpectedClientDisconnectError(err)) return true as const; + throw err; + } + ), + new Promise((resolve) => { + timeout = setTimeout(() => resolve(false), timeoutMs); + timeout.unref?.(); + }), + ]); + } finally { + if (timeout) clearTimeout(timeout); + } +} + /** * A small wrapper around `waitUntil` that also returns * the result of the awaited promise. diff --git a/packages/core/src/serialization.test.ts b/packages/core/src/serialization.test.ts index 5c21c55c3c..781fef6682 100644 --- a/packages/core/src/serialization.test.ts +++ b/packages/core/src/serialization.test.ts @@ -36,14 +36,17 @@ import { maybeEncrypt, SerializationFormat, } from './serialization.js'; +import type { StepContext } from './step/context-storage.js'; import { ABORT_HOOK_TOKEN, ABORT_READER_CANCEL, ABORT_STREAM_NAME, STABLE_ULID, + STREAM_CLIENT_NAME_SYMBOL, STREAM_NAME_SYMBOL, STREAM_SERVER_DEPLOYMENT_ID_SYMBOL, STREAM_SERVER_RUN_ID_SYMBOL, + WEBHOOK_RESPONSE_WRITABLE, } from './symbols.js'; import { createContext } from './vm/index.js'; @@ -686,6 +689,83 @@ describe('workflow arguments', () => { expect(streamName).toMatch(/^strm_[0-9A-Z]{26}$/); }); + it('preserves returned ReadableStream metadata for external hydration', async () => { + const ops: Promise[] = []; + const stream = new ReadableStream({ + start(controller) { + controller.close(); + }, + }); + const stepSerialized = await dehydrateStepReturnValue( + stream, + mockRunId, + noEncryptionKey, + ops, + globalThis, + false, + true + ); + const workflowHandle = await hydrateStepReturnValue( + stepSerialized, + mockRunId, + noEncryptionKey + ); + const workflowSerialized = await dehydrateWorkflowReturnValue( + workflowHandle, + mockRunId, + noEncryptionKey + ); + const hydrated = await hydrateWorkflowReturnValue( + workflowSerialized, + mockRunId, + noEncryptionKey + ); + + expect(hydrated[STREAM_CLIENT_NAME_SYMBOL]).toBe( + workflowHandle[STREAM_NAME_SYMBOL] + ); + expect(hydrated[STREAM_NAME_SYMBOL]).toBeUndefined(); + await Promise.all(ops); + }); + + it('does not tag step-hydrated ReadableStreams as reusable handles', async () => { + const ops: Promise[] = []; + const stream = new ReadableStream({ + start(controller) { + controller.close(); + }, + }); + const stepSerialized = await dehydrateStepReturnValue( + stream, + mockRunId, + noEncryptionKey, + ops, + globalThis, + false, + true + ); + const workflowHandle = await hydrateStepReturnValue( + stepSerialized, + mockRunId, + noEncryptionKey + ); + const stepArgs = await dehydrateStepArguments( + workflowHandle, + mockRunId, + noEncryptionKey + ); + const stepOps: Promise[] = []; + const hydrated = await hydrateStepArguments( + stepArgs, + mockRunId, + noEncryptionKey, + stepOps + ); + + expect(hydrated[STREAM_NAME_SYMBOL]).toBeUndefined(); + await Promise.all([...ops, ...stepOps]); + }); + it('should work with Headers', async () => { const headers = new Headers(); headers.set('foo', 'bar'); @@ -2009,6 +2089,75 @@ describe('workflow return value', () => { }); describe('step arguments', () => { + const webhookResponseStreamName = 'strm_manual_webhook_response'; + + async function hydrateManualWebhookRequest(world = makeMockWorld()) { + const { getWorldLazy } = await import('./runtime/get-world-lazy.js'); + vi.mocked(getWorldLazy).mockReturnValue(world as any); + + const responseWritable = new WritableStream(); + Object.defineProperty(responseWritable, STREAM_NAME_SYMBOL, { + value: webhookResponseStreamName, + writable: false, + }); + Object.defineProperty(responseWritable, STREAM_SERVER_RUN_ID_SYMBOL, { + value: mockRunId, + writable: false, + }); + + const request = new Request('https://example.com/webhook', { + method: 'POST', + }); + (request as any)[WEBHOOK_RESPONSE_WRITABLE] = responseWritable; + + const serialized = await dehydrateStepArguments( + request, + mockRunId, + noEncryptionKey, + globalThis + ); + const ops: Promise[] = []; + const hydrated = (await hydrateStepArguments( + serialized, + mockRunId, + noEncryptionKey, + ops, + globalThis + )) as Request & { respondWith(response: Response): Promise }; + + const preCompletionOps: Promise[] = []; + const stepContext: StepContext = { + stepMetadata: { + stepName: 'sendWebhookResponse', + stepId: 'step_manual_webhook_response', + stepStartedAt: new Date(), + attempt: 1, + }, + workflowMetadata: { + workflowName: 'webhookWorkflow', + workflowRunId: mockRunId, + workflowStartedAt: new Date(), + url: 'https://example.com', + features: { encryption: false }, + }, + ops, + preCompletionOps, + encryptionKey: undefined, + }; + + return { + hydrated, + ops, + preCompletionOps, + resetWorld: () => + vi + .mocked(getWorldLazy) + .mockImplementation(() => makeMockWorld() as any), + stepContext, + world, + }; + } + it('should throw error for an unsupported type', async () => { class Foo {} let err: WorkflowRuntimeError | undefined; @@ -2030,6 +2179,108 @@ describe('step arguments', () => { `Ensure you're passing workflow serializable types. Check the serialization docs to see what's serializable: https://workflow-sdk.dev/docs/foundations/serialization` ); }); + + it('waits for manual webhook response streams to flush before respondWith resolves', async () => { + const { contextStorage } = await import('./step/context-storage.js'); + + let resolveWrite!: () => void; + const writeGate = new Promise((resolve) => { + resolveWrite = resolve; + }); + const world = makeMockWorld(); + world.streams.write.mockImplementation((_runId, name) => { + if (name === webhookResponseStreamName) { + return writeGate; + } + return Promise.resolve(); + }); + const { hydrated, preCompletionOps, resetWorld, stepContext } = + await hydrateManualWebhookRequest(world); + + try { + // Let the writable lock poller observe the idle stream before the + // response is written. respondWith must still wait for this specific + // write/close, not for a stale idle-state promise. + await new Promise((resolve) => setTimeout(resolve, 30)); + + let responded = false; + const respondPromise = contextStorage + .run(stepContext, () => + hydrated.respondWith(new Response('Hello from webhook!')) + ) + .then(() => { + responded = true; + }); + + await vi.waitFor(() => { + expect(world.streams.write).toHaveBeenCalledWith( + mockRunId, + webhookResponseStreamName, + expect.any(Uint8Array) + ); + }); + + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(responded).toBe(false); + expect(preCompletionOps).toHaveLength(1); + + resolveWrite(); + await respondPromise; + await Promise.all(preCompletionOps); + + expect(responded).toBe(true); + expect(world.streams.close).toHaveBeenCalledWith( + mockRunId, + webhookResponseStreamName + ); + } finally { + resetWorld(); + } + }); + + it('rejects manual webhook respondWith when response stream write fails', async () => { + const { contextStorage } = await import('./step/context-storage.js'); + + const streamError = new Error('manual webhook response write failed'); + let rejectWrite!: (reason?: unknown) => void; + const writeFailure = new Promise((_resolve, reject) => { + rejectWrite = reject; + }); + writeFailure.catch(() => {}); + const world = makeMockWorld(); + world.streams.write.mockImplementation((_runId, name) => { + if (name === webhookResponseStreamName) { + return writeFailure; + } + return Promise.resolve(); + }); + const { hydrated, ops, preCompletionOps, resetWorld, stepContext } = + await hydrateManualWebhookRequest(world); + const opsDone = Promise.all(ops).catch(() => undefined); + + try { + const respondPromise = contextStorage.run(stepContext, () => + hydrated.respondWith(new Response('Hello from webhook!')) + ); + + await vi.waitFor(() => { + expect(world.streams.write).toHaveBeenCalledWith( + mockRunId, + webhookResponseStreamName, + expect.any(Uint8Array) + ); + }); + rejectWrite(streamError); + + await expect(respondPromise).rejects.toThrow(streamError.message); + + expect(preCompletionOps).toHaveLength(1); + await expect(Promise.all(preCompletionOps)).resolves.toEqual([undefined]); + await opsDone; + } finally { + resetWorld(); + } + }); }); describe('cross-VM Error serialization', () => { @@ -2667,7 +2918,7 @@ describe('step function serialization', () => { const stepId = 'step//workflows/test.ts//addNumbers'; // Create a VM context like the workflow runner does - const { context, globalThis: vmGlobalThis } = createContext({ + const { globalThis: vmGlobalThis } = createContext({ seed: 'test', fixedTimestamp: 1714857600000, }); @@ -2729,7 +2980,7 @@ describe('step function serialization', () => { const stepId = 'step//workflows/test.ts//missingUseStep'; // Create a VM context WITHOUT setting up WORKFLOW_USE_STEP - const { context, globalThis: vmGlobalThis } = createContext({ + const { globalThis: vmGlobalThis } = createContext({ seed: 'test', fixedTimestamp: 1714857600000, }); @@ -5561,7 +5812,7 @@ describe('isEncrypted', () => { // ============================================================================ describe('AbortController serialization', () => { - const { context, globalThis: vmGlobalThis } = createContext({ + const { globalThis: vmGlobalThis } = createContext({ seed: 'test-abort-serde', fixedTimestamp: 1714857600000, }); diff --git a/packages/core/src/serialization.ts b/packages/core/src/serialization.ts index 603bfa7120..6b5ee984d2 100644 --- a/packages/core/src/serialization.ts +++ b/packages/core/src/serialization.ts @@ -75,6 +75,8 @@ import { ABORT_STREAM_NAME, BODY_INIT_SYMBOL, STABLE_ULID, + STREAM_CLIENT_NAME_SYMBOL, + STREAM_FLUSH_PROMISE_SYMBOL, STREAM_FRAMING_SYMBOL, STREAM_NAME_SYMBOL, STREAM_SERVER_DEPLOYMENT_ID_SYMBOL, @@ -942,6 +944,17 @@ import type { SerializableSpecial, } from './serialization/types.js'; +function tagExternalReadableStream( + stream: T, + value: Extract +): T { + Object.defineProperty(stream, STREAM_CLIENT_NAME_SYMBOL, { + value: value.name, + writable: false, + }); + return stream; +} + // ---- Composed reducers ---- // Composes modular reducers (common, class, step-function) with // mode-specific Request/Response/Stream reducers below. @@ -1032,11 +1045,15 @@ type AbortInternals = { type AbortSignalLike = AbortInternals & { aborted: boolean; reason?: unknown; - addEventListener?: Function; + addEventListener?: AbortSignal['addEventListener']; }; type AbortHolder = AbortInternals & { signal?: AbortInternals }; +type FlushableWritableStream = WritableStream & { + [STREAM_FLUSH_PROMISE_SYMBOL]?: Promise; +}; + /** * Shared logic for AbortController/AbortSignal reducers in external and step * contexts. Assigns stream/hook names if not already present, optionally @@ -1963,7 +1980,7 @@ export function getExternalRevivers( // Start polling to detect when user releases lock pollReadableLock(userReadable, state); - return userReadable; + return tagExternalReadableStream(userReadable, value); } else { // Non-byte streams carry length-prefixed frames, so we can count // completed frames and transparently reconnect when the server @@ -1988,7 +2005,7 @@ export function getExternalRevivers( // Start polling to detect when user releases lock pollReadableLock(transform.readable, state); - return transform.readable; + return tagExternalReadableStream(transform.readable, value); } }, WritableStream: (value) => { @@ -2015,7 +2032,12 @@ export function getExternalRevivers( ops.push(state.promise); // Start the flushable pipe in the background - flushablePipe(serialize.readable, serverWritable, state).catch(() => { + const pipePromise = flushablePipe( + serialize.readable, + serverWritable, + state + ); + pipePromise.catch(() => { // Errors are handled via state.reject }); @@ -2040,6 +2062,10 @@ export function getExternalRevivers( } ); } + Object.defineProperty(serialize.writable, STREAM_FLUSH_PROMISE_SYMBOL, { + value: pipePromise, + writable: false, + }); return serialize.writable; }, @@ -2309,9 +2335,19 @@ function getStepRevivers( if (value.signal) copyAbortInternals(value.signal, request.signal); if (responseWritable) { request.respondWith = async (response: Response) => { - const writer = responseWritable.getWriter(); - await writer.write(response); - await writer.close(); + const ctx = contextStorage.getStore(); + const responseFlush = ( + responseWritable as FlushableWritableStream + )[STREAM_FLUSH_PROMISE_SYMBOL]; + const responseWrite = (async () => { + const writer = responseWritable.getWriter(); + writer.closed.catch(() => {}); + await writer.write(response); + await writer.close(); + await responseFlush; + })(); + ctx?.preCompletionOps.push(responseWrite.catch(() => {})); + await responseWrite; }; } return request; @@ -2418,7 +2454,12 @@ function getStepRevivers( ops.push(state.promise); // Start the flushable pipe in the background - flushablePipe(serialize.readable, serverWritable, state).catch(() => { + const pipePromise = flushablePipe( + serialize.readable, + serverWritable, + state + ); + pipePromise.catch(() => { // Errors are handled via state.reject }); @@ -2449,6 +2490,10 @@ function getStepRevivers( } ); } + Object.defineProperty(serialize.writable, STREAM_FLUSH_PROMISE_SYMBOL, { + value: pipePromise, + writable: false, + }); return serialize.writable; }, diff --git a/packages/core/src/symbols.ts b/packages/core/src/symbols.ts index a22712fbb3..c47cb0779f 100644 --- a/packages/core/src/symbols.ts +++ b/packages/core/src/symbols.ts @@ -8,6 +8,15 @@ export const STABLE_ULID = Symbol.for('WORKFLOW_STABLE_ULID'); export const STREAM_NAME_SYMBOL = Symbol.for('WORKFLOW_STREAM_NAME'); export const STREAM_TYPE_SYMBOL = Symbol.for('WORKFLOW_STREAM_TYPE'); export const STREAM_FRAMING_SYMBOL = Symbol.for('WORKFLOW_STREAM_FRAMING'); +/** + * Stamped on externally hydrated readable streams for diagnostics and retry + * helpers. Keep this distinct from `STREAM_NAME_SYMBOL`: reducers use that + * symbol as a durable-handle fast path, which would be incorrect for a client + * stream that may have already been consumed. + */ +export const STREAM_CLIENT_NAME_SYMBOL = Symbol.for( + 'WORKFLOW_STREAM_CLIENT_NAME' +); /** * Stamped on a real `WritableStream` (the user-visible `serialize.writable` * returned from a step-side reviver or step-context `getWritable()`) to @@ -34,6 +43,15 @@ export const STREAM_SERVER_RUN_ID_SYMBOL = Symbol.for( export const STREAM_SERVER_DEPLOYMENT_ID_SYMBOL = Symbol.for( 'WORKFLOW_STREAM_SERVER_DEPLOYMENT_ID' ); +/** + * Stamped on user-visible stream handles revived from server-backed writable + * descriptors. Awaiting this promise waits for that handle's background pipe + * to flush/close, without tying callers to unrelated stream ops in the same + * step. + */ +export const STREAM_FLUSH_PROMISE_SYMBOL = Symbol.for( + 'WORKFLOW_STREAM_FLUSH_PROMISE' +); export const BODY_INIT_SYMBOL = Symbol.for('BODY_INIT'); export const WEBHOOK_RESPONSE_WRITABLE = Symbol.for( 'WEBHOOK_RESPONSE_WRITABLE' diff --git a/packages/world-testing/src/inline-batches-debug.mts b/packages/world-testing/src/inline-batches-debug.mts index bf7a356cb3..903b30344d 100644 --- a/packages/world-testing/src/inline-batches-debug.mts +++ b/packages/world-testing/src/inline-batches-debug.mts @@ -1,5 +1,5 @@ -import { expect, test, vi } from 'vitest'; import { hydrateWorkflowReturnValue } from '@workflow/core/serialization'; +import { expect, test, vi } from 'vitest'; import { createFetcher, startServer } from './util.mjs'; /** @@ -13,7 +13,7 @@ import { createFetcher, startServer } from './util.mjs'; * - unconsumed-event skips * * Not part of the default suite; run with: - * pnpm vitest run packages/world-testing/test/inline-batches-debug.test.ts + * WORKFLOW_INLINE_BATCHES_DEBUG=1 pnpm vitest run packages/world-testing/test/inline-batches-debug.test.ts */ export function inlineBatchesDebug(world: string) { test( diff --git a/packages/world-testing/test/inline-batches-debug.test.ts b/packages/world-testing/test/inline-batches-debug.test.ts index 6cb5285105..6334747e98 100644 --- a/packages/world-testing/test/inline-batches-debug.test.ts +++ b/packages/world-testing/test/inline-batches-debug.test.ts @@ -1,3 +1,10 @@ +import { test } from 'vitest'; import { inlineBatchesDebug } from '../src/inline-batches-debug.mjs'; -inlineBatchesDebug('local'); +if (process.env.WORKFLOW_INLINE_BATCHES_DEBUG === '1') { + inlineBatchesDebug('local'); +} else { + test.skip( + 'three batches of five parallel steps — report V2 handler behavior' + ); +} diff --git a/workbench/example/workflows/97_bench.ts b/workbench/example/workflows/97_bench.ts index 3e1488ef31..6e2da54034 100644 --- a/workbench/example/workflows/97_bench.ts +++ b/workbench/example/workflows/97_bench.ts @@ -271,8 +271,14 @@ async function consumeAndVerifyStreams( // Return a summary stream so the bench harness can measure TTFB/slurp const encoder = new TextEncoder(); const summary = JSON.stringify({ totalBytes, streamCount: streams.length }); + let sent = false; return new ReadableStream({ - start(controller) { + pull(controller) { + if (sent) { + controller.close(); + return; + } + sent = true; controller.enqueue(encoder.encode(summary)); controller.close(); }, @@ -280,7 +286,6 @@ async function consumeAndVerifyStreams( } // Workflow 1: Pipeline — generate 1 stream, pipe through N XOR transform steps -// Returns the final transformed stream (same size as input) export async function streamPipelineWorkflow( steps: number, totalBytes: number @@ -290,7 +295,7 @@ export async function streamPipelineWorkflow( for (let i = 0; i < steps; i++) { stream = await transformStreamXor(stream, (i + 1) % 256); } - return stream; + return await consumeAndVerifyStreams(totalBytes, stream); } // Workflow 2: N steps generate streams in parallel, one step consumes + verifies all diff --git a/workbench/example/workflows/99_e2e.ts b/workbench/example/workflows/99_e2e.ts index 304578eebb..37e0efd65b 100644 --- a/workbench/example/workflows/99_e2e.ts +++ b/workbench/example/workflows/99_e2e.ts @@ -378,17 +378,21 @@ export async function outputStreamInsideStepWorkflow() { ////////////////////////////////////////////////////////// -async function stepWriteUtf8Text(writable: WritableStream, text: string) { +async function stepWriteUtf8Sequence(writable: WritableStream) { 'use step'; const writer = writable.getWriter(); - await writer.write(new TextEncoder().encode(text)); - writer.releaseLock(); -} - -async function stepWriteUtf8Json(writable: WritableStream, value: unknown) { - 'use step'; - const writer = writable.getWriter(); - await writer.write(new TextEncoder().encode(JSON.stringify(value))); + const encoder = new TextEncoder(); + for (const text of [ + 'Hello, world!', + 'Café — naïve résumé', + '你好,世界!🌍✨', + 'مرحبا بالعالم', + ]) { + await writer.write(encoder.encode(text)); + } + await writer.write( + encoder.encode(JSON.stringify({ greeting: '안녕하세요', emoji: '🎉' })) + ); writer.releaseLock(); } @@ -400,11 +404,7 @@ export async function utf8StreamWorkflow() { 'use workflow'; const writable = getWritable(); await sleep('1s'); - await stepWriteUtf8Text(writable, 'Hello, world!'); - await stepWriteUtf8Text(writable, 'Café — naïve résumé'); - await stepWriteUtf8Text(writable, '你好,世界!🌍✨'); - await stepWriteUtf8Text(writable, 'مرحبا بالعالم'); - await stepWriteUtf8Json(writable, { greeting: '안녕하세요', emoji: '🎉' }); + await stepWriteUtf8Sequence(writable); await stepCloseOutputStream(writable); return 'done'; } @@ -847,7 +847,7 @@ export async function hookSignalOwnerWorkflow(token: string, message: string) { export async function hookSupersedeOwnerWorkflow(token: string) { 'use workflow'; - for (let attempt = 0; attempt < 5; attempt++) { + for (let attempt = 0; attempt < 10; attempt++) { using hook = createHook<{ message: string }>({ token }); const conflict = await hook.getConflict(); @@ -861,6 +861,7 @@ export async function hookSupersedeOwnerWorkflow(token: string) { } await conflict.cancel(); + await sleep('1s'); } throw new Error(`Could not claim ${token} after cancelling the owner`);