diff --git a/pkgs/website/astro.config.mjs b/pkgs/website/astro.config.mjs index 8f85d3bf7..7f0e31859 100644 --- a/pkgs/website/astro.config.mjs +++ b/pkgs/website/astro.config.mjs @@ -271,6 +271,10 @@ export default defineConfig({ }, ], }, + { + label: 'Conditional Steps', + autogenerate: { directory: 'build/conditional-steps/' }, + }, { label: 'Starting Flows', autogenerate: { directory: 'build/starting-flows/' }, diff --git a/pkgs/website/src/assets/pgflow-theme.d2 b/pkgs/website/src/assets/pgflow-theme.d2 index c4794e52b..65fc375b8 100644 --- a/pkgs/website/src/assets/pgflow-theme.d2 +++ b/pkgs/website/src/assets/pgflow-theme.d2 @@ -69,7 +69,7 @@ classes: { style.stroke: "#e85c5c" } - # Step state classes (created, started, completed, failed) + # Step state classes (created, started, completed, failed, skipped) step_created: { style.fill: "#95a0a3" style.stroke: "#4a5759" @@ -86,6 +86,11 @@ classes: { style.fill: "#a33636" style.stroke: "#e85c5c" } + step_skipped: { + style.fill: "#4a5759" + style.stroke: "#6b7a7d" + style.stroke-dash: 3 + } # Task state classes (queued, completed, failed) task_queued: { diff --git a/pkgs/website/src/content/docs/build/conditional-steps/error-handling.mdx b/pkgs/website/src/content/docs/build/conditional-steps/error-handling.mdx new file mode 100644 index 000000000..7969f3855 --- /dev/null +++ b/pkgs/website/src/content/docs/build/conditional-steps/error-handling.mdx @@ -0,0 +1,254 @@ +--- +title: Error Handling +description: Handle step failures gracefully with retriesExhausted option. +sidebar: + order: 3 +--- + +import { Aside, CardGrid, LinkCard } from '@astrojs/starlight/components'; + +The `retriesExhausted` option controls what happens when a step fails after exhausting all retry attempts. Instead of failing the entire run, you can mark the step as skipped and continue. + +## Default Behavior + +By default, when a step fails after all retries, the entire run fails: + +```typescript +.step({ + slug: 'sendNotification', + maxAttempts: 3, + // retriesExhausted: 'fail' // This is the default +}, async (input) => { + await sendEmail(input.run.email); // If this fails 3 times, run fails +}) +``` + +## Graceful Failure with skip + +Use `retriesExhausted: 'skip'` for non-critical steps that shouldn't block the workflow: + +```typescript +.step({ + slug: 'sendWelcomeEmail', + maxAttempts: 3, + retriesExhausted: 'skip', // If email fails, continue anyway +}, async (input) => { + return await sendEmail(input.run.email); +}) +.step({ + slug: 'createAccount', + dependsOn: ['sendWelcomeEmail'], +}, async (input) => { + // This runs even if email failed + // input.sendWelcomeEmail may be undefined + const emailSent = input.sendWelcomeEmail !== undefined; + return { accountId: await createUser(input.run), emailSent }; +}) +``` + + + +## Cascade Failure with skip-cascade + +Use `retriesExhausted: 'skip-cascade'` when a failed step should skip its entire dependency chain: + +```typescript +.step({ + slug: 'fetchExternalData', + maxAttempts: 5, + retriesExhausted: 'skip-cascade', // Skip this + dependents on failure +}, async (input) => { + return await fetchFromUnreliableAPI(input.run.id); +}) +.step({ + slug: 'processExternalData', + dependsOn: ['fetchExternalData'], +}, processHandler) // Also skipped if fetch fails +.step({ + slug: 'useLocalFallback', + // No dependency on external data +}, fallbackHandler) // This always runs +``` + +```d2 width="700" pad="20" +...@../../../../assets/pgflow-theme.d2 + +direction: right + +fetch: "fetchExternalData" +fetch.class: step_skipped + +process: "processExternalData" +process.class: step_skipped + +fallback: "useLocalFallback" +fallback.class: step_completed + +fetch -> process: "cascade skip" { + style.stroke-dash: 3 +} + +note: "fetch failed after 5 attempts\nprocess cascaded to skipped\nfallback runs independently" { + style.fill: transparent + style.stroke: transparent +} +``` + +## Type Safety + +Like `whenUnmet`, the `retriesExhausted` option affects TypeScript types: + +| Mode | Dependent Input Type | +| -------------- | --------------------------- | +| `fail` | `T` (required) | +| `skip` | `T \| undefined` (optional) | +| `skip-cascade` | `T` (required) | + +```typescript +// With retriesExhausted: 'skip' +.step({ + slug: 'optional', + retriesExhausted: 'skip', +}, () => ({ data: 'value' })) + +.step({ + slug: 'consumer', + dependsOn: ['optional'], +}, async (input) => { + // TypeScript knows input.optional may be undefined + if (input.optional) { + return processData(input.optional.data); + } + return processWithoutData(); +}) +``` + +## TYPE_VIOLATION: The Exception + +Some errors indicate programming mistakes that should never be silently ignored. These **TYPE_VIOLATION** errors always fail the run, regardless of `retriesExhausted`: + +```typescript +// This will ALWAYS fail the run, even with retriesExhausted: 'skip' +.step({ + slug: 'fetchItems', + retriesExhausted: 'skip', +}, () => "not an array") // Returns string instead of array! +.map({ + slug: 'processItems', + array: 'fetchItems', // Expects array, gets string +}, (item) => item * 2) +``` + + + +## Skip Reason for Failed Steps + +When `retriesExhausted: 'skip'` triggers, the step gets: + +- `status: 'skipped'` +- `skip_reason: 'handler_failed'` +- Original error preserved in `error_message` + +Query failed-but-skipped steps: + +```sql +SELECT step_slug, error_message, skip_reason +FROM pgflow.step_states +WHERE run_id = 'your-run-id' + AND skip_reason = 'handler_failed'; +``` + +## Combining with Conditions + +You can use both condition options and failure handling on the same step: + +```typescript +.step({ + slug: 'premiumNotification', + if: { plan: 'premium' }, // Only for premium users + whenUnmet: 'skip', // Skip if not premium + maxAttempts: 3, + retriesExhausted: 'skip', // Skip if notification fails +}, async (input) => { + return await sendPremiumEmail(input.run.email); +}) +``` + +The step can be skipped for two different reasons: + +1. `condition_unmet` - User isn't premium +2. `handler_failed` - Email service failed after 3 attempts + +## Best Practices + +### Do Use skip for Side Effects + +```typescript +// Good: Notifications shouldn't block core logic +.step({ + slug: 'sendSlackNotification', + maxAttempts: 2, + retriesExhausted: 'skip', +}, notifySlack) +``` + +### Don't Use skip for Critical Steps + +```typescript +// Bad: Payment processing should never silently fail +.step({ + slug: 'chargePayment', + maxAttempts: 3, + retriesExhausted: 'skip', // Don't do this! +}, chargeCard) +``` + +### Use skip-cascade for Feature Branches + +```typescript +// Good: If enrichment source is down, skip all enrichment +.step({ + slug: 'fetchEnrichmentData', + retriesExhausted: 'skip-cascade', +}, fetchEnrichment) +.step({ + slug: 'applyEnrichment', + dependsOn: ['fetchEnrichmentData'], +}, applyEnrichment) // Skipped if fetch failed +``` + +## Learn More + + + + + + diff --git a/pkgs/website/src/content/docs/build/conditional-steps/index.mdx b/pkgs/website/src/content/docs/build/conditional-steps/index.mdx new file mode 100644 index 000000000..23d3f7129 --- /dev/null +++ b/pkgs/website/src/content/docs/build/conditional-steps/index.mdx @@ -0,0 +1,117 @@ +--- +title: Conditional Steps +description: Control which steps execute based on input patterns and handle failures gracefully. +sidebar: + order: 27 +--- + +import { Aside, CardGrid, LinkCard } from '@astrojs/starlight/components'; + +pgflow supports conditional step execution based on input data patterns and configurable failure handling. This enables dynamic workflows that adapt to runtime conditions. + +## Two Systems, One Goal + +pgflow provides two orthogonal systems for controlling step execution: + +| System | When Evaluated | Purpose | +| ----------------------------------------- | ---------------- | ----------------------------------------------------- | +| **Pattern Matching** (`if`/`ifNot`) | Before step runs | Skip steps based on input values | +| **Failure Handling** (`retriesExhausted`) | After step fails | Handle failures gracefully instead of failing the run | + +Both systems share the same three behavior modes: `fail`, `skip`, and `skip-cascade`. + +## Behavior Modes + +When a condition is unmet or a step fails, you control what happens: + +| Mode | Behavior | +| -------------- | ----------------------------------------------------------------------------------------------- | +| `fail` | Step fails, entire run fails (default for `retriesExhausted`) | +| `skip` | Step marked as skipped, run continues, dependents receive `undefined` (default for `whenUnmet`) | +| `skip-cascade` | Step AND all downstream dependents skipped, run continues | + +## Quick Examples + +### Conditional Execution + +Run premium-only features based on input: + +```typescript +new Flow<{ userId: string; plan: 'free' | 'premium' }>({ + slug: 'userOnboarding', +}) + .step({ slug: 'createAccount' }, async (input) => { + return { accountId: await createUser(input.run.userId) }; + }) + .step( + { + slug: 'setupPremiumFeatures', + dependsOn: ['createAccount'], + if: { plan: 'premium' }, // Only run for premium users + whenUnmet: 'skip', // Skip (don't fail) for free users + }, + async (input) => { + return await enablePremium(input.createAccount.accountId); + } + ); +``` + +### Graceful Failure Handling + +Continue the workflow even if an optional step fails: + +```typescript +.step({ + slug: 'sendWelcomeEmail', + dependsOn: ['createAccount'], + maxAttempts: 3, + retriesExhausted: 'skip', // If email fails, continue anyway +}, async (input) => { + return await sendEmail(input.createAccount.accountId); +}) +``` + + + +## Type Safety + +pgflow's type system tracks which steps may be skipped: + +- **`skip` mode**: Dependent steps receive `T | undefined` - you must handle the missing case +- **`skip-cascade` mode**: Dependents are also skipped, so if they run, output is guaranteed + +```typescript +.step({ + slug: 'processResults', + dependsOn: ['optionalEnrichment'], +}, async (input) => { + // TypeScript knows this may be undefined + if (input.optionalEnrichment) { + return processWithEnrichment(input.optionalEnrichment); + } + return processBasic(input.run); +}) +``` + +## Learn More + + + + + + diff --git a/pkgs/website/src/content/docs/build/conditional-steps/pattern-matching.mdx b/pkgs/website/src/content/docs/build/conditional-steps/pattern-matching.mdx new file mode 100644 index 000000000..fe8f44a3f --- /dev/null +++ b/pkgs/website/src/content/docs/build/conditional-steps/pattern-matching.mdx @@ -0,0 +1,239 @@ +--- +title: Pattern Matching +description: Use if/ifNot conditions to control step execution based on input patterns. +sidebar: + order: 1 +--- + +import { Aside, Code } from '@astrojs/starlight/components'; + +Pattern matching lets you conditionally execute steps based on input data. pgflow uses PostgreSQL's `@>` JSON containment operator for pattern matching. + +## Basic Syntax + +Use `if` to run a step only when input contains a pattern: + +```typescript +.step({ + slug: 'premiumFeature', + if: { plan: 'premium' }, // Run if input contains plan: 'premium' + whenUnmet: 'skip', // Skip if condition not met +}, handler) +``` + +Use `ifNot` to run a step only when input does NOT contain a pattern: + +```typescript +.step({ + slug: 'trialReminder', + ifNot: { plan: 'premium' }, // Run if input does NOT contain plan: 'premium' + whenUnmet: 'skip', +}, handler) +``` + + + +## What Gets Checked + +The pattern is checked against different data depending on the step type: + +| Step Type | Pattern Checked Against | +| -------------------------- | ------------------------------------------------------------ | +| Root step (no `dependsOn`) | Flow input | +| Dependent step | Aggregated dependency outputs: `{ depSlug: depOutput, ... }` | + +### Root Step Example + +For root steps, the pattern matches against the flow input: + +```typescript +type Input = { userId: string; plan: 'free' | 'premium' }; + +new Flow({ slug: 'onboarding' }).step( + { + slug: 'setupPremium', + if: { plan: 'premium' }, // Checks flow input.plan + whenUnmet: 'skip', + }, + async (input) => { + // input.run.plan is guaranteed to be 'premium' here + return await enablePremiumFeatures(input.run.userId); + } +); +``` + +### Dependent Step Example + +For dependent steps, the pattern matches against an object containing all dependency outputs: + +```typescript +new Flow<{ url: string }>({ slug: 'contentPipeline' }) + .step({ slug: 'analyze' }, async (input) => { + const result = await analyzeContent(input.run.url); + return { needsModeration: result.flagged, content: result.text }; + }) + .step( + { + slug: 'moderate', + dependsOn: ['analyze'], + if: { analyze: { needsModeration: true } }, // Check analyze output + whenUnmet: 'skip', + }, + async (input) => { + return await moderateContent(input.analyze.content); + } + ); +``` + +The pattern `{ analyze: { needsModeration: true } }` matches the object `{ analyze: }`. + +## JSON Containment Semantics + +pgflow uses PostgreSQL's `@>` containment operator. Understanding its behavior helps write correct patterns. + +### Key Rules + +1. **Partial matching**: Your pattern only needs to include the fields you care about +2. **Nested matching**: Patterns can match nested objects recursively +3. **Array containment**: Array patterns check if elements exist (order doesn't matter) + +### Examples + + + +### Practical Pattern Examples + +```typescript +// Match a specific value +if: { status: 'active' } + +// Match nested object +if: { user: { verified: true } } + +// Match array containing element +if: { tags: ['priority'] } + +// Match multiple conditions (AND) +if: { status: 'active', type: 'premium' } +``` + +## Combining if and ifNot + +You can use both `if` and `ifNot` on the same step. Both conditions must be satisfied: + +```typescript +.step({ + slug: 'standardUserFeature', + if: { status: 'active' }, // Must be active + ifNot: { role: 'admin' }, // Must NOT be admin + whenUnmet: 'skip', +}, handler) +``` + +This step runs only for active non-admin users. + +## Branching Patterns + +Create mutually exclusive branches using opposite conditions: + +```typescript +new Flow<{ userType: 'individual' | 'business' }>({ slug: 'pricing' }) + .step( + { + slug: 'individualPricing', + if: { userType: 'individual' }, + whenUnmet: 'skip', + }, + calculateIndividualPrice + ) + .step( + { + slug: 'businessPricing', + if: { userType: 'business' }, + whenUnmet: 'skip', + }, + calculateBusinessPrice + ) + .step( + { + slug: 'finalize', + dependsOn: ['individualPricing', 'businessPricing'], + }, + async (input) => { + // Exactly one will have a value + const price = input.individualPricing ?? input.businessPricing; + return { finalPrice: price }; + } + ); +``` + + + +## Detecting Skipped Dependencies + +Use `ifNot` with an empty object to detect when a dependency was skipped: + +```typescript +.step({ + slug: 'primaryAction', + if: { someCondition: true }, + whenUnmet: 'skip', +}, primaryHandler) +.step({ + slug: 'fallbackAction', + dependsOn: ['primaryAction'], + ifNot: { primaryAction: {} }, // Run if primaryAction was skipped + whenUnmet: 'skip', +}, fallbackHandler) +``` + +The pattern `{ primaryAction: {} }` matches any non-null output. Using `ifNot` inverts it to match when the dependency is absent (skipped). + +## Type Safety + +pgflow's type system ensures your patterns match the actual input shape: + +```typescript +type Input = { plan: 'free' | 'premium'; count: number }; + +new Flow({ slug: 'example' }).step( + { + slug: 'test', + // TypeScript error: 'invalid' is not assignable to 'free' | 'premium' + if: { plan: 'invalid' }, + whenUnmet: 'skip', + }, + handler +); +``` + +For dependent steps, patterns are typed against the dependency output structure: + +```typescript +.step({ slug: 'analyze' }, () => ({ score: 85, passed: true })) +.step({ + slug: 'report', + dependsOn: ['analyze'], + // TypeScript knows 'analyze' output has 'score' and 'passed' + if: { analyze: { passed: true } }, + whenUnmet: 'skip', +}, handler) +``` diff --git a/pkgs/website/src/content/docs/build/conditional-steps/skip-modes.mdx b/pkgs/website/src/content/docs/build/conditional-steps/skip-modes.mdx new file mode 100644 index 000000000..1444f7a31 --- /dev/null +++ b/pkgs/website/src/content/docs/build/conditional-steps/skip-modes.mdx @@ -0,0 +1,323 @@ +--- +title: Skip Modes +description: Understand fail, skip, and skip-cascade behaviors when conditions aren't met. +sidebar: + order: 2 +--- + +import { Aside, Code } from '@astrojs/starlight/components'; + +When a step's condition isn't met (or a step fails after retries), the `whenUnmet` option controls what happens next. Understanding these modes is crucial for building resilient workflows. + +## The Three Modes + +| Mode | Step | Dependents | Run | +| -------------- | ------- | ------------------------ | --------- | +| `fail` | Fails | Not executed | Fails | +| `skip` | Skipped | Execute with `undefined` | Continues | +| `skip-cascade` | Skipped | All skipped | Continues | + +## fail Mode + +When a condition isn't met, the step fails and the entire run fails. + +```typescript +.step({ + slug: 'requirePremium', + if: { plan: 'premium' }, + whenUnmet: 'fail', // Explicit - must fail if not premium +}, handler) +``` + +**Use when:** The condition is a hard requirement. If not met, the workflow cannot proceed meaningfully. + +```d2 width="600" pad="20" +...@../../../../assets/pgflow-theme.d2 + +direction: right + +validate: "validate" +validate.class: step_completed + +premium: "requirePremium" +premium.class: step_failed + +notify: "notify" +notify.class: step_created + +validate -> premium +premium -> notify + +run: "Run: failed" { + style.stroke: "#a33636" + style.stroke-dash: 3 +} +``` + +## skip Mode + +The default behavior for `whenUnmet`. The step is marked as skipped, but the run continues. Downstream steps receive `undefined` for this dependency. + +```typescript +.step({ + slug: 'enrichData', + if: { includeEnrichment: true }, + // whenUnmet: 'skip' is the default, so this could be omitted +}, async (input) => { + return await fetchEnrichment(input.run.id); +}) +.step({ + slug: 'processResults', + dependsOn: ['enrichData'], +}, async (input) => { + // TypeScript knows enrichData may be undefined + if (input.enrichData) { + return processWithEnrichment(input.enrichData); + } + return processBasic(input.run); +}) +``` + +**Use when:** The step is optional and downstream steps can handle its absence. + +```d2 width="600" pad="20" +...@../../../../assets/pgflow-theme.d2 + +direction: right + +validate: "validate" +validate.class: step_completed + +enrich: "enrichData" +enrich.class: step_skipped + +process: "processResults" +process.class: step_completed + +validate -> enrich: "if: { includeEnrichment: true }" +validate -> process +enrich -> process: "undefined" { + style.stroke-dash: 3 +} + +run: "Run: completed" { + style.stroke: "#247056" + style.stroke-dash: 3 +} +``` + +### Type Safety with skip + +When a step uses `whenUnmet: 'skip'`, TypeScript makes its output optional for dependents: + +```typescript +// Step with skip mode +.step({ + slug: 'optional', + if: { feature: 'enabled' }, + whenUnmet: 'skip', +}, () => ({ data: 'enriched' })) + +// Dependent step - TypeScript enforces undefined check +.step({ + slug: 'consumer', + dependsOn: ['optional'], +}, async (input) => { + // Type is: { data: string } | undefined + const optionalData = input.optional; + + // TypeScript error if you don't handle undefined: + // optionalData.data // Error: optionalData is possibly undefined + + // Correct: + if (optionalData) { + return optionalData.data; + } + return 'default'; +}) +``` + + + +## skip-cascade Mode + +The step is skipped AND all downstream dependents are automatically skipped too. The run continues with whatever steps don't depend on the skipped step. + +```typescript +.step({ + slug: 'loadPremiumData', + if: { plan: 'premium' }, + whenUnmet: 'skip-cascade', // Skip this AND all dependents +}, loadPremiumHandler) +.step({ + slug: 'processPremiumData', + dependsOn: ['loadPremiumData'], +}, processHandler) // Also skipped if loadPremiumData skips +.step({ + slug: 'sendBasicReport', + // No dependency on premium steps - always runs +}, basicReportHandler) +``` + +**Use when:** A group of steps only makes sense together. If the first can't run, the rest shouldn't either. + +```d2 width="700" pad="20" +...@../../../../assets/pgflow-theme.d2 + +direction: right + +validate: "validate" +validate.class: step_completed + +load: "loadPremiumData" +load.class: step_skipped + +process: "processPremiumData" +process.class: step_skipped + +basic: "sendBasicReport" +basic.class: step_completed + +validate -> load: "if: { plan: 'premium' }" +validate -> basic +load -> process: "cascaded skip" { + style.stroke-dash: 3 +} + +run: "Run: completed" { + style.stroke: "#247056" + style.stroke-dash: 3 +} +``` + +### Type Safety with skip-cascade + +Unlike `skip`, the `skip-cascade` mode keeps dependency types **required** (not optional): + +```typescript +// Step with skip-cascade mode +.step({ + slug: 'loadData', + if: { enabled: true }, + whenUnmet: 'skip-cascade', +}, () => ({ items: [1, 2, 3] })) + +// Dependent step - output is guaranteed if this step runs +.step({ + slug: 'processData', + dependsOn: ['loadData'], +}, async (input) => { + // Type is: { items: number[] } - NOT optional! + // If loadData was skipped, processData is also skipped + // So if we're running, loadData definitely completed + return input.loadData.items.map(x => x * 2); +}) +``` + + + +## Choosing the Right Mode + +| Scenario | Recommended Mode | +| ------------------------------------------- | --------------------- | +| Hard requirement - can't proceed without it | `fail` | +| Optional enrichment - nice to have | `skip` | +| Feature flag - entire feature branch | `skip-cascade` | +| Premium features - all or nothing | `skip-cascade` | +| Fallback handling - try A, fallback to B | `skip` with detection | + +## Skip Reasons + +When a step is skipped, pgflow records why in the `skip_reason` field: + +| Skip Reason | Meaning | +| -------------------- | ------------------------------------------------- | +| `condition_unmet` | Step's `if` or `ifNot` condition wasn't satisfied | +| `dependency_skipped` | A dependency was skipped with `skip-cascade` | +| `handler_failed` | Handler failed with `retriesExhausted: 'skip'` | + +You can query these in the database: + +```sql +SELECT step_slug, status, skip_reason +FROM pgflow.step_states +WHERE run_id = 'your-run-id' + AND status = 'skipped'; +``` + +## Multi-Level Cascades + +Skip cascades propagate through the entire dependency chain: + +```typescript +new Flow({ slug: 'pipeline' }) + .step( + { + slug: 'step_a', + if: { enabled: true }, + whenUnmet: 'skip-cascade', + }, + handlerA + ) + .step( + { + slug: 'step_b', + dependsOn: ['step_a'], + }, + handlerB + ) + .step( + { + slug: 'step_c', + dependsOn: ['step_b'], + }, + handlerC + ); +// If step_a skips, step_b AND step_c are also skipped +``` + +```d2 width="600" pad="20" +...@../../../../assets/pgflow-theme.d2 + +direction: right + +a: "step_a" +a.class: step_skipped + +b: "step_b" +b.class: step_skipped + +c: "step_c" +c.class: step_skipped + +d: "step_d" +d.class: step_completed + +a -> b: "cascade" {style.stroke-dash: 3} +b -> c: "cascade" {style.stroke-dash: 3} + +note: "step_d has no dependency\non A/B/C, so it runs" { + style.fill: transparent + style.stroke: transparent +} +``` + +## Realtime Events + +pgflow broadcasts `step:skipped` events for each skipped step, which you can subscribe to: + +```typescript +const client = createPgflowClient(supabase); + +client.subscribe(runId, (event) => { + if (event.event_type === 'step:skipped') { + console.log(`Step ${event.step_slug} skipped: ${event.skip_reason}`); + } +}); +``` diff --git a/pkgs/website/src/content/docs/build/index.mdx b/pkgs/website/src/content/docs/build/index.mdx index c3d8a7f3e..f8c0ef582 100644 --- a/pkgs/website/src/content/docs/build/index.mdx +++ b/pkgs/website/src/content/docs/build/index.mdx @@ -25,6 +25,11 @@ Now that you've created your first flow, learn how to structure your code, integ href="/build/organize-flow-code/" description="Structure your flows and tasks for maintainability and reusability" /> + -For scheduling delays between steps, see [Delaying Steps](/build/delaying-steps/). + For scheduling delays between steps, see [Delaying + Steps](/build/delaying-steps/). For detailed information about each configuration option, see the [Step Execution Options](/reference/configuration/step-execution/) reference. @@ -22,12 +23,14 @@ Not all failures should retry. Understanding the difference helps you configure ### Transient Failures Temporary problems that might succeed on retry: + - Network timeouts - Rate limiting (429 responses) - Temporary service unavailability (503 responses) - Database connection issues Configure with retries: + ```typescript .step({ slug: 'fetchExternalData', @@ -39,12 +42,14 @@ Configure with retries: ### Permanent Failures Problems that will never succeed on retry: + - Invalid input format (malformed email, negative numbers) - Missing required fields - Business rule violations - Schema validation errors Configure without retries: + ```typescript .step({ slug: 'validInput', @@ -56,11 +61,21 @@ Configure without retries: ``` + + For detailed guidance on validation patterns, see [Validation Steps](/build/validation-steps/). @@ -78,25 +93,35 @@ When different steps have different reliability requirements: ```typescript new Flow({ slug: 'dataPipeline', - maxAttempts: 3, // Sensible defaults + maxAttempts: 3, // Sensible defaults baseDelay: 1, }) - .step({ - slug: 'validateInput', - maxAttempts: 1, // No retries - validation should not fail - }, validateHandler) - .step({ - slug: 'fetchExternal', - maxAttempts: 5, // External API might be flaky - baseDelay: 10, // Longer delays for external service - }, fetchHandler) - .step({ - slug: 'saveResults', - // Use flow defaults - }, saveHandler) + .step( + { + slug: 'validateInput', + maxAttempts: 1, // No retries - validation should not fail + }, + validateHandler + ) + .step( + { + slug: 'fetchExternal', + maxAttempts: 5, // External API might be flaky + baseDelay: 10, // Longer delays for external service + }, + fetchHandler + ) + .step( + { + slug: 'saveResults', + // Use flow defaults + }, + saveHandler + ); ``` **Why this approach:** + - Set reasonable flow-level defaults - Override only where needed - Validation steps need no retries (fail immediately on bad input) @@ -106,6 +131,11 @@ new Flow({ ## Learn More + -After deployment, you can update these settings without recompiling your flow. See [Tune Deployed Flows](/deploy/tune-flow-config/) for details. + After deployment, you can update these settings without recompiling your flow. + See [Tune Deployed Flows](/deploy/tune-flow-config/) for details. ## Default Configuration @@ -18,14 +19,15 @@ After deployment, you can update these settings without recompiling your flow. S ```typescript new Flow({ slug: 'myFlow', - maxAttempts: 3, // max retry attempts before marking as failed - baseDelay: 1, // initial retry delay in seconds - timeout: 60 // visibility timeout in seconds + maxAttempts: 3, // max retry attempts before marking as failed + baseDelay: 1, // initial retry delay in seconds + timeout: 60, // visibility timeout in seconds // Note: startDelay is step-level only, not available as a default at flow level -}) +}); ``` ## `maxAttempts` + **Type:** `number` **Default:** `3` @@ -35,11 +37,12 @@ The maximum number of times a task will be attempted before being marked as perm // Flow level new Flow({ slug: 'myFlow', maxAttempts: 5 }) -// Step level (overrides flow default) -.step({ slug: 'myStep', maxAttempts: 7 }, handler) + // Step level (overrides flow default) + .step({ slug: 'myStep', maxAttempts: 7 }, handler); ``` ## `baseDelay` + **Type:** `number` **Default:** `1` @@ -49,11 +52,12 @@ The initial delay (in seconds) before the first retry. pgflow uses exponential b // Flow level new Flow({ slug: 'myFlow', baseDelay: 2 }) -// Step level (overrides flow default) -.step({ slug: 'myStep', baseDelay: 10 }, handler) + // Step level (overrides flow default) + .step({ slug: 'myStep', baseDelay: 10 }, handler); ``` ## `timeout` + **Type:** `number` **Default:** `60` @@ -73,17 +77,19 @@ Set `timeout` higher than your task's maximum processing time. Currently, pgflow uses timeout only for visibility. In the future, the Edge Worker will also use it to terminate tasks that exceed their timeout. + ```ts // Flow level new Flow({ slug: 'myFlow', timeout: 120 }) -// Step level (overrides flow default) -.step({ slug: 'myStep', timeout: 300 }, handler) + // Step level (overrides flow default) + .step({ slug: 'myStep', timeout: 300 }, handler); ``` ## `startDelay` + **Type:** `number` **Default:** `0` @@ -112,20 +118,148 @@ Time 40: Step C starts (waits 10s after B completes) This results in 40+ seconds of delays, not the expected 10s. **Better alternatives:** + - **Need uniform delays?** Use a constant as shown below - **Rate limiting?** Use worker's `maxConcurrent` setting - **Debug delays?** Add only to specific steps you're debugging - **Compliance delays?** Make them explicit on relevant steps - + To apply the same delay to multiple steps, use a constant: ```typescript const RATE_LIMIT_DELAY = 2; flow - .step({ slug: "apiCall1", startDelay: RATE_LIMIT_DELAY }, handler1) - .step({ slug: "apiCall2", startDelay: RATE_LIMIT_DELAY }, handler2) + .step({ slug: 'apiCall1', startDelay: RATE_LIMIT_DELAY }, handler1) + .step({ slug: 'apiCall2', startDelay: RATE_LIMIT_DELAY }, handler2); ``` + + + +## Conditional Execution Options + +These options control which steps execute based on input patterns and how failures are handled. See [Conditional Steps](/build/conditional-steps/) for detailed explanations and examples. + +### `if` + +**Type:** `ContainmentPattern` +**Default:** Not applicable (must be explicitly set) + +Run the step only if input contains the specified pattern. pgflow uses PostgreSQL's `@>` containment operator for matching. + +```typescript +// Root step - checks flow input +.step({ + slug: 'premiumFeature', + if: { plan: 'premium' }, // Only run for premium users +}, handler) + +// Dependent step - checks dependency output +.step({ + slug: 'notify', + dependsOn: ['analyze'], + if: { analyze: { needsAlert: true } }, // Check analyze output +}, handler) +``` + + + +### `ifNot` + +**Type:** `ContainmentPattern` +**Default:** Not applicable (must be explicitly set) + +Run the step only if input does NOT contain the specified pattern. + +```typescript +.step({ + slug: 'standardUserFlow', + ifNot: { role: 'admin' }, // Skip admin users +}, handler) +``` + +You can combine `if` and `ifNot` - both conditions must be satisfied: + +```typescript +.step({ + slug: 'targetedNotification', + if: { status: 'active' }, // Must be active + ifNot: { role: 'admin' }, // AND must not be admin +}, handler) +``` + +### `whenUnmet` + +**Type:** `'fail' | 'skip' | 'skip-cascade'` +**Default:** `'skip'` + +Controls what happens when `if` or `ifNot` condition is not met. + +| Mode | Behavior | +| ---------------- | ------------------------------------------------------------------------------- | +| `'fail'` | Step fails, entire run fails | +| `'skip'` | Step marked as skipped, run continues, dependents receive `undefined` (default) | +| `'skip-cascade'` | Step AND all downstream dependents skipped, run continues | + +```typescript +.step({ + slug: 'enrichData', + if: { includeEnrichment: true }, + whenUnmet: 'skip', // Default - could be omitted +}, handler) + +.step({ + slug: 'criticalPath', + if: { plan: 'premium' }, + whenUnmet: 'fail', // Explicit - fail if not premium +}, handler) +``` + + + +### `retriesExhausted` + +**Type:** `'fail' | 'skip' | 'skip-cascade'` +**Default:** `'fail'` + +Controls what happens when a step fails after exhausting all `maxAttempts` retry attempts. + +| Mode | Behavior | +| ---------------- | --------------------------------------------------------------------- | +| `'fail'` | Step fails, entire run fails (default) | +| `'skip'` | Step marked as skipped, run continues, dependents receive `undefined` | +| `'skip-cascade'` | Step AND all downstream dependents skipped, run continues | + +```typescript +.step({ + slug: 'sendEmail', + maxAttempts: 3, + retriesExhausted: 'skip', // Don't fail run if email service is down +}, handler) + +.step({ + slug: 'criticalOperation', + maxAttempts: 5, + retriesExhausted: 'fail', // Default - fail if operation fails +}, handler) +``` + + + + ## Configuration Examples @@ -137,12 +271,12 @@ When all steps can use the same configuration: ```typescript new Flow({ slug: 'myFlow', - maxAttempts: 3, // Default for all steps - baseDelay: 1, // Default for all steps - timeout: 60 // Default for all steps + maxAttempts: 3, // Default for all steps + baseDelay: 1, // Default for all steps + timeout: 60, // Default for all steps }) - .step({ slug: 'step1' }, handler1) // Uses flow defaults - .step({ slug: 'step2' }, handler2) // Uses flow defaults + .step({ slug: 'step1' }, handler1) // Uses flow defaults + .step({ slug: 'step2' }, handler2); // Uses flow defaults ``` ### Mixed Configuration @@ -152,25 +286,34 @@ Override flow defaults for specific steps that need different behavior: ```typescript new Flow({ slug: 'analyzeData', - maxAttempts: 3, // Flow defaults + maxAttempts: 3, // Flow defaults baseDelay: 1, - timeout: 60 + timeout: 60, }) - .step({ - slug: 'fetchData', - // Uses all flow defaults - }, fetchHandler) - .step({ - slug: 'processData', - maxAttempts: 5, // Override: more retries - timeout: 300 // Override: needs more time - // baseDelay uses flow default (1) - }, processHandler) - .step({ - slug: 'callApi', - baseDelay: 10, // Override: longer initial delay - // maxAttempts and timeout use flow defaults - }, apiHandler) + .step( + { + slug: 'fetchData', + // Uses all flow defaults + }, + fetchHandler + ) + .step( + { + slug: 'processData', + maxAttempts: 5, // Override: more retries + timeout: 300, // Override: needs more time + // baseDelay uses flow default (1) + }, + processHandler + ) + .step( + { + slug: 'callApi', + baseDelay: 10, // Override: longer initial delay + // maxAttempts and timeout use flow defaults + }, + apiHandler + ); ``` ## Retry Behavior @@ -182,7 +325,11 @@ delay = baseDelay * 2^attemptCount ``` ### Retry Delay Examples @@ -190,18 +337,19 @@ Unlike [Background Jobs Mode](/get-started/faq/#what-are-the-two-edge-worker-mod Here's how retry delays grow with different base delays: | Attempt | Delay (baseDelay: 2s) | Delay (baseDelay: 5s) | Delay (baseDelay: 10s) | -|---------|----------------------|----------------------|------------------------| -| 1 | 2s | 5s | 10s | -| 2 | 4s | 10s | 20s | -| 3 | 8s | 20s | 40s | -| 4 | 16s | 40s | 80s | -| 5 | 32s | 80s | 160s | -| 6 | 64s | 160s | 320s | -| 7 | 128s | 320s | 640s | +| ------- | --------------------- | --------------------- | ---------------------- | +| 1 | 2s | 5s | 10s | +| 2 | 4s | 10s | 20s | +| 3 | 8s | 20s | 40s | +| 4 | 16s | 40s | 80s | +| 5 | 32s | 80s | 160s | +| 6 | 64s | 160s | 320s | +| 7 | 128s | 320s | 640s | ### When Tasks Fail Permanently A task is marked as permanently failed when: + - It has been attempted `maxAttempts` times - Each attempt resulted in an error - The task status changes from `queued` to `failed`