/** * apps/worker — automation dead-letter + terminal bookkeeping (TASK-0113 §4/§8). * * `deadLetterRun` is the single place a run becomes DEAD_LETTER (retryCount * hit the cap): terminal run row, the frozen `automation.executed` domain * event (`status: "dead_letter"`), an in-app notification to every * Admin+Owner of the workspace, an `automation.dead_lettered` audit row * (system actor), and the `automations:dead_letter{workspace}` counter. * * `emitAutomationExecuted` is the §8 emitter: the frozen payload * `{ workspaceId, automationId, runId, status, triggerType, durationMs, * error? }` is emitted on EVERY terminal status, but ONLY at depth 0 — a * chained run (depth ≥ 1) does NOT re-emit, so an A→B→… chain produces * exactly one `automation.executed` (the root's), not a storm per hop. */ import type { PrismaClient } from "@prisma/client"; import type { Redis } from "ioredis"; import { asJson } from "../../lib/prisma.js"; import { getLogger } from "../../logger.js"; import { assertTransition } from "./run-state.js"; import { runLog } from "./log.js"; import type { WorkerEnv } from "../../config.js"; import type { AutomationRunStatus } from "@prisma/client"; const log = getLogger("automation-dlq"); export interface ExecutedEventInput { redis: Redis; workspaceId: string; automationId: string; runId: string; status: AutomationRunStatus; triggerType: string; durationMs: number; error?: { code: string }; /** Incoming loop depth (0 for human/API-originated). Only depth 0 emits. */ depth: number; } /** * Emit the frozen `automation.executed` event. No-op (returns false) when * depth ≥ 1 — chained runs don't re-emit (event-storm guard, §8). */ export async function emitAutomationExecuted(input: ExecutedEventInput): Promise { if (input.depth >= 1) return false; const payload: Record = { workspaceId: input.workspaceId, automationId: input.automationId, runId: input.runId, status: input.status, triggerType: input.triggerType, durationMs: input.durationMs }; if (input.error) payload.error = { code: input.error.code }; await input.redis.xadd( "factory:events", "*", "event", "automation.executed", "payload", JSON.stringify(payload), "ts", new Date().toISOString() ); return true; } /** * Mark a run DEAD_LETTER + all the §4 side effects. Idempotent-ish: only * valid from FAILED (the transition is asserted). Never throws — a DLQ * bookkeeping failure is a WARN (the run row is already terminal-safe). */ export async function deadLetterRun( prisma: PrismaClient, redis: Redis, env: WorkerEnv, args: { runId: string; workspaceId: string; automationId: string; automationName: string; fromStatus: AutomationRunStatus; code: string; message: string; actionIndex?: number; actionType?: string; attempts: number; triggerType: string; durationMs: number; depth: number; } ): Promise { try { assertTransition(args.fromStatus, "DEAD_LETTER"); const now = new Date(); await prisma.automationRun.update({ where: { id: args.runId }, data: { status: "DEAD_LETTER", finishedAt: now, error: asJson({ code: args.code, message: args.message, actionIndex: args.actionIndex ?? null, actionType: args.actionType ?? null, attempts: args.attempts }) } }); await runLog(prisma, args.runId, "error", "run dead-lettered", { code: args.code, message: args.message, attempts: args.attempts, actionIndex: args.actionIndex ?? null, actionType: args.actionType ?? null }); await emitAutomationExecuted({ redis, workspaceId: args.workspaceId, automationId: args.automationId, runId: args.runId, status: "DEAD_LETTER", triggerType: args.triggerType, durationMs: args.durationMs, error: { code: args.code }, depth: args.depth }); // Admin notification (in-app + email when SMTP is configured). const { notifyWorkspaceAdmins } = await import("../../lib/notify.js"); await notifyWorkspaceAdmins( prisma, redis, env.SMTP_HOST, args.workspaceId, `Automation "${args.automationName}" dead-lettered`, `Failed after ${args.attempts} attempts: ${args.code} — ${args.message}`, { type: "automation", id: args.automationId }, "AUTOMATION" ); // Audit (system actor). try { await prisma.auditLog.create({ data: { actorType: "SYSTEM", actorId: `automation:${args.automationId}`, workspaceId: args.workspaceId, resourceType: "automation_run", resourceId: args.runId, action: "automation.dead_lettered", metadata: asJson({ automationId: args.automationId, runId: args.runId, code: args.code, attempts: args.attempts }) } }); } catch (err) { log.warn({ err: (err as Error).message, runId: args.runId }, "dead-letter audit write failed"); } // Metric counter (lightweight — no prometheus infra in the worker). await redis.incr(`automations:dead_letter:${args.workspaceId}`).catch(() => undefined); log.error( { runId: args.runId, automationId: args.automationId, code: args.code, attempts: args.attempts }, "automation run → DEAD_LETTER" ); } catch (err) { log.error({ err: (err as Error).message, runId: args.runId }, "dead-letter bookkeeping failed"); } }