/** * TASK-0095 R1 — the outbox worker (the S12 gate's named unit test): * - per-resource FIFO ordering (a later mutation to the same resource never * replays before an earlier one), * - parallel resource chains (different resources interleave, bounded * concurrency — the pool caps the in-flight chains), * - the failed-list transitions (4xx → failed; 5xx / network → backoff * retry; 2xx incl. an idempotent hit → done), * - the 24 h GC (pending only — failed items need an explicit Discard), * - the 500-item cap (oldest evicted, surfaced via onEvicted), * - retry = a FRESH key (the old key is dead), * - the backoff curve (1 s · 2^attempts, capped at 30 s), * - drain re-entrancy + the clear-on-logout rule. * * Deterministic: an injected clock + a fake `send` (no real timers, no network * — AGENTS rule 25). The concurrency test flushes microtasks (not wall clock) * to observe the pool's in-flight peak. */ import { describe, expect, it } from "vitest"; import { OUTBOX_BACKOFF_MAX_MS, OUTBOX_GC_MS, OUTBOX_MAX_ITEMS, MemoryOutboxStore, OutboxQueue, backoffForAttempt, generateIdempotencyKey, resourceKey, } from "./outbox.js"; import type { OutboxItem, OutboxResult } from "./types.js"; // --- fakes ------------------------------------------------------------------- /** Flush a handful of microtask turns (lets the drain pool pull chains). */ async function flushMicrotasks(turns = 8): Promise { for (let i = 0; i < turns; i += 1) await Promise.resolve(); } /** A fake `send`: records call order + in-flight concurrency, optional hold. */ function fakeSend() { const records: string[] = []; let active = 0; let maxActive = 0; let hold: Promise | null = null; const responses = new Map(); const send = async (item: OutboxItem): Promise => { active += 1; maxActive = Math.max(maxActive, active); records.push(item.key); try { if (hold) await hold; return responses.get(item.key) ?? { ok: true, status: 200 }; } finally { active -= 1; } }; return { send, order: () => records, maxActive: () => maxActive, onKey: (key: string, result: OutboxResult) => responses.set(key, result), hold: (p: Promise) => { hold = p; }, }; } /** The queue's send options (a default send keeps the tests' focus on the * state machine; most tests inject their own via `fakeSend`). */ type QueueOverrides = { send?: (item: OutboxItem) => Promise; maxConcurrentResources?: number; onCountChange?: (info: { pending: number; failed: number }) => void; onEvicted?: (item: OutboxItem) => void; }; /** A queue over an in-memory store with a manually-advanced clock. */ function makeQueue(over: QueueOverrides = {}) { let nowMs = 1_000_000; const store = new MemoryOutboxStore(); const send = over.send ?? (async (): Promise => ({ ok: true, status: 200 })); const queue = new OutboxQueue({ store, send, now: () => nowMs, ...(over.maxConcurrentResources !== undefined ? { maxConcurrentResources: over.maxConcurrentResources } : {}), ...(over.onCountChange ? { onCountChange: over.onCountChange } : {}), ...(over.onEvicted ? { onEvicted: over.onEvicted } : {}), }); return { queue, store, advance: (ms: number) => { nowMs += ms; }, }; } /** A fresh pending item with a controlled field set (bypasses enqueue). */ function mkItem(over: Partial = {}): OutboxItem { return { key: over.key ?? generateIdempotencyKey(), method: over.method ?? "POST", path: over.path ?? "/api/v1/things", queuedAt: over.queuedAt ?? 0, resource: over.resource ?? { type: "thing", id: "t1" }, attempts: over.attempts ?? 0, ...(over.body !== undefined ? { body: over.body } : {}), ...(over.nextAttemptAt !== undefined ? { nextAttemptAt: over.nextAttemptAt } : {}), ...(over.lastError !== undefined ? { lastError: over.lastError } : {}), }; } /** Push items straight into the queue's pending/failed lists (test seam). */ function push(queue: OutboxQueue, list: "pending" | "failed", ...items: OutboxItem[]): void { const target = queue as unknown as { pending: OutboxItem[]; failed: OutboxItem[] }; for (const i of items) target[list].push(i); } // --- pure helpers --------------------------------------------------------------- describe("pure helpers", () => { it("generateIdempotencyKey: key_ + 32 hex, unique", () => { const a = generateIdempotencyKey(); const b = generateIdempotencyKey(); expect(a).toMatch(/^key_[0-9a-f]{32}$/); expect(b).toMatch(/^key_[0-9a-f]{32}$/); expect(a).not.toBe(b); }); it("resourceKey scopes the ordering chain per (type, id)", () => { expect(resourceKey({ type: "page", id: "p1" })).toBe("page:p1"); expect(resourceKey({ type: "page", id: "p1" })).not.toBe( resourceKey({ type: "page", id: "p2" }), ); }); it("backoffForAttempt: 1 s · 2^attempts, capped at 30 s", () => { expect(backoffForAttempt(0)).toBe(1_000); expect(backoffForAttempt(1)).toBe(2_000); expect(backoffForAttempt(2)).toBe(4_000); expect(backoffForAttempt(4)).toBe(16_000); expect(backoffForAttempt(5)).toBe(OUTBOX_BACKOFF_MAX_MS); // 32 s → capped expect(backoffForAttempt(-3)).toBe(1_000); // negative → clamped }); }); // --- enqueue + cap --------------------------------------------------------------- describe("enqueue", () => { it("creates a pending item with a fresh key + fires onCountChange", () => { const counts: { pending: number; failed: number }[] = []; const { queue } = makeQueue({ onCountChange: (c) => counts.push(c) }); const item = queue.enqueue("PATCH", "/api/v1/pages/p1", { title: "x" }, { type: "page", id: "p1", }); expect(item.key).toMatch(/^key_[0-9a-f]{32}$/); expect(item.attempts).toBe(0); expect(queue.count()).toBe(1); expect(queue.failedCount()).toBe(0); expect(counts.at(-1)).toEqual({ pending: 1, failed: 0 }); }); it("persists to the store on enqueue", () => { const { queue, store } = makeQueue(); queue.enqueue("POST", "/api/v1/things", undefined, { type: "thing", id: "t1" }); expect(store.items).toHaveLength(1); }); it("evicts the OLDEST pending item at the 500-cap (onEvicted)", () => { const evicted: OutboxItem[] = []; const { queue } = makeQueue({ onEvicted: (i) => evicted.push(i) }); for (let i = 0; i < OUTBOX_MAX_ITEMS; i += 1) { queue.enqueue("POST", "/api/v1/things", undefined, { type: "thing", id: `t${i}` }); } expect(queue.count()).toBe(OUTBOX_MAX_ITEMS); queue.enqueue("POST", "/api/v1/things", undefined, { type: "thing", id: "new" }); expect(queue.count()).toBe(OUTBOX_MAX_ITEMS); // capped, not +1 expect(evicted).toHaveLength(1); expect(evicted[0]?.resource.id).toBe("t0"); // the oldest is evicted }); }); // --- drain: ordering + concurrency ------------------------------------------------- describe("drain — ordering + concurrency", () => { it("replays a resource chain strictly FIFO (a later mutation never precedes an earlier one)", async () => { const fake = fakeSend(); const { queue } = makeQueue({ send: fake.send }); // Two mutations to the SAME resource (queued 1 before queued 2) + one to a // different resource (may interleave anywhere). const a1 = mkItem({ resource: { type: "page", id: "p1" }, queuedAt: 10 }); const a2 = mkItem({ resource: { type: "page", id: "p1" }, queuedAt: 20 }); const b1 = mkItem({ resource: { type: "record", id: "r1" }, queuedAt: 15 }); push(queue, "pending", a1, a2, b1); const res = await queue.drain(); expect(res.sent).toBe(3); expect(queue.count()).toBe(0); // Within the p1 chain: a1 strictly before a2. The record may be anywhere. const order = fake.order(); expect(order.indexOf(a1.key)).toBeGreaterThanOrEqual(0); expect(order.indexOf(a1.key)).toBeLessThan(order.indexOf(a2.key)); expect(order).toContain(b1.key); }); it("bounds parallel resource chains by maxConcurrentResources", async () => { const fake = fakeSend(); let resolveGate: () => void = () => undefined; const gate = new Promise((r) => { resolveGate = r; }); fake.hold(gate); const { queue } = makeQueue({ send: fake.send, maxConcurrentResources: 2, }); // 3 distinct resources (3 chains) with a held send → the pool (cap 2) // in-flights exactly 2 chains; the 3rd waits for a free worker. push( queue, "pending", mkItem({ resource: { type: "page", id: "p1" } }), mkItem({ resource: { type: "record", id: "r1" } }), mkItem({ resource: { type: "comment", id: "c1" } }), ); const drainP = queue.drain(); await flushMicrotasks(); // let the pool pull the chains into their held send expect(fake.maxActive(), "the pool caps the in-flight chains at the concurrency").toBe(2); resolveGate(); const res = await drainP; expect(res.sent).toBe(3); expect(queue.count()).toBe(0); expect(fake.maxActive(), "the peak never exceeded the cap").toBe(2); }); }); // --- drain: failure classification --------------------------------------------------- describe("drain — failure classification", () => { it("4xx → the failed list (a user decision: Retry/Discard)", async () => { const fake = fakeSend(); const { queue } = makeQueue({ send: fake.send }); const item = mkItem({ resource: { type: "page", id: "p1" } }); fake.onKey(item.key, { ok: false, status: 403, error: "Permission revoked" }); push(queue, "pending", item); const res = await queue.drain(); expect(res.failed).toBe(1); expect(res.sent).toBe(0); expect(queue.count()).toBe(0); // left pending expect(queue.failedCount()).toBe(1); const failed = queue.failedItems[0]; expect(failed?.key).toBe(item.key); expect(failed?.lastError).toBe("Permission revoked"); }); it("5xx → stays pending with attempts+1 + a backoff window (not re-sent early)", async () => { const fake = fakeSend(); const { queue, advance } = makeQueue({ send: fake.send }); const item = mkItem({ resource: { type: "page", id: "p1" } }); fake.onKey(item.key, { ok: false, status: 503, error: "upstream down" }); push(queue, "pending", item); const res = await queue.drain(); expect(res.retrying).toBe(1); expect(queue.count()).toBe(1); expect(queue.failedCount()).toBe(0); expect(queue.pendingItems[0]?.attempts).toBe(1); expect(queue.pendingItems[0]?.nextAttemptAt).toBeGreaterThan(0); // A drain BEFORE the backoff window closes does NOT re-send. const res2 = await queue.drain(); expect(res2.sent).toBe(0); expect(res2.retrying).toBe(1); expect(fake.order()).toHaveLength(1); // only the first send happened // Advance past the backoff → the item re-sends (same key — safe: the // server never executed, so the middleware checks before run). advance(OUTBOX_BACKOFF_MAX_MS + 5_000); fake.onKey(item.key, { ok: true, status: 200 }); const res3 = await queue.drain(); expect(res3.sent).toBe(1); expect(queue.count()).toBe(0); expect(fake.order()).toHaveLength(2); }); it("network failure (send throws) → retryable, like a 5xx", async () => { let calls = 0; const { queue, advance } = makeQueue({ send: async () => { calls += 1; if (calls === 1) throw new Error("socket closed"); return { ok: true, status: 200 }; }, }); push(queue, "pending", mkItem()); const res = await queue.drain(); expect(res.retrying).toBe(1); expect(queue.pendingItems[0]?.lastError).toBe("socket closed"); advance(OUTBOX_BACKOFF_MAX_MS + 5_000); const res2 = await queue.drain(); expect(res2.sent).toBe(1); }); it("2xx with idempotentHit → done, removed (the server replayed, no re-run)", async () => { const fake = fakeSend(); const { queue } = makeQueue({ send: fake.send }); const item = mkItem(); fake.onKey(item.key, { ok: true, status: 200, idempotentHit: true }); push(queue, "pending", item); const res = await queue.drain(); expect(res.sent).toBe(1); expect(queue.count()).toBe(0); }); it("is re-entrancy safe (a second concurrent drain does not re-send)", async () => { const fake = fakeSend(); const { queue } = makeQueue({ send: fake.send }); push(queue, "pending", mkItem()); const [r1, r2] = await Promise.all([queue.drain(), queue.drain()]); expect(r1.sent + r2.sent).toBe(1); // sent exactly once expect(fake.order()).toHaveLength(1); expect(queue.count()).toBe(0); }); }); // --- GC ------------------------------------------------------------------------------ describe("GC (24 h)", () => { it("drops pending items queued > 24 h ago (on gc() + before each enqueue)", () => { const { queue } = makeQueue(); const fresh = mkItem({ queuedAt: 1_000_000 }); // now const stale = mkItem({ queuedAt: 1_000_000 - (OUTBOX_GC_MS + 1) }); // 24 h + 1 ms push(queue, "pending", stale, fresh); queue.gc(); expect(queue.pendingItems.map((i) => i.key)).toEqual([fresh.key]); // enqueue GCs first (a day-old queue makes room). push(queue, "pending", mkItem({ queuedAt: 1_000_000 - (OUTBOX_GC_MS + 2) })); const added = queue.enqueue("POST", "/api/v1/x", undefined, { type: "x", id: "x" }); expect(queue.pendingItems.map((i) => i.key)).toEqual([fresh.key, added.key]); }); it("does NOT auto-GC failed items (they need an explicit Discard)", () => { const { queue } = makeQueue(); const oldFailed = mkItem({ queuedAt: 1_000_000 - OUTBOX_GC_MS * 2, lastError: "Permission revoked", }); push(queue, "failed", oldFailed); queue.gc(); expect(queue.failedCount()).toBe(1); expect(queue.failedItems[0]?.key).toBe(oldFailed.key); }); }); // --- retry / discard / clear ---------------------------------------------------------- describe("retry / discard / clear", () => { it("retry: a failed item re-enters pending with a FRESH key (the old key is dead)", async () => { const fake = fakeSend(); const { queue } = makeQueue({ send: fake.send }); const item = mkItem(); push(queue, "failed", { ...item, lastError: "Permission revoked", attempts: 2, nextAttemptAt: 9_999_999, }); const failedItem = queue.failedItems[0]; if (failedItem === undefined) throw new Error("expected one failed item"); const fresh = queue.retry(failedItem); expect(fresh.key).not.toBe(item.key); // fresh key expect(fresh.attempts).toBe(0); // reset expect(fresh.lastError).toBeUndefined(); expect(fresh.nextAttemptAt).toBeUndefined(); expect(queue.failedCount()).toBe(0); expect(queue.count()).toBe(1); // A drain sends the FRESH key (the middleware sees a new key → executes). const res = await queue.drain(); expect(res.sent).toBe(1); expect(fake.order()).toEqual([fresh.key]); }); it("retry of an unknown key is a no-op", () => { const { queue } = makeQueue(); const ghost = mkItem(); expect(queue.retry(ghost)).toBe(ghost); expect(queue.count()).toBe(0); }); it("discard: the user's explicit choice removes it (failed + pending, idempotent)", () => { const { queue } = makeQueue(); const f = mkItem({ lastError: "x" }); const p = mkItem(); push(queue, "failed", f); push(queue, "pending", p); queue.discard(f); queue.discard(p); expect(queue.failedCount()).toBe(0); expect(queue.count()).toBe(0); queue.discard(f); // a double discard is safe }); it("clear: logout abandons the outbox (pending + failed)", async () => { const { queue } = makeQueue(); push(queue, "pending", mkItem()); push(queue, "failed", mkItem({ lastError: "x" })); await queue.clear(); expect(queue.count()).toBe(0); expect(queue.failedCount()).toBe(0); }); }); // --- init (store load) ------------------------------------------------------------------ describe("init (persistence round-trip)", () => { it("loads persisted items, splitting pending vs failed by lastError", async () => { const store = new MemoryOutboxStore(); store.items = [mkItem({ queuedAt: 1 }), mkItem({ queuedAt: 2, lastError: "403" })]; const q = new OutboxQueue({ store, send: async () => ({ ok: true, status: 200 }), now: () => 1_000_000, }); await q.init(); expect(q.count()).toBe(1); expect(q.failedCount()).toBe(1); expect(q.failedItems[0]?.lastError).toBe("403"); }); });