/** * AI Software Factory — E2E: Webhooks exit scenario (TASK-0130) * * The acceptance-criteria exit scenario, end to end against the REAL stack * (web :3000 + api :4000 + worker + postgres/redis — no mocked clock, no * in-process worker): * * 1. The owner creates an API key (WRITE) AND a webhook in the Developers * hub UI; both raw secrets are read from their reveal dialogs (the only * place they appear). * 2. `curl`-style POST /pages with the API key → a local in-test HTTP * receiver gets the `page.created` delivery within 10 s carrying * `X-Factory-Signature`; the test RECOMPUTES * HMAC-SHA256(secret, ".") and asserts it equals `v1` * (signature verified in-test, exact header shape `t=…,v1=…`). * 3. The webhook URL is PATCHed to a dead loopback target * (`http://127.0.0.1:1/dead`) → a second page is created → the delivery * log (via the API) shows the SAME delivery row going PENDING → FAILED * with `attempts` incrementing 1 → 2 → 3 (visible retries) and * `nextRetryAt` gaps matching the configured backoff * (min(base·2^(n−1), max) ±20 % jitter — dev env runs a short * base/cap so the window is observable in seconds; the canonical * 30 s/15 min math is proven by the integration suite on a mocked * clock). * 4. The delivery inspector UI shows the failed delivery; its drawer * renders the payload JSON + the signature hint. * * DEV-ENV REQUIREMENT (recorded, C-13.8e): this spec needs * `WEBHOOK_ALLOW_PRIVATE=true` (loopback receiver + dead URL are private * targets — blocked by the SSRF guard otherwise) and short * `WEBHOOK_BACKOFF_BASE_MS`/`WEBHOOK_BACKOFF_MAX_MS` in the dev `.env`. * Both are dev-only overrides; production keeps the guard ON with the * 30 s/15 min defaults. * * NOTE: `@factory/shared` is not linked into `tests/e2e`, so the HMAC is * recomputed with `node:crypto` directly (same scheme: HMAC-SHA256 over * `"."`). */ import { test, expect } from "../fixtures"; import { request as pwRequest } from "@playwright/test"; import type { APIRequestContext } from "@playwright/test"; import { createHmac, timingSafeEqual } from "node:crypto"; import { createServer, type Server } from "node:http"; import type { AddressInfo } from "node:net"; const API = process.env.E2E_API_URL ?? "http://localhost:4000"; const DEAD_URL = "http://127.0.0.1:1/dead"; const PAGE1_TITLE = "hook probe 1 — signed"; const PAGE2_TITLE = "hook probe 2 — dead"; /** Dev-env backoff (`.env`): base 2 s, cap 16 s, ±20 % jitter. */ const BACKOFF_BASE_MS = 2000; interface Hit { rawBody: string; headers: Record; at: number; } /** In-test receiver: captures raw bodies + headers, answers 200. */ function startReceiver(): Promise<{ url: string; hits: Hit[]; stop: () => Promise; }> { const hits: Hit[] = []; const server: Server = createServer((req, res) => { let body = ""; req.on("data", (c) => (body += c)); req.on("end", () => { hits.push({ rawBody: body, headers: req.headers as Hit["headers"], at: Date.now() }); res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify({ ok: true })); }); }); return new Promise((resolve) => { server.listen(0, "127.0.0.1", () => { const { port } = server.address() as AddressInfo; resolve({ url: `http://127.0.0.1:${port}/hook`, hits, stop: () => new Promise((r) => { server.closeAllConnections?.(); server.close(() => r()); }), }); }); }); } /** Recompute the delivery signature the way a consumer would. */ function recomputeV1(secret: string, rawBody: string, t: number): string { return createHmac("sha256", secret).update(`${t}.${rawBody}`).digest("hex"); } /** Parse `X-Factory-Signature: t=,v1=` (canonical shape). */ function parseSignature(header: string | string[] | undefined): { t: number; v1: string } | null { if (!header || Array.isArray(header)) return null; const m = header.match(/t=(\d+)[;,] ?v1=([0-9a-f]{64})/); return m ? { t: Number(m[1]), v1: m[2] } : null; } /** A bare context for the API-key credential (the key IS the auth). */ async function bareCtx(): Promise { return pwRequest.newContext({ baseURL: API, timeout: 30_000 }); } /** * Click a button, immune to the Playwright 1.49 mobile-emulation hit-test * desync (see developers.spec.ts): on the chromium-mobile project Chromium * intermittently zooms the layout viewport out, so the CDP actionability * hit-test resolves the button center onto a different element. Fallback: * dispatch the element's real `click` event (no coordinates) — the same * event React's onClick is wired to — bypassing the broken hit-test. The * caller still asserts the full post-click state, so a fallback that did * nothing fails the test rather than masking a broken UI. */ async function safeClick( page: import("@playwright/test").Page, locator: import("@playwright/test").Locator, label: string, ): Promise { try { await locator.click(); return; } catch { /* actionability hit-test flake (mobile emulation) — fall through */ } await expect(locator, `${label}: resolvable for click fallback`).toBeAttached(); await locator.evaluate((el) => el.click()); } /** Same hit-test fallback for checkboxes. */ async function safeCheck( page: import("@playwright/test").Page, locator: import("@playwright/test").Locator, label: string, ): Promise { try { await locator.check(); return; } catch { /* fall through */ } await expect(locator, `${label}: resolvable for check fallback`).toBeAttached(); await locator.evaluate((el) => (el as HTMLInputElement).click()); } interface DeliveryRow { id: string; event: string; status: string; attempts: number; nextRetryAt: string | null; } /** Poll the delivery log until `pred` matches (the row is live). */ async function waitForDelivery( api: APIRequestContext, wid: string, hookId: string, pred: (row: DeliveryRow) => boolean, timeoutMs: number, ): Promise<{ row: DeliveryRow; at: number }> { const deadline = Date.now() + timeoutMs; for (;;) { const res = await api.get(`/api/v1/workspaces/${wid}/webhooks/${hookId}/deliveries`); expect(res.status()).toBe(200); const body = (await res.json()) as { data?: DeliveryRow[] }; const row = (body.data ?? []).find(pred); if (row) return { row, at: Date.now() }; if (Date.now() > deadline) { throw new Error( `delivery not observed within ${timeoutMs} ms; log: ${JSON.stringify(body.data ?? [], null, 0)}`, ); } await new Promise((r) => setTimeout(r, 300)); } } test.describe("Webhooks exit scenario (TASK-0130)", () => { test("signed delivery ≤10 s → in-test HMAC → dead URL → visible retries", async ({ api, authPage: page, freshWorkspace, }) => { test.setTimeout(120_000); // full UI + real worker roundtrip (≤10 s delivery + ~8 s retry window) const ws = await freshWorkspace(`HOOK E2E ${Date.now()}`); const receiver = await startReceiver(); try { // -- 1. Create the API key in the Developers hub UI ----------------- await page.goto(`/w/${ws.id}/settings/developers`); await expect(page.getByTestId("apikey-create-open")).toBeVisible({ timeout: 20_000 }); await safeClick(page, page.getByTestId("apikey-create-open"), "apikey-create-open"); await page.getByLabel("Name").fill("Hook E2E"); await safeCheck(page, page.getByRole("checkbox", { name: /WRITE/ }), "WRITE scope"); // + READ await safeClick(page, page.getByRole("button", { name: "Create key" }), "Create key"); const keyField = page.getByTestId("apikey-raw"); await expect(keyField).toBeVisible(); const rawKey = (await keyField.inputValue()).trim(); expect(rawKey).toMatch(/^fk_live_/); await safeCheck(page, page.getByTestId("apikey-stored-confirm"), "apikey-stored-confirm"); await safeClick(page, page.getByRole("button", { name: "I have stored it" }), "key stored"); await expect(page.getByTestId("apikey-raw")).toHaveCount(0); // -- 2. Create the webhook in the Webhooks tab (URL = receiver) ----- await safeClick(page, page.getByRole("tab", { name: "Webhooks" }), "Webhooks tab"); await page.getByPlaceholder("https://example.com/hook").fill(receiver.url); // `*` (every event) is pre-selected; no other chip changes needed. await safeClick(page, page.getByRole("button", { name: "Create webhook" }), "Create webhook"); const secretField = page.getByTestId("webhook-secret-raw"); await expect(secretField).toBeVisible({ timeout: 20_000 }); const secret = (await secretField.inputValue()).trim(); expect(secret).toMatch(/^whsec_[0-9a-f]{64}$/); // The reveal dialog's stored-secret gate. await safeCheck( page, page.getByRole("dialog").getByLabel("I have stored this secret securely."), "webhook stored gate", ); await safeClick( page, page.getByRole("dialog").getByRole("button", { name: "I have stored it" }), "secret stored", ); await expect(page.getByTestId("webhook-secret-raw")).toHaveCount(0); // The webhook row is visible in the UI (URL + masked prefix). const hookRow = page.getByRole("listitem").filter({ hasText: receiver.url }); await expect(hookRow).toBeVisible({ timeout: 20_000 }); // Deterministic id from the API (the UI row carries no id attribute). const list = await api.get(`/api/v1/workspaces/${ws.id}/webhooks`); expect(list.status()).toBe(200); const listBody = (await list.json()) as { data: Array<{ id: string; url: string }> }; const hook = listBody.data.find((h) => h.url === receiver.url); expect(hook, `webhook ${receiver.url} present in the API list`).toBeTruthy(); const hookId = hook!.id; // -- 3. POST /pages with the key → signed delivery within 10 s ------ const keyReq = await bareCtx(); const keyAuth = { Authorization: `Bearer ${rawKey}` }; const t0 = Date.now(); const created = await keyReq.post(`/api/v1/workspaces/${ws.id}/pages`, { headers: keyAuth, data: { title: PAGE1_TITLE }, }); expect(created.status(), await created.text()).toBeLessThan(300); const page1 = (await created.json()) as { id: string }; // The receiver gets it — within the 10 s acceptance budget. Match by // pageId (not just the first page.created) so a stray event can't mask // the assertion. const hit = await new Promise((resolve, reject) => { const check = () => { const found = receiver.hits.find((h) => { if (h.headers["x-factory-event"] !== "page.created") return false; try { const env = JSON.parse(h.rawBody) as { data?: { pageId?: string } }; return env.data?.pageId === page1.id; } catch { return false; } }); if (found) return resolve(found); if (Date.now() - t0 > 10_000) return reject(new Error(`no page.created delivery for ${page1.id} within 10 s`)); setTimeout(check, 200); }; check(); }); const deliveryLatencyMs = hit.at - t0; expect(deliveryLatencyMs, `delivery took ${deliveryLatencyMs} ms`).toBeLessThanOrEqual(10_000); // Signature verified IN-TEST: recompute HMAC-SHA256(secret, "."). const sig = parseSignature(hit.headers["x-factory-signature"]); expect(sig, `signature header: ${String(hit.headers["x-factory-signature"])}`).not.toBeNull(); expect(hit.headers["user-agent"]).toBe("factory-webhooks/1.0"); expect(hit.headers["content-type"]).toBe("application/json"); const expectedV1 = recomputeV1(secret, hit.rawBody, sig!.t); expect( timingSafeEqual(Buffer.from(expectedV1, "hex"), Buffer.from(sig!.v1, "hex")), `HMAC mismatch: expected ${expectedV1}, got ${sig!.v1}`, ).toBe(true); // Timestamp is recent (the delivery, not a replay). expect(Math.abs(Date.now() - sig!.t * 1000)).toBeLessThan(60_000); // Envelope: type + the created page in data. const envelope = JSON.parse(hit.rawBody) as { id: string; type: string; data: { pageId?: string; title?: string }; }; expect(envelope.type).toBe("page.created"); expect(envelope.id).toMatch(/^evt_/); expect(envelope.data?.pageId ?? envelope.data?.id).toBe(page1.id); expect(envelope.data?.title).toBe(PAGE1_TITLE); // -- 4. Dead URL → second page → visible retries in the log -------- const patched = await api.patch(`/api/v1/workspaces/${ws.id}/webhooks/${hookId}`, { data: { url: DEAD_URL }, }); expect(patched.status(), await patched.text()).toBe(200); const created2 = await keyReq.post(`/api/v1/workspaces/${ws.id}/pages`, { headers: keyAuth, data: { title: PAGE2_TITLE }, }); expect(created2.status(), await created2.text()).toBeLessThan(300); const page2 = (await created2.json()) as { id: string }; // The SAME delivery row retries in place: watch attempts 1 → 2 → 3. const seen = new Map(); // attempts -> first-observed time let row: DeliveryRow | undefined; let rowAt = 0; // when the final row state was observed const deadline = Date.now() + 45_000; for (;;) { const found = await waitForDelivery( api, ws.id, hookId, (r) => r.event === "page.created" && r.id !== "" && r.status === "FAILED", 1_500, ).catch(() => undefined); if (found) { row = found.row; rowAt = found.at; if (!seen.has(row.attempts)) seen.set(row.attempts, found.at); if (row.attempts >= 3 && row.nextRetryAt) break; } if (Date.now() > deadline) break; await new Promise((r) => setTimeout(r, 300)); } expect(row, "dead-URL delivery row in the log").toBeTruthy(); expect(row!.attempts, `attempts observed: ${row!.attempts}`).toBeGreaterThanOrEqual(3); expect(seen.get(1), "attempt 1 observed").toBeTruthy(); expect(seen.get(2), "attempt 2 observed").toBeTruthy(); expect(seen.get(3), "attempt 3 observed").toBeTruthy(); // Retries VISIBLE: the gaps between attempt transitions match the // configured backoff (dev: 2 s base, ±20 % jitter + scheduling/poll // slack ~1.5 s). A flat-30 s default or a missing retry would fail. const gap1 = seen.get(2)! - seen.get(1)!; // ≈ base·2^0 = 2 s const gap2 = seen.get(3)! - seen.get(2)!; // ≈ base·2^1 = 4 s expect(gap1, `gap 1→2: ${gap1} ms`).toBeGreaterThanOrEqual(BACKOFF_BASE_MS * 0.6 - 750); expect(gap1, `gap 1→2: ${gap1} ms`).toBeLessThanOrEqual(BACKOFF_BASE_MS * 1.2 + 1500); expect(gap2, `gap 2→3: ${gap2} ms`).toBeGreaterThanOrEqual(BACKOFF_BASE_MS * 2 * 0.6 - 750); expect(gap2, `gap 2→3: ${gap2} ms`).toBeLessThanOrEqual(BACKOFF_BASE_MS * 2 * 1.2 + 1500); // The schedule GROWS (exponential), it does not stay flat. expect(gap2, `gap 2→3 (${gap2} ms) should exceed gap 1→2 (${gap1} ms)`).toBeGreaterThan( gap1 - 400, ); // nextRetryAt after attempt 3 ≈ base·2^2 = 8 s in the future (measured // from the observation of that row state). const retryInMs = new Date(row!.nextRetryAt!).getTime() - rowAt; expect(retryInMs, `nextRetryAt in ${retryInMs} ms`).toBeGreaterThanOrEqual(8000 * 0.8 - 1000); expect(retryInMs, `nextRetryAt in ${retryInMs} ms`).toBeLessThanOrEqual(8000 * 1.2 + 1500); // -- 5. Delivery inspector UI: failed row + drawer ------------------ await safeClick(page, hookRow.getByRole("button", { name: "Deliveries" }), "Deliveries"); const failedRow = page .getByRole("row") .filter({ hasText: "page.created" }) .filter({ hasText: "FAILED" }); await expect(failedRow).toBeVisible({ timeout: 20_000 }); await safeClick(page, failedRow.getByRole("button", { name: "Open" }), "Open delivery"); const drawer = page.getByRole("dialog"); await expect(drawer).toBeVisible(); // Payload JSON (the dead page) + the signature verification hint. await expect(drawer.getByText(PAGE2_TITLE)).toBeVisible(); expect(await drawer.getByText(page2.id).count()).toBeGreaterThanOrEqual(1); await expect(drawer.getByText("X-Factory-Signature")).toBeVisible(); // .first(): "HMAC-SHA256" also appears in the explanatory caption. await expect(drawer.getByText("HMAC-SHA256").first()).toBeVisible(); await keyReq.dispose(); } finally { await receiver.stop(); } }); });