webhooks.spec.ts

webhooks.spec.tstests/e2e/specs/webhooks.spec.tsraw ← specs17,118 B← Back to the app
1/**
2 * AI Software Factory — E2E: Webhooks exit scenario (TASK-0130)
3 *
4 * The acceptance-criteria exit scenario, end to end against the REAL stack
5 * (web :3000 + api :4000 + worker + postgres/redis — no mocked clock, no
6 * in-process worker):
7 *
8 *   1. The owner creates an API key (WRITE) AND a webhook in the Developers
9 *      hub UI; both raw secrets are read from their reveal dialogs (the only
10 *      place they appear).
11 *   2. `curl`-style POST /pages with the API key → a local in-test HTTP
12 *      receiver gets the `page.created` delivery within 10 s carrying
13 *      `X-Factory-Signature`; the test RECOMPUTES
14 *      HMAC-SHA256(secret, "<t>.<rawBody>") and asserts it equals `v1`
15 *      (signature verified in-test, exact header shape `t=…,v1=…`).
16 *   3. The webhook URL is PATCHed to a dead loopback target
17 *      (`http://127.0.0.1:1/dead`) → a second page is created → the delivery
18 *      log (via the API) shows the SAME delivery row going PENDING → FAILED
19 *      with `attempts` incrementing 1 → 2 → 3 (visible retries) and
20 *      `nextRetryAt` gaps matching the configured backoff
21 *      (min(base·2^(n−1), max) ±20 % jitter — dev env runs a short
22 *      base/cap so the window is observable in seconds; the canonical
23 *      30 s/15 min math is proven by the integration suite on a mocked
24 *      clock).
25 *   4. The delivery inspector UI shows the failed delivery; its drawer
26 *      renders the payload JSON + the signature hint.
27 *
28 * DEV-ENV REQUIREMENT (recorded, C-13.8e): this spec needs
29 * `WEBHOOK_ALLOW_PRIVATE=true` (loopback receiver + dead URL are private
30 * targets — blocked by the SSRF guard otherwise) and short
31 * `WEBHOOK_BACKOFF_BASE_MS`/`WEBHOOK_BACKOFF_MAX_MS` in the dev `.env`.
32 * Both are dev-only overrides; production keeps the guard ON with the
33 * 30 s/15 min defaults.
34 *
35 * NOTE: `@factory/shared` is not linked into `tests/e2e`, so the HMAC is
36 * recomputed with `node:crypto` directly (same scheme: HMAC-SHA256 over
37 * `"<t>.<rawBody>"`).
38 */
39import { test, expect } from "../fixtures";
40import { request as pwRequest } from "@playwright/test";
41import type { APIRequestContext } from "@playwright/test";
42import { createHmac, timingSafeEqual } from "node:crypto";
43import { createServer, type Server } from "node:http";
44import type { AddressInfo } from "node:net";
45 
46const API = process.env.E2E_API_URL ?? "http://localhost:4000";
47const DEAD_URL = "http://127.0.0.1:1/dead";
48const PAGE1_TITLE = "hook probe 1 — signed";
49const PAGE2_TITLE = "hook probe 2 — dead";
50 
51/** Dev-env backoff (`.env`): base 2 s, cap 16 s, ±20 % jitter. */
52const BACKOFF_BASE_MS = 2000;
53 
54interface Hit {
55  rawBody: string;
56  headers: Record<string, string | string[] | undefined>;
57  at: number;
58}
59 
60/** In-test receiver: captures raw bodies + headers, answers 200. */
61function startReceiver(): Promise<{
62  url: string;
63  hits: Hit[];
64  stop: () => Promise<void>;
65}> {
66  const hits: Hit[] = [];
67  const server: Server = createServer((req, res) => {
68    let body = "";
69    req.on("data", (c) => (body += c));
70    req.on("end", () => {
71      hits.push({ rawBody: body, headers: req.headers as Hit["headers"], at: Date.now() });
72      res.writeHead(200, { "content-type": "application/json" });
73      res.end(JSON.stringify({ ok: true }));
74    });
75  });
76  return new Promise((resolve) => {
77    server.listen(0, "127.0.0.1", () => {
78      const { port } = server.address() as AddressInfo;
79      resolve({
80        url: `http://127.0.0.1:${port}/hook`,
81        hits,
82        stop: () =>
83          new Promise<void>((r) => {
84            server.closeAllConnections?.();
85            server.close(() => r());
86          }),
87      });
88    });
89  });
90}
91 
92/** Recompute the delivery signature the way a consumer would. */
93function recomputeV1(secret: string, rawBody: string, t: number): string {
94  return createHmac("sha256", secret).update(`${t}.${rawBody}`).digest("hex");
95}
96 
97/** Parse `X-Factory-Signature: t=<ts>,v1=<hex>` (canonical shape). */
98function parseSignature(header: string | string[] | undefined): { t: number; v1: string } | null {
99  if (!header || Array.isArray(header)) return null;
100  const m = header.match(/t=(\d+)[;,] ?v1=([0-9a-f]{64})/);
101  return m ? { t: Number(m[1]), v1: m[2] } : null;
102}
103 
104/** A bare context for the API-key credential (the key IS the auth). */
105async function bareCtx(): Promise<APIRequestContext> {
106  return pwRequest.newContext({ baseURL: API, timeout: 30_000 });
107}
108 
109/**
110 * Click a button, immune to the Playwright 1.49 mobile-emulation hit-test
111 * desync (see developers.spec.ts): on the chromium-mobile project Chromium
112 * intermittently zooms the layout viewport out, so the CDP actionability
113 * hit-test resolves the button center onto a different element. Fallback:
114 * dispatch the element's real `click` event (no coordinates) — the same
115 * event React's onClick is wired to — bypassing the broken hit-test. The
116 * caller still asserts the full post-click state, so a fallback that did
117 * nothing fails the test rather than masking a broken UI.
118 */
119async function safeClick(
120  page: import("@playwright/test").Page,
121  locator: import("@playwright/test").Locator,
122  label: string,
123): Promise<void> {
124  try {
125    await locator.click();
126    return;
127  } catch {
128    /* actionability hit-test flake (mobile emulation) — fall through */
129  }
130  await expect(locator, `${label}: resolvable for click fallback`).toBeAttached();
131  await locator.evaluate((el) => el.click());
132}
133 
134/** Same hit-test fallback for checkboxes. */
135async function safeCheck(
136  page: import("@playwright/test").Page,
137  locator: import("@playwright/test").Locator,
138  label: string,
139): Promise<void> {
140  try {
141    await locator.check();
142    return;
143  } catch {
144    /* fall through */
145  }
146  await expect(locator, `${label}: resolvable for check fallback`).toBeAttached();
147  await locator.evaluate((el) => (el as HTMLInputElement).click());
148}
149 
150interface DeliveryRow {
151  id: string;
152  event: string;
153  status: string;
154  attempts: number;
155  nextRetryAt: string | null;
156}
157 
158/** Poll the delivery log until `pred` matches (the row is live). */
159async function waitForDelivery(
160  api: APIRequestContext,
161  wid: string,
162  hookId: string,
163  pred: (row: DeliveryRow) => boolean,
164  timeoutMs: number,
165): Promise<{ row: DeliveryRow; at: number }> {
166  const deadline = Date.now() + timeoutMs;
167  for (;;) {
168    const res = await api.get(`/api/v1/workspaces/${wid}/webhooks/${hookId}/deliveries`);
169    expect(res.status()).toBe(200);
170    const body = (await res.json()) as { data?: DeliveryRow[] };
171    const row = (body.data ?? []).find(pred);
172    if (row) return { row, at: Date.now() };
173    if (Date.now() > deadline) {
174      throw new Error(
175        `delivery not observed within ${timeoutMs} ms; log: ${JSON.stringify(body.data ?? [], null, 0)}`,
176      );
177    }
178    await new Promise((r) => setTimeout(r, 300));
179  }
180}
181 
182test.describe("Webhooks exit scenario (TASK-0130)", () => {
183  test("signed delivery ≤10 s → in-test HMAC → dead URL → visible retries", async ({
184    api,
185    authPage: page,
186    freshWorkspace,
187  }) => {
188    test.setTimeout(120_000); // full UI + real worker roundtrip (≤10 s delivery + ~8 s retry window)
189 
190    const ws = await freshWorkspace(`HOOK E2E ${Date.now()}`);
191    const receiver = await startReceiver();
192    try {
193      // -- 1. Create the API key in the Developers hub UI -----------------
194      await page.goto(`/w/${ws.id}/settings/developers`);
195      await expect(page.getByTestId("apikey-create-open")).toBeVisible({ timeout: 20_000 });
196      await safeClick(page, page.getByTestId("apikey-create-open"), "apikey-create-open");
197      await page.getByLabel("Name").fill("Hook E2E");
198      await safeCheck(page, page.getByRole("checkbox", { name: /WRITE/ }), "WRITE scope"); // + READ
199      await safeClick(page, page.getByRole("button", { name: "Create key" }), "Create key");
200      const keyField = page.getByTestId("apikey-raw");
201      await expect(keyField).toBeVisible();
202      const rawKey = (await keyField.inputValue()).trim();
203      expect(rawKey).toMatch(/^fk_live_/);
204      await safeCheck(page, page.getByTestId("apikey-stored-confirm"), "apikey-stored-confirm");
205      await safeClick(page, page.getByRole("button", { name: "I have stored it" }), "key stored");
206      await expect(page.getByTestId("apikey-raw")).toHaveCount(0);
207 
208      // -- 2. Create the webhook in the Webhooks tab (URL = receiver) -----
209      await safeClick(page, page.getByRole("tab", { name: "Webhooks" }), "Webhooks tab");
210      await page.getByPlaceholder("https://example.com/hook").fill(receiver.url);
211      // `*` (every event) is pre-selected; no other chip changes needed.
212      await safeClick(page, page.getByRole("button", { name: "Create webhook" }), "Create webhook");
213      const secretField = page.getByTestId("webhook-secret-raw");
214      await expect(secretField).toBeVisible({ timeout: 20_000 });
215      const secret = (await secretField.inputValue()).trim();
216      expect(secret).toMatch(/^whsec_[0-9a-f]{64}$/);
217      // The reveal dialog's stored-secret gate.
218      await safeCheck(
219        page,
220        page.getByRole("dialog").getByLabel("I have stored this secret securely."),
221        "webhook stored gate",
222      );
223      await safeClick(
224        page,
225        page.getByRole("dialog").getByRole("button", { name: "I have stored it" }),
226        "secret stored",
227      );
228      await expect(page.getByTestId("webhook-secret-raw")).toHaveCount(0);
229 
230      // The webhook row is visible in the UI (URL + masked prefix).
231      const hookRow = page.getByRole("listitem").filter({ hasText: receiver.url });
232      await expect(hookRow).toBeVisible({ timeout: 20_000 });
233 
234      // Deterministic id from the API (the UI row carries no id attribute).
235      const list = await api.get(`/api/v1/workspaces/${ws.id}/webhooks`);
236      expect(list.status()).toBe(200);
237      const listBody = (await list.json()) as { data: Array<{ id: string; url: string }> };
238      const hook = listBody.data.find((h) => h.url === receiver.url);
239      expect(hook, `webhook ${receiver.url} present in the API list`).toBeTruthy();
240      const hookId = hook!.id;
241 
242      // -- 3. POST /pages with the key → signed delivery within 10 s ------
243      const keyReq = await bareCtx();
244      const keyAuth = { Authorization: `Bearer ${rawKey}` };
245      const t0 = Date.now();
246      const created = await keyReq.post(`/api/v1/workspaces/${ws.id}/pages`, {
247        headers: keyAuth,
248        data: { title: PAGE1_TITLE },
249      });
250      expect(created.status(), await created.text()).toBeLessThan(300);
251      const page1 = (await created.json()) as { id: string };
252 
253      // The receiver gets it — within the 10 s acceptance budget. Match by
254      // pageId (not just the first page.created) so a stray event can't mask
255      // the assertion.
256      const hit = await new Promise<Hit>((resolve, reject) => {
257        const check = () => {
258          const found = receiver.hits.find((h) => {
259            if (h.headers["x-factory-event"] !== "page.created") return false;
260            try {
261              const env = JSON.parse(h.rawBody) as { data?: { pageId?: string } };
262              return env.data?.pageId === page1.id;
263            } catch {
264              return false;
265            }
266          });
267          if (found) return resolve(found);
268          if (Date.now() - t0 > 10_000)
269            return reject(new Error(`no page.created delivery for ${page1.id} within 10 s`));
270          setTimeout(check, 200);
271        };
272        check();
273      });
274      const deliveryLatencyMs = hit.at - t0;
275      expect(deliveryLatencyMs, `delivery took ${deliveryLatencyMs} ms`).toBeLessThanOrEqual(10_000);
276 
277      // Signature verified IN-TEST: recompute HMAC-SHA256(secret, "<t>.<body>").
278      const sig = parseSignature(hit.headers["x-factory-signature"]);
279      expect(sig, `signature header: ${String(hit.headers["x-factory-signature"])}`).not.toBeNull();
280      expect(hit.headers["user-agent"]).toBe("factory-webhooks/1.0");
281      expect(hit.headers["content-type"]).toBe("application/json");
282      const expectedV1 = recomputeV1(secret, hit.rawBody, sig!.t);
283      expect(
284        timingSafeEqual(Buffer.from(expectedV1, "hex"), Buffer.from(sig!.v1, "hex")),
285        `HMAC mismatch: expected ${expectedV1}, got ${sig!.v1}`,
286      ).toBe(true);
287      // Timestamp is recent (the delivery, not a replay).
288      expect(Math.abs(Date.now() - sig!.t * 1000)).toBeLessThan(60_000);
289 
290      // Envelope: type + the created page in data.
291      const envelope = JSON.parse(hit.rawBody) as {
292        id: string;
293        type: string;
294        data: { pageId?: string; title?: string };
295      };
296      expect(envelope.type).toBe("page.created");
297      expect(envelope.id).toMatch(/^evt_/);
298      expect(envelope.data?.pageId ?? envelope.data?.id).toBe(page1.id);
299      expect(envelope.data?.title).toBe(PAGE1_TITLE);
300 
301      // -- 4. Dead URL → second page → visible retries in the log --------
302      const patched = await api.patch(`/api/v1/workspaces/${ws.id}/webhooks/${hookId}`, {
303        data: { url: DEAD_URL },
304      });
305      expect(patched.status(), await patched.text()).toBe(200);
306 
307      const created2 = await keyReq.post(`/api/v1/workspaces/${ws.id}/pages`, {
308        headers: keyAuth,
309        data: { title: PAGE2_TITLE },
310      });
311      expect(created2.status(), await created2.text()).toBeLessThan(300);
312      const page2 = (await created2.json()) as { id: string };
313 
314      // The SAME delivery row retries in place: watch attempts 1 → 2 → 3.
315      const seen = new Map<number, number>(); // attempts -> first-observed time
316      let row: DeliveryRow | undefined;
317      let rowAt = 0; // when the final row state was observed
318      const deadline = Date.now() + 45_000;
319      for (;;) {
320        const found = await waitForDelivery(
321          api,
322          ws.id,
323          hookId,
324          (r) => r.event === "page.created" && r.id !== "" && r.status === "FAILED",
325          1_500,
326        ).catch(() => undefined);
327        if (found) {
328          row = found.row;
329          rowAt = found.at;
330          if (!seen.has(row.attempts)) seen.set(row.attempts, found.at);
331          if (row.attempts >= 3 && row.nextRetryAt) break;
332        }
333        if (Date.now() > deadline) break;
334        await new Promise((r) => setTimeout(r, 300));
335      }
336      expect(row, "dead-URL delivery row in the log").toBeTruthy();
337      expect(row!.attempts, `attempts observed: ${row!.attempts}`).toBeGreaterThanOrEqual(3);
338      expect(seen.get(1), "attempt 1 observed").toBeTruthy();
339      expect(seen.get(2), "attempt 2 observed").toBeTruthy();
340      expect(seen.get(3), "attempt 3 observed").toBeTruthy();
341 
342      // Retries VISIBLE: the gaps between attempt transitions match the
343      // configured backoff (dev: 2 s base, ±20 % jitter + scheduling/poll
344      // slack ~1.5 s). A flat-30 s default or a missing retry would fail.
345      const gap1 = seen.get(2)! - seen.get(1)!; // ≈ base·2^0 = 2 s
346      const gap2 = seen.get(3)! - seen.get(2)!; // ≈ base·2^1 = 4 s
347      expect(gap1, `gap 1→2: ${gap1} ms`).toBeGreaterThanOrEqual(BACKOFF_BASE_MS * 0.6 - 750);
348      expect(gap1, `gap 1→2: ${gap1} ms`).toBeLessThanOrEqual(BACKOFF_BASE_MS * 1.2 + 1500);
349      expect(gap2, `gap 2→3: ${gap2} ms`).toBeGreaterThanOrEqual(BACKOFF_BASE_MS * 2 * 0.6 - 750);
350      expect(gap2, `gap 2→3: ${gap2} ms`).toBeLessThanOrEqual(BACKOFF_BASE_MS * 2 * 1.2 + 1500);
351      // The schedule GROWS (exponential), it does not stay flat.
352      expect(gap2, `gap 2→3 (${gap2} ms) should exceed gap 1→2 (${gap1} ms)`).toBeGreaterThan(
353        gap1 - 400,
354      );
355 
356      // nextRetryAt after attempt 3 ≈ base·2^2 = 8 s in the future (measured
357      // from the observation of that row state).
358      const retryInMs = new Date(row!.nextRetryAt!).getTime() - rowAt;
359      expect(retryInMs, `nextRetryAt in ${retryInMs} ms`).toBeGreaterThanOrEqual(8000 * 0.8 - 1000);
360      expect(retryInMs, `nextRetryAt in ${retryInMs} ms`).toBeLessThanOrEqual(8000 * 1.2 + 1500);
361 
362      // -- 5. Delivery inspector UI: failed row + drawer ------------------
363      await safeClick(page, hookRow.getByRole("button", { name: "Deliveries" }), "Deliveries");
364      const failedRow = page
365        .getByRole("row")
366        .filter({ hasText: "page.created" })
367        .filter({ hasText: "FAILED" });
368      await expect(failedRow).toBeVisible({ timeout: 20_000 });
369      await safeClick(page, failedRow.getByRole("button", { name: "Open" }), "Open delivery");
370      const drawer = page.getByRole("dialog");
371      await expect(drawer).toBeVisible();
372      // Payload JSON (the dead page) + the signature verification hint.
373      await expect(drawer.getByText(PAGE2_TITLE)).toBeVisible();
374      expect(await drawer.getByText(page2.id).count()).toBeGreaterThanOrEqual(1);
375      await expect(drawer.getByText("X-Factory-Signature")).toBeVisible();
376      // .first(): "HMAC-SHA256" also appears in the explanatory caption.
377      await expect(drawer.getByText("HMAC-SHA256").first()).toBeVisible();
378 
379      await keyReq.dispose();
380    } finally {
381      await receiver.stop();
382    }
383  });
384});
385