ai-generate.integration.test.ts

ai-generate.integration.test.tstests/integration/ai-generate.integration.test.tsraw ← integration18,796 B← Back to the app
1/**
2 * TASK-0126 — AI page-generation integration suite (the golden structure +
3 * rollback + timeout + idempotency + permissions).
4 *
5 * Full stack (AppModule + Postgres + Redis) against the deterministic
6 * `MockProvider` (`AI_PROVIDER=mock`; the `mode:"generate"` branch emits the
7 * deterministic "product launch plan" plan: a root page, six HEADING_2
8 * sections, and a child "Tasks" database with a Due date column + 2 records).
9 *
10 * Covers the spec's integration acceptance criteria:
11 *   - golden (API-asserted structure): the SSE stream emits the per-step
12 *     `tool` frames + a terminal `done {pageId, created:{pages,blocks,
13 *     databases,relations}}`; the root page has the six HEADING_2 sections; the
14 *     Tasks child database has Title/Status/Due (Due is DATE) + the 2 sample
15 *     records (Due stored as `{start}`); the cost row carries
16 *     `metadata.feature="generate"`; the hidden `generate:` conversation never
17 *     appears in the list.
18 *   - rollback: the `mock-generate` seam arms a broken step → terminal in-band
19 *     `error {reason:"generation_failed"}`, the partial tree is rolled back
20 *     (no live generated page remains), audit `ai.generation_failed`.
21 *   - timeout: the seam arms a slow plan + the env cap is lowered to 2 s →
22 *     terminal `error {code:"SERVICE_UNAVAILABLE", reason:"ai_timeout"}` in
23 *     ≈2 s, nothing created, audit `failedStep:"timeout"`.
24 *   - idempotency: a retried request with the SAME `Idempotency-Key` replays
25 *     the terminal `done` (same pageId) without double-creating the page.
26 *   - permissions: a GUEST (VIEW) gets 403 pre-stream, a non-member gets 404
27 *     (fail-closed) — both create nothing.
28 */
29// Config must be fixed BEFORE the app boots (loadEnv + the workspace cache).
30process.env.AI_PROVIDER = "mock";
31process.env.SEARCH_VECTOR_DIMENSION = "8";
32// Pacing keeps the suite fast but lets the SSE reader frame cleanly.
33process.env.AI_MOCK_DELTA_DELAY_MS = "10";
34 
35import { afterAll, beforeAll, describe, expect, it } from "vitest";
36import { createTestApp, type TestAppHandle } from "./helpers/api";
37import { addMemberViaSeam, authHeaders, createWorkspace, createUser } from "./helpers/seed";
38// Deep import (no `.js` — vite resolution): the budget guard reads workspaces
39// through a 30 s cache, so the direct-DB budget writes need the reset to take
40// effect immediately. The arming maps (mock-generate) are also cleared by the
41// reset, which keeps the per-workspace arming isolated.
42const { resetWorkspaceCache } = await import("@factory/api/src/modules/ai/gateway-provider");
43 
44let app: TestAppHandle;
45 
46beforeAll(async () => {
47  app = await createTestApp();
48}, 180_000);
49afterAll(async () => {
50  await app?.close();
51});
52 
53// ---------------------------------------------------------------------------
54// helpers
55// ---------------------------------------------------------------------------
56 
57type GenEvent = {
58  type: "tool" | "done" | "cancelled" | "error";
59  name?: string;
60  status?: "started" | "done";
61  ref?: string;
62  pageId?: string;
63  created?: { pages: string[]; blocks: number; databases: string[]; relations: number };
64  tokensIn?: number;
65  tokensOut?: number;
66  code?: string;
67  message?: string;
68  reason?: string;
69  retryable?: boolean;
70};
71 
72/** POST the generate endpoint (SSE), collect the frames. `signal` lets a test
73 *  abort a stuck stream (the timeout test relies on the server capping, not on
74 *  this). */
75async function streamGenerate(
76  path: string,
77  accessToken: string,
78  body: Record<string, unknown>,
79  opts: { idempotencyKey?: string } = {},
80): Promise<{ status: number; events: GenEvent[]; contentType: string; ms: number }> {
81  const started = Date.now();
82  const res = await fetch(`${app.baseUrl}${path}`, {
83    method: "POST",
84    headers: {
85      "Content-Type": "application/json",
86      Authorization: `Bearer ${accessToken}`,
87      ...(opts.idempotencyKey ? { "Idempotency-Key": opts.idempotencyKey } : {}),
88    },
89    body: JSON.stringify(body),
90  });
91  const contentType = res.headers.get("content-type") ?? "";
92  const events: GenEvent[] = [];
93  if (res.body) {
94    const reader = res.body.getReader();
95    const decoder = new TextDecoder();
96    let buf = "";
97    for (;;) {
98      const chunk = await reader.read();
99      if (chunk.done) break;
100      buf += decoder.decode(chunk.value, { stream: true });
101      let idx: number;
102      while ((idx = buf.indexOf("\n\n")) !== -1) {
103        const frame = buf.slice(0, idx);
104        buf = buf.slice(idx + 2);
105        for (const line of frame.split("\n")) {
106          if (!line.startsWith("data:")) continue; // skip `: keepalive`
107          events.push(JSON.parse(line.slice(5).trim()) as GenEvent);
108        }
109      }
110    }
111  }
112  return { status: res.status, events, contentType, ms: Date.now() - started };
113}
114 
115const json = async (res: Response): Promise<Record<string, unknown>> =>
116  (await res.json().catch(() => ({}))) as Record<string, unknown>;
117 
118/** A fresh workspace with an OWNER user. */
119async function freshWorkspace(name: string) {
120  const owner = await createUser(app.request, { name });
121  const { workspaceId } = await createWorkspace(app.request, authHeaders(owner.accessToken!), `${name} WS`);
122  return { owner, workspaceId, h: authHeaders(owner.accessToken!) };
123}
124 
125/** Arm the mock-generate seam for one workspace (single-shot). */
126async function armGenerateSeam(workspaceId: string, body: { failStep?: number; slowMs?: number }): Promise<void> {
127  const res = await app.request("POST", `/internal/workspaces/${workspaceId}/ai/mock-generate`, {
128    headers: { Authorization: `Bearer ${app.internalServiceToken}` },
129    body,
130  });
131  expect(res.status).toBe(200);
132  const body2 = (await json(res)) as { armed: boolean };
133  expect(body2.armed).toBe(true);
134}
135 
136/** The persisted assistant cost row for a finished call. */
137async function costRow(assistantMessageId: string) {
138  return app.prisma.aiMessage.findUniqueOrThrow({ where: { id: assistantMessageId } });
139}
140 
141/** Assert none of the hidden per-call conversation titles appear in the list. */
142async function assertHiddenConvsAbsent(token: string, workspaceId: string, titles: string[]): Promise<void> {
143  const res = await app.request("GET", `/api/v1/workspaces/${workspaceId}/ai/conversations`, {
144    headers: authHeaders(token),
145  });
146  expect(res.status).toBe(200);
147  const list = (await json(res)) as { data: Array<{ title: string }> };
148  const shown = new Set(list.data.map((c) => c.title));
149  for (const t of titles) expect(shown.has(t), `hidden conversation "${t}" must not be listed`).toBe(false);
150}
151 
152// ---------------------------------------------------------------------------
153// golden
154// ---------------------------------------------------------------------------
155 
156describe("TASK-0126 generate — golden structure", () => {
157  it("the product-launch prompt yields a page (6 headings) + a Tasks database (Due date + 2 records)", async () => {
158    const { owner, workspaceId, h } = await freshWorkspace("Gen Owner");
159    const out = await streamGenerate(`/api/v1/workspaces/${workspaceId}/ai/generate`, owner.accessToken!, {
160      prompt: "a product launch plan with a tasks database",
161    });
162    expect(out.status).toBe(200);
163    expect(out.contentType).toContain("text/event-stream");
164 
165    // Per-step tool frames (createPage, insertBlocks, createDatabase) — each
166    // started + done, in plan order.
167    const toolNames = out.events.filter((e) => e.type === "tool").map((e) => e.name);
168    expect(toolNames).toContain("createPage");
169    expect(toolNames).toContain("insertBlocks");
170    expect(toolNames).toContain("createDatabase");
171 
172    // Exactly ONE terminal frame, of type `done`, with the created counts.
173    const terminals = out.events.filter((e) => e.type === "done" || e.type === "error" || e.type === "cancelled");
174    expect(terminals).toHaveLength(1);
175    const done = terminals[0] as GenEvent & { type: "done" };
176    expect(done.type).toBe("done");
177    expect(done.pageId).toBeTruthy();
178    expect(done.created?.pages).toHaveLength(1);
179    expect(done.created?.blocks).toBe(6);
180    expect(done.created?.databases).toHaveLength(1);
181    expect(done.created?.relations).toBe(0);
182 
183    const pageId = done.pageId!;
184    // The root page title + its six HEADING_2 sections (re-read via the API).
185    const page = await app.prisma.page.findUniqueOrThrow({ where: { id: pageId } });
186    expect(page.title).toBe("Product Launch Plan");
187    const blocks = await app.prisma.block.findMany({ where: { pageId, deletedAt: null } });
188    const headings = blocks.filter((b) => b.type === "HEADING_2").map((b) => (b.content as { text: string }).text);
189    expect(headings).toEqual(["Objectives", "Requirements", "Timeline", "Risks", "Team", "Meeting Notes"]);
190 
191    // The Tasks child database: a child page (parentId=root) with a Database row.
192    const childPage = await app.prisma.page.findFirst({ where: { parentId: pageId, workspaceId, deletedAt: null } });
193    expect(childPage).toBeTruthy();
194    expect(childPage!.title).toBe("Tasks");
195    const db = await app.prisma.database.findFirst({ where: { pageId: childPage!.id } });
196    expect(db).toBeTruthy();
197 
198    // Properties: seeded Title + Status + the AI-added Due (DATE).
199    const props = await app.prisma.databaseProperty.findMany({ where: { databaseId: db!.id, deletedAt: null } });
200    const byName = Object.fromEntries(props.map((p) => [p.name, p]));
201    expect(byName.Title).toBeTruthy();
202    expect(byName.Status).toBeTruthy();
203    expect(byName.Due).toBeTruthy();
204    expect(byName.Due.type).toBe("DATE");
205    // The seeded Status select carries the To Do / In Progress / Done options.
206    const statusOpts = (byName.Status.config as { options: Array<{ name: string }> }).options.map((o) => o.name);
207    expect(statusOpts).toEqual(expect.arrayContaining(["To Do", "In Progress", "Done"]));
208 
209    // The 2 sample records, with the Due date stored as `{ start }`. A record
210    // IS a page (title on the page); property values live in RecordProperty.
211    const records = await app.prisma.databaseRecord.findMany({ where: { databaseId: db!.id } });
212    expect(records).toHaveLength(2);
213    const dueId = byName.Due.id;
214    const recordPages = await app.prisma.page.findMany({ where: { id: { in: records.map((r) => r.pageId).filter(Boolean) } } });
215    const titles = recordPages.map((p) => p.title).sort();
216    expect(titles).toEqual(["Draft launch announcement", "Schedule launch review"]);
217    for (const r of records) {
218      const dueRow = await app.prisma.recordProperty.findUnique({
219        where: { recordId_propertyId: { recordId: r.id, propertyId: dueId } },
220      });
221      expect(dueRow?.value).toMatchObject({ start: expect.stringMatching(/^2026-01-(15|20)$/) });
222    }
223 
224    // Audit + cost row (feature="generate").
225    const audit = await app.prisma.auditLog.findFirst({
226      where: { workspaceId, action: "ai.page_generated", resourceId: pageId },
227    });
228    expect(audit).toBeTruthy();
229    const hidden = await app.prisma.aiConversation.findFirst({ where: { workspaceId, title: "generate:" } });
230    expect(hidden).toBeTruthy();
231    const row = await app.prisma.aiMessage.findFirst({
232      where: { conversationId: hidden!.id, role: "assistant" },
233    });
234    expect(row).toBeTruthy();
235    expect((row!.metadata as { feature: string }).feature).toBe("generate");
236    expect(row!.tokensIn!).toBeGreaterThan(0);
237 
238    // The hidden per-call conversation is not listed in the panel.
239    await assertHiddenConvsAbsent(owner.accessToken!, workspaceId, ["generate:"]);
240    void h;
241  });
242});
243 
244// ---------------------------------------------------------------------------
245// rollback
246// ---------------------------------------------------------------------------
247 
248describe("TASK-0126 generate — rollback on failure", () => {
249  it("a failing step rolls back the partial tree (no live generated page, audit generation_failed)", async () => {
250    const { owner, workspaceId } = await freshWorkspace("GenRoll Owner");
251    // failStep:2 → the plan's 2nd step (insertBlocks) is broken (unknown ref).
252    await armGenerateSeam(workspaceId, { failStep: 2 });
253 
254    const out = await streamGenerate(`/api/v1/workspaces/${workspaceId}/ai/generate`, owner.accessToken!, {
255      prompt: "a product launch plan",
256    });
257    expect(out.status).toBe(200); // the error is in-band (SSE already flushed)
258 
259    // Exactly ONE terminal frame: the in-band error (not `done`).
260    const terminals = out.events.filter((e) => e.type === "done" || e.type === "error" || e.type === "cancelled");
261    expect(terminals).toHaveLength(1);
262    const err = terminals[0] as GenEvent;
263    expect(err.type).toBe("error");
264    expect(err.reason).toBe("generation_failed");
265    expect(err.retryable).toBe(true);
266 
267    // The partial tree (the root page) was rolled back — no live page remains.
268    const leftover = await app.prisma.page.findMany({
269      where: { workspaceId, title: "Product Launch Plan", trashedAt: null },
270    });
271    expect(leftover).toHaveLength(0);
272 
273    // Audit records the failure with the failed step.
274    const audit = await app.prisma.auditLog.findFirst({
275      where: { workspaceId, action: "ai.generation_failed" },
276    });
277    expect(audit).toBeTruthy();
278    expect((audit!.metadata as { failedStep: string }).failedStep).toBe("insertBlocks");
279  });
280});
281 
282// ---------------------------------------------------------------------------
283// timeout
284// ---------------------------------------------------------------------------
285 
286describe("TASK-0126 generate — total timeout", () => {
287  it("a slow plan past the cap → SERVICE_UNAVAILABLE ai_timeout in ≈2 s, nothing created", async () => {
288    const { owner, workspaceId } = await freshWorkspace("GenTimeout Owner");
289    // The seam makes the plan fetch take 5 s; the env cap is lowered to 2 s so
290    // the watchdog fires first (the service reads the cap per call).
291    await armGenerateSeam(workspaceId, { slowMs: 5000 });
292    process.env.AI_GENERATE_TIMEOUT_MS = "2000";
293    try {
294      const out = await streamGenerate(`/api/v1/workspaces/${workspaceId}/ai/generate`, owner.accessToken!, {
295        prompt: "a product launch plan",
296      });
297      expect(out.status).toBe(200);
298 
299      const terminals = out.events.filter((e) => e.type === "done" || e.type === "error" || e.type === "cancelled");
300      expect(terminals).toHaveLength(1);
301      const err = terminals[0] as GenEvent;
302      expect(err.type).toBe("error");
303      expect(err.code).toBe("SERVICE_UNAVAILABLE");
304      expect(err.reason).toBe("ai_timeout");
305      expect(err.retryable).toBe(true);
306      // Capped at ≈2 s (the watchdog), NOT the mock's 5 s plan latency.
307      expect(out.ms).toBeLessThan(4000);
308      expect(out.ms).toBeGreaterThanOrEqual(1900);
309 
310      // Nothing was created (the plan never reached the executor).
311      const leftover = await app.prisma.page.findMany({
312        where: { workspaceId, title: "Product Launch Plan" },
313      });
314      expect(leftover).toHaveLength(0);
315 
316      const audit = await app.prisma.auditLog.findFirst({
317        where: { workspaceId, action: "ai.generation_failed" },
318      });
319      expect(audit).toBeTruthy();
320      expect((audit!.metadata as { failedStep: string }).failedStep).toBe("timeout");
321    } finally {
322      delete process.env.AI_GENERATE_TIMEOUT_MS;
323    }
324  });
325});
326 
327// ---------------------------------------------------------------------------
328// idempotency
329// ---------------------------------------------------------------------------
330 
331describe("TASK-0126 generate — idempotency (Idempotency-Key)", () => {
332  it("a retry with the SAME key replays the terminal `done` (same pageId) without double-creating", async () => {
333    const { owner, workspaceId } = await freshWorkspace("GenIdem Owner");
334    const key = `gen_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
335    const path = `/api/v1/workspaces/${workspaceId}/ai/generate`;
336 
337    const first = await streamGenerate(path, owner.accessToken!, { prompt: "a product launch plan" }, { idempotencyKey: key });
338    const done1 = first.events.find((e) => e.type === "done") as GenEvent;
339    expect(done1).toBeTruthy();
340    expect(done1.pageId).toBeTruthy();
341 
342    const pagesAfterFirst = await app.prisma.page.count({ where: { workspaceId, title: "Product Launch Plan", trashedAt: null } });
343    expect(pagesAfterFirst).toBe(1);
344 
345    // The retry: SAME key → the service replays the terminal `done` (the same
346    // pageId) WITHOUT re-running the executor (no second page).
347    const second = await streamGenerate(path, owner.accessToken!, { prompt: "a product launch plan" }, { idempotencyKey: key });
348    const done2 = second.events.find((e) => e.type === "done") as GenEvent;
349    expect(done2).toBeTruthy();
350    expect(done2.pageId).toBe(done1.pageId);
351 
352    const pagesAfterRetry = await app.prisma.page.count({ where: { workspaceId, title: "Product Launch Plan", trashedAt: null } });
353    expect(pagesAfterRetry).toBe(1); // no double-create
354  });
355});
356 
357// ---------------------------------------------------------------------------
358// permissions
359// ---------------------------------------------------------------------------
360 
361describe("TASK-0126 generate — permissions", () => {
362  it("a GUEST (VIEW) gets 403 pre-stream and nothing is created", async () => {
363    const { workspaceId } = await freshWorkspace("GenPerm Owner");
364    const guest = await createUser(app.request, { name: "Gen Guest" });
365    await addMemberViaSeam(app.request, app.internalServiceToken, workspaceId, guest.email, "GUEST");
366 
367    const res = await app.request("POST", `/api/v1/workspaces/${workspaceId}/ai/generate`, {
368      headers: authHeaders(guest.accessToken!),
369      body: { prompt: "a product launch plan" },
370    });
371    expect(res.status).toBe(403);
372    const body = await json(res);
373    expect(body.error.code).toBe("FORBIDDEN");
374    const leftover = await app.prisma.page.count({ where: { workspaceId, title: "Product Launch Plan" } });
375    expect(leftover).toBe(0);
376    const convs = await app.prisma.aiConversation.findMany({ where: { workspaceId, title: "generate:" } });
377    expect(convs).toHaveLength(0);
378  });
379 
380  it("a non-member gets 404 (fail-closed) and nothing is created", async () => {
381    const { workspaceId } = await freshWorkspace("GenNonMember Owner");
382    const stranger = await createUser(app.request, { name: "Gen Stranger" });
383 
384    const res = await app.request("POST", `/api/v1/workspaces/${workspaceId}/ai/generate`, {
385      headers: authHeaders(stranger.accessToken!),
386      body: { prompt: "a product launch plan" },
387    });
388    expect(res.status).toBe(404);
389    const body = await json(res);
390    expect(body.error.code).toBe("NOT_FOUND");
391    const leftover = await app.prisma.page.count({ where: { workspaceId, title: "Product Launch Plan" } });
392    expect(leftover).toBe(0);
393  });
394});
395