/** * TASK-0124 — Database AI (generate / extract / formula) integration suite. * * Full stack (AppModule + Postgres + Redis) against the deterministic * `MockProvider` (`AI_PROVIDER=mock`; the `mode:"database"` branch emits a * single structured tool call derived from the prompt markers). * * Covers the spec's integration acceptance criteria (the "six scenarios" + * budget exhaustion + cost-row assertions): * - generate happy: a prompt returns a database whose properties are all in * the PropertyType enum (re-read via API), sample records exist as pages, * select options carry generated `opt-*` ids, values are stored per type, * the cost row carries `metadata.feature="db.generate"` + tokens, and the * hidden per-call conversation never appears in the list; * - generate permission: a GUEST (workspace VIEW) gets `403 FORBIDDEN` * pre-stream and nothing is created (empty database diff); * - generate hostile: the mock `mock-db-invalid` seam makes the model emit a * non-generatable property type → retry-once → in-band * `error {reason:"ai_invalid_structure"}` + `done {reason:"error"}`, with * NO orphan `Database` row; * - extract happy: a 3-row markdown table with one unmatched column → 3 * records created with correct per-type values (number `{value}`, select * `{optionId}`, text `{text}`), `ignoredColumns` lists the unmatched * column, cost row `feature="db.extract"`; * - extract preview: the dry-run endpoint returns the mapping (`isTable`, * `columns`, `mapped`, `ignored`, `rowCount`) with NO write and NO LLM; * - formula happy: a description produces an engine-validated expression that * is saved to `Formula` (with the engine AST + returnType, NOT `{raw}`), a * FORMULA property is created, and the sample rows are evaluated; * - formula hostile: an unparseable expression → retry-once → 422 * `formula_invalid` with NO `Formula` row written; * - budget exhaustion → 402 AI_BUDGET_EXCEEDED (pre-stream JSON), no * `db:generate` conversation row. */ // Config must be fixed BEFORE the app boots (loadEnv + the workspace cache). process.env.AI_PROVIDER = "mock"; process.env.SEARCH_VECTOR_DIMENSION = "8"; // Pacing keeps the suite fast but lets the SSE reader frame cleanly. process.env.AI_MOCK_DELTA_DELAY_MS = "25"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { createTestApp, type TestAppHandle } from "./helpers/api"; import { addMemberViaSeam, authHeaders, createWorkspace, createUser } from "./helpers/seed"; // Deep import (no `.js` — vite resolution), the same convention as the // ai-writing / ai-capability suites: the budget guard reads workspaces through // a 30 s cache, so the direct-DB budget writes in this file need the reset to // take effect immediately. The arming map (mock-db-invalid) is also cleared by // the reset, which keeps the persistent-per-workspace arming isolated. const { resetWorkspaceCache, clearMockDbInvalidStructure } = await import( "@factory/api/src/modules/ai/gateway-provider" ); let app: TestAppHandle; beforeAll(async () => { app = await createTestApp(); }, 180_000); afterAll(async () => { await app?.close(); }); // --------------------------------------------------------------------------- // constants // --------------------------------------------------------------------------- /** The real `PropertyType` enum (Prisma `enum PropertyType`) — single source. */ const PROPERTY_TYPE_ENUM = [ "TITLE", "TEXT", "RICH_TEXT", "NUMBER", "SELECT", "MULTI_SELECT", "STATUS", "DATE", "DATE_RANGE", "PERSON", "EMAIL", "PHONE", "URL", "CHECKBOX", "FILES", "CREATED_TIME", "CREATED_BY", "LAST_EDITED_TIME", "LAST_EDITED_BY", "RELATION", "ROLLUP", "FORMULA", "BUTTON", "UNIQUE_ID", ] as const; // --------------------------------------------------------------------------- // helpers // --------------------------------------------------------------------------- type SseEvent = { type: "delta" | "tool_call" | "usage" | "error" | "done"; text?: string; id?: string; name?: string; arguments?: Record; tokensIn?: number; tokensOut?: number; code?: string; message?: string; reason?: string; assistantMessageId?: string; userMessageId?: string; databaseId?: string; propertyIds?: string[]; recordIds?: string[]; ignoredColumns?: string[]; formulaId?: string; expression?: string; returnType?: string; samples?: Array<{ recordId: string; value: unknown; error?: string }>; }; /** POST an SSE database-AI endpoint, collect the frames. */ async function streamDb( path: string, accessToken: string, body: Record, ): Promise<{ status: number; events: SseEvent[]; contentType: string }> { const res = await fetch(`${app.baseUrl}${path}`, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${accessToken}` }, body: JSON.stringify(body), }); const contentType = res.headers.get("content-type") ?? ""; const events: SseEvent[] = []; if (!res.body) return { status: res.status, events, contentType }; const reader = res.body.getReader(); const decoder = new TextDecoder(); let buf = ""; for (;;) { const chunk = await reader.read(); if (chunk.done) break; buf += decoder.decode(chunk.value, { stream: true }); let idx: number; while ((idx = buf.indexOf("\n\n")) !== -1) { const frame = buf.slice(0, idx); buf = buf.slice(idx + 2); for (const line of frame.split("\n")) { if (!line.startsWith("data:")) continue; // skip `: keepalive` events.push(JSON.parse(line.slice(5).trim()) as SseEvent); } } } return { status: res.status, events, contentType }; } const json = async (res: Response): Promise> => (await res.json().catch(() => ({}))) as Record; /** A fresh workspace with an OWNER user. */ async function freshWorkspace(name: string) { const owner = await createUser(app.request, { name }); const { workspaceId } = await createWorkspace(app.request, authHeaders(owner.accessToken!), `${name} WS`); return { owner, workspaceId, h: authHeaders(owner.accessToken!) }; } /** Create a database via the normal page path (`type:"database"`). */ async function createDb(token: string, workspaceId: string, title: string): Promise { const res = await app.request("POST", `/api/v1/workspaces/${workspaceId}/pages`, { headers: authHeaders(token), body: { title, type: "database" }, }); expect([200, 201]).toContain(res.status); const body = (await res.json()) as { id: string; databaseId?: string }; expect(body.databaseId).toBeTruthy(); return body.databaseId!; } /** Add a property through the normal API (config passthrough, no option ids). */ async function addProperty( token: string, databaseId: string, def: { name: string; type: string; config?: unknown }, ): Promise<{ id: string; name: string; type: string }> { const res = await app.request("POST", `/api/v1/databases/${databaseId}/properties`, { headers: authHeaders(token), body: { name: def.name, type: def.type, config: def.config ?? {} }, }); expect([200, 201]).toContain(res.status); return (await res.json()) as { id: string; name: string; type: string }; } /** Add a record through the normal API. Returns the record DTO. */ async function addRecord( token: string, databaseId: string, def: { title?: string; properties?: Record }, ): Promise> { const res = await app.request("POST", `/api/v1/databases/${databaseId}/records`, { headers: authHeaders(token), body: { title: def.title, properties: def.properties ?? {}, links: {} }, }); expect([200, 201]).toContain(res.status); return (await res.json()) as Record; } async function listProperties(token: string, databaseId: string): Promise>> { const res = await app.request("GET", `/api/v1/databases/${databaseId}/properties`, { headers: authHeaders(token) }); expect(res.status).toBe(200); return ((await json(res)) as { data: Array> }).data; } async function getDb(token: string, databaseId: string): Promise> { const res = await app.request("GET", `/api/v1/databases/${databaseId}`, { headers: authHeaders(token) }); expect(res.status).toBe(200); return json(res); } async function listRecords(token: string, databaseId: string): Promise>> { const res = await app.request("GET", `/api/v1/databases/${databaseId}/records`, { headers: authHeaders(token) }); expect(res.status).toBe(200); return ((await json(res)) as { data: Array> }).data; } /** Arm the mock to emit a hostile structured tool call for one action. */ async function armDbInvalid(workspaceId: string, action: "generate" | "extract" | "formula"): Promise { const res = await app.request("POST", `/internal/workspaces/${workspaceId}/ai/mock-db-invalid`, { headers: { Authorization: `Bearer ${app.internalServiceToken}` }, body: { action }, }); expect(res.status).toBe(200); const body = (await json(res)) as { armed: boolean; action: string }; expect(body.armed).toBe(true); expect(body.action).toBe(action); } /** The persisted assistant cost row for a finished call. */ async function costRow(assistantMessageId: string) { return app.prisma.aiMessage.findUniqueOrThrow({ where: { id: assistantMessageId } }); } /** Assert none of the hidden per-call conversation titles appear in the list. */ async function assertHiddenConvsAbsent(token: string, workspaceId: string, titles: string[]): Promise { const res = await app.request("GET", `/api/v1/workspaces/${workspaceId}/ai/conversations`, { headers: authHeaders(token), }); expect(res.status).toBe(200); const list = (await json(res)) as { data: Array<{ title: string }> }; const shown = new Set(list.data.map((c) => c.title)); for (const t of titles) expect(shown.has(t), `hidden conversation "${t}" must not be listed`).toBe(false); } /** A markdown table: 3 rows, columns Name/Score/State/Notes (Notes unmatched). */ const EXTRACT_TABLE = [ "| Name | Score | State | Notes |", "| --- | --- | --- | --- |", "| Alice | 42 | Open | first note |", "| Bob | 7 | Closed | second note |", "| Carol | 99 | Open | third note |", ].join("\n"); // --------------------------------------------------------------------------- // generate // --------------------------------------------------------------------------- describe("TASK-0124 generate — happy path", () => { it("a prompt returns a valid database (enum props, page-backed records, cost row, hidden conv)", async () => { const { owner, workspaceId } = await freshWorkspace("Gen Owner"); const out = await streamDb(`/api/v1/workspaces/${workspaceId}/ai/databases/generate`, owner.accessToken!, { prompt: "a bug tracker", }); expect(out.status).toBe(200); expect(out.contentType).toContain("text/event-stream"); const tool = out.events.find((e) => e.type === "tool_call"); expect(tool?.name).toBe("createDatabase"); const usage = out.events.find((e) => e.type === "usage"); expect(usage).toBeTruthy(); const done = out.events.find((e) => e.type === "done"); expect(done?.reason).toBe("ok"); const databaseId = done!.databaseId!; expect(done!.propertyIds).toHaveLength(4); expect(done!.recordIds).toHaveLength(2); // Re-read the database through the API: every property type must be a real // PropertyType enum value (the spec's "22/24-type" assertion). const db = await getDb(owner.accessToken!, databaseId); const props = db.properties as Array>; const names = props.map((p) => String(p.name)); // seeded Title + Status + the 4 AI properties for (const n of ["Title", "Status", "Name", "State", "Due", "Done?"]) expect(names).toContain(n); for (const p of props) expect(PROPERTY_TYPE_ENUM).toContain(p.type as never); // The generated select "State" carries deterministic `opt-*` option ids. const state = props.find((p) => p.name === "State")!; const stateOptions = (state.config as { options: Array<{ id: string; name: string }> }).options; expect(stateOptions.map((o) => o.name).sort()).toEqual(["Done", "To Do"]); expect(stateOptions.map((o) => o.id).sort()).toEqual(["opt-done", "opt-to-do"]); // Sample records exist as pages, with per-type stored values. const records = await listRecords(owner.accessToken!, databaseId); expect(records).toHaveLength(2); for (const r of records) expect(r.pageId).toBeTruthy(); // each record IS a page const nameId = props.find((p) => p.name === "Name")!.id as string; const dueId = props.find((p) => p.name === "Due")!.id as string; const doneId = props.find((p) => p.name === "Done?")!.id as string; const rec0 = records[0]!; expect(rec0.title).toBe("First item"); expect(rec0.properties![nameId]).toMatchObject({ text: "First item" }); expect(rec0.properties![state.id]).toEqual({ optionId: "opt-to-do" }); expect(rec0.properties![dueId]).toMatchObject({ start: "2026-01-01" }); expect(rec0.properties![doneId]).toEqual({ value: false }); expect(records[1]!.properties![state.id]).toEqual({ optionId: "opt-done" }); expect(records[1]!.properties![doneId]).toEqual({ value: true }); // Audit + cost row. const audit = await app.prisma.auditLog.findFirst({ where: { workspaceId, action: "ai.database_generated", resourceId: databaseId }, }); expect(audit).toBeTruthy(); const row = await costRow(done!.assistantMessageId!); expect(row.role).toBe("assistant"); expect((row.metadata as { feature: string }).feature).toBe("db.generate"); expect(row.tokensIn).toBeGreaterThan(0); expect(row.tokensOut).toBeGreaterThan(0); expect(row.model).toBeTruthy(); // The hidden per-call conversation is not listed. await assertHiddenConvsAbsent(owner.accessToken!, workspaceId, ["db:generate"]); }); }); describe("TASK-0124 generate — permissions", () => { it("a GUEST (VIEW) gets 403 FORBIDDEN pre-stream and nothing is created", async () => { const { workspaceId } = await freshWorkspace("GenPerm Owner"); const guest = await createUser(app.request, { name: "Gen Guest" }); await addMemberViaSeam(app.request, app.internalServiceToken, workspaceId, guest.email, "GUEST"); const before = await app.prisma.database.count({ where: { workspaceId } }); const res = await app.request("POST", `/api/v1/workspaces/${workspaceId}/ai/databases/generate`, { headers: authHeaders(guest.accessToken!), body: { prompt: "a bug tracker" }, }); expect(res.status).toBe(403); const body = await json(res); expect(body.error.code).toBe("FORBIDDEN"); // empty database diff — nothing created const after = await app.prisma.database.count({ where: { workspaceId } }); expect(after).toBe(before); // and no hidden conversation row const convs = await app.prisma.aiConversation.findMany({ where: { workspaceId, title: "db:generate" } }); expect(convs).toHaveLength(0); }); }); describe("TASK-0124 generate — hostile structure", () => { it("a non-generatable type → retry-once → in-band ai_invalid_structure, no orphan Database", async () => { const { owner, workspaceId } = await freshWorkspace("GenHostile Owner"); await armDbInvalid(workspaceId, "generate"); const out = await streamDb(`/api/v1/workspaces/${workspaceId}/ai/databases/generate`, owner.accessToken!, { prompt: "a bug tracker", }); expect(out.status).toBe(200); // the error is in-band (SSE already flushed) const err = out.events.find((e) => e.type === "error"); expect(err).toBeTruthy(); expect(err!.reason).toBe("ai_invalid_structure"); const done = out.events.find((e) => e.type === "done"); expect(done?.reason).toBe("error"); // No orphan database (nothing was created before the validation failed). const dbs = await app.prisma.database.count({ where: { workspaceId } }); expect(dbs).toBe(0); // The cost row records the error (assistant content "error: …"). const row = await costRow(done!.assistantMessageId!); expect((row.metadata as { feature: string; error: string }).feature).toBe("db.generate"); expect((row.metadata as { error: string }).error).toBe("ai_invalid_structure"); expect(row.content).toMatch(/^error:/); }); }); // --------------------------------------------------------------------------- // extract // --------------------------------------------------------------------------- async function makeExtractDb(token: string, workspaceId: string) { const databaseId = await createDb(token, workspaceId, "Extract DB"); const name = await addProperty(token, databaseId, { name: "Name", type: "TEXT" }); const score = await addProperty(token, databaseId, { name: "Score", type: "NUMBER" }); const state = await addProperty(token, databaseId, { name: "State", type: "SELECT", config: { options: [{ name: "Open" }, { name: "Closed" }] }, }); return { databaseId, name, score, state }; } describe("TASK-0124 extract — happy path", () => { it("a 3-row markdown table with an unmatched column → 3 records, per-type values, ignoredColumns", async () => { const { owner, workspaceId } = await freshWorkspace("Extract Owner"); const { databaseId, name, score, state } = await makeExtractDb(owner.accessToken!, workspaceId); const out = await streamDb(`/api/v1/workspaces/${workspaceId}/ai/databases/extract`, owner.accessToken!, { databaseId, text: EXTRACT_TABLE, }); expect(out.status).toBe(200); const tool = out.events.find((e) => e.type === "tool_call"); expect(tool?.name).toBe("insertRecords"); const done = out.events.find((e) => e.type === "done"); expect(done?.reason).toBe("ok"); expect(done!.recordIds).toHaveLength(3); expect(done!.ignoredColumns).toEqual(["Notes"]); // Stored per-type values (number → {value}, select → {optionId}, text → {text}). const records = await listRecords(owner.accessToken!, databaseId); expect(records).toHaveLength(3); const r0 = records[0]!; expect(r0.properties![name.id]).toMatchObject({ text: "Alice" }); expect(r0.properties![score.id]).toEqual({ value: 42 }); // "Open" → the deterministic option id (the API-created option has no id) expect(r0.properties![state.id]).toEqual({ optionId: "opt-open" }); const r1 = records[1]!; expect(r1.properties![score.id]).toEqual({ value: 7 }); expect(r1.properties![state.id]).toEqual({ optionId: "opt-closed" }); // Audit + cost row. const audit = await app.prisma.auditLog.findFirst({ where: { workspaceId, action: "ai.records_extracted", resourceId: databaseId }, }); expect(audit).toBeTruthy(); const row = await costRow(done!.assistantMessageId!); expect((row.metadata as { feature: string }).feature).toBe("db.extract"); expect(row.tokensIn).toBeGreaterThan(0); await assertHiddenConvsAbsent(owner.accessToken!, workspaceId, ["db:extract"]); }); }); describe("TASK-0124 extract — preview (dry-run)", () => { it("returns the mapping with no write and no LLM", async () => { const { owner, workspaceId } = await freshWorkspace("Preview Owner"); const { databaseId, name, score, state } = await makeExtractDb(owner.accessToken!, workspaceId); const before = await app.prisma.databaseRecord.count({ where: { databaseId } }); const res = await app.request("POST", `/api/v1/workspaces/${workspaceId}/ai/databases/extract/preview`, { headers: authHeaders(owner.accessToken!), body: { databaseId, text: EXTRACT_TABLE }, }); expect(res.status).toBe(200); const body = (await json(res)) as { databaseId: string; isTable: boolean; columns: string[]; mapped: Array<{ column: string; propertyId: string }>; ignored: string[]; rowCount: number; }; expect(body.databaseId).toBe(databaseId); expect(body.isTable).toBe(true); expect(body.columns).toEqual(["Name", "Score", "State", "Notes"]); expect(body.rowCount).toBe(3); expect(body.ignored).toEqual(["Notes"]); expect(body.mapped).toEqual([ { column: "Name", propertyId: name.id }, { column: "Score", propertyId: score.id }, { column: "State", propertyId: state.id }, ]); // No write (and no hidden conversation — the preview never touches the LLM). const after = await app.prisma.databaseRecord.count({ where: { databaseId } }); expect(after).toBe(before); const convs = await app.prisma.aiConversation.findMany({ where: { workspaceId, title: "db:extract" } }); expect(convs).toHaveLength(0); }); }); // --------------------------------------------------------------------------- // formula // --------------------------------------------------------------------------- describe("TASK-0124 formula — happy path", () => { it("a description → engine-validated expression saved to Formula (AST + returnType), FORMULA prop, evaluated samples", async () => { const { owner, workspaceId } = await freshWorkspace("Formula Owner"); const databaseId = await createDb(owner.accessToken!, workspaceId, "Formula DB"); // The mock formula references the FIRST [PROP] line = the seeded "Title". // Give the records distinctive titles so the sample values are observable. const recIds: string[] = []; for (const title of ["Done", "Open", "Done"]) { const r = await addRecord(owner.accessToken!, databaseId, { title }); recIds.push(String(r.id)); } const res = await app.request("POST", `/api/v1/workspaces/${workspaceId}/ai/databases/formulas`, { headers: authHeaders(owner.accessToken!), body: { databaseId, description: "score a row 100 when it is done", name: "Points" }, }); expect(res.status).toBe(200); const body = (await json(res)) as { formulaId: string; propertyId: string; expression: string; returnType: string; samples: Array<{ recordId: string; value: unknown; error?: string }>; tokensIn: number; tokensOut: number; }; expect(body.expression).toBe('if(prop("Title") == "Done", 100, 0)'); expect(body.returnType).toBe("number"); expect(body.samples).toHaveLength(3); // Titles "Done"/"Open"/"Done" → 100 / 0 / 100 expect(body.samples.map((s) => s.value)).toEqual([100, 0, 100]); for (const s of body.samples) expect(s.error).toBeUndefined(); // The Formula row stores the engine AST (NOT the legacy `{raw}` placeholder) // + the inferred return type + the dependency (the Title property id). const formula = await app.prisma.formula.findUniqueOrThrow({ where: { id: body.formulaId } }); expect(formula.expression).toBe(body.expression); expect(formula.returnType).toBe("number"); const ast = formula.ast as Record; expect(ast).toBeTruthy(); expect(ast.raw).toBeUndefined(); // NOT the legacy {raw, safe:true} placeholder const titleProp = (await listProperties(owner.accessToken!, databaseId)).find((p) => p.name === "Title")!; expect(formula.dependencies).toEqual([titleProp.id]); // A FORMULA property was created through the normal path. const formulaProp = await app.prisma.databaseProperty.findFirst({ where: { databaseId, name: "Points", type: "FORMULA" }, }); expect(formulaProp).toBeTruthy(); expect(body.propertyId).toBe(formulaProp!.id); // Audits: the AI action + the normal formula-created action. const aiAudit = await app.prisma.auditLog.findFirst({ where: { workspaceId, action: "ai.formula_generated" }, }); expect(aiAudit).toBeTruthy(); const created = await app.prisma.auditLog.findFirst({ where: { workspaceId, action: "database.formula_created", resourceId: body.formulaId }, }); expect(created).toBeTruthy(); // The formula endpoint returns JSON (no assistantMessageId), so the cost // row is looked up through the hidden `db:formula` conversation. const conv = await app.prisma.aiConversation.findFirst({ where: { workspaceId, title: "db:formula" }, orderBy: { createdAt: "desc" }, }); expect(conv).toBeTruthy(); const row = await app.prisma.aiMessage.findFirst({ where: { conversationId: conv!.id, role: "assistant" }, orderBy: { createdAt: "desc" }, }); expect(row).toBeTruthy(); expect((row!.metadata as { feature: string }).feature).toBe("db.formula"); await assertHiddenConvsAbsent(owner.accessToken!, workspaceId, ["db:formula"]); void recIds; }); }); describe("TASK-0124 formula — hostile", () => { it("an unparseable expression → retry-once → 422 formula_invalid, no Formula row", async () => { const { owner, workspaceId } = await freshWorkspace("FormulaHostile Owner"); const databaseId = await createDb(owner.accessToken!, workspaceId, "FormulaHostile DB"); await addRecord(owner.accessToken!, databaseId, { title: "a row" }); await armDbInvalid(workspaceId, "formula"); const res = await app.request("POST", `/api/v1/workspaces/${workspaceId}/ai/databases/formulas`, { headers: authHeaders(owner.accessToken!), body: { databaseId, description: "something impossible" }, }); expect(res.status).toBe(422); const body = (await json(res)) as { error: { code: string; details?: Array<{ reason?: string; parseError?: string }> } }; expect(body.error.code).toBe("UNPROCESSABLE_ENTITY"); expect(body.error.details?.[0]?.reason).toBe("formula_invalid"); expect(body.error.details?.[0]?.parseError).toBeTruthy(); // No Formula row was written. const count = await app.prisma.formula.count({ where: { databaseId } }); expect(count).toBe(0); // No FORMULA property either. const prop = await app.prisma.databaseProperty.findFirst({ where: { databaseId, type: "FORMULA" } }); expect(prop).toBeNull(); }); }); // --------------------------------------------------------------------------- // budget // --------------------------------------------------------------------------- describe("TASK-0124 budget", () => { it("an exhausted budget → 402 AI_BUDGET_EXCEEDED (pre-stream, no db:generate conv)", async () => { const { owner, workspaceId, h } = await freshWorkspace("BudgetDB Owner"); await app.prisma.workspace.update({ where: { id: workspaceId }, data: { usageLimits: { aiMonthlyTokens: 1 } }, }); resetWorkspaceCache(); // the guard reads the workspace through a 30 s cache const res = await app.request("POST", `/api/v1/workspaces/${workspaceId}/ai/databases/generate`, { headers: h, body: { prompt: "a bug tracker" }, }); expect(res.status).toBe(402); const body = await json(res); expect(body.error.code).toBe("AI_BUDGET_EXCEEDED"); const convs = await app.prisma.aiConversation.findMany({ where: { workspaceId, title: "db:generate" } }); expect(convs).toHaveLength(0); // restore await app.prisma.workspace.update({ where: { id: workspaceId }, data: { usageLimits: {} } }); resetWorkspaceCache(); }); });