prompts.test.ts

prompts.test.tsapps/api/src/modules/ai/__tests__/prompts.test.tsraw ← __tests__19,748 B← Back to the app
1/**
2 * TASK-0122 — unit tests for the server-side context assembler (prompts.ts).
3 *
4 * Covers the spec's unit acceptance criteria:
5 *   - caps: at most 40 blocks of page content / 20 sample database rows;
6 *   - HTML is stripped (no raw block HTML reaches the prompt);
7 *   - permission drop: a resource the requester can't VIEW is excluded from
8 *     the prompt (content never included) and a note is set (IDOR guard:
9 *     cross-workspace IDs drop silently too, because lookups are scoped);
10 *   - record value rendering (title/select/checkbox/number/date/person/url).
11 */
12import { describe, expect, it, vi } from "vitest";
13import { actorLevel } from "../../../lib/authorize.js";
14import {
15  assembleContext,
16  BASE_SYSTEM_PROMPT,
17  MAX_CONTEXT_BLOCKS,
18  MAX_CONTEXT_ROWS,
19  recordValueText,
20  stripHtml,
21} from "../prompts.js";
22 
23// TASK-0127: prompts.ts resolves visibility through the ACTOR-aware
24// `actorLevel` (no HTTP context in unit tests → the user path, same result
25// as the raw `effectiveLevel`). Stub the function the assembler actually
26// calls.
27vi.mock("../../../lib/authorize.js", async (importOriginal) => {
28  const actual = await importOriginal<typeof import("../../../lib/authorize.js")>();
29  return { ...actual, actorLevel: vi.fn() };
30});
31 
32const lvl = actorLevel as unknown as ReturnType<typeof vi.fn>;
33 
34/** A mock Prisma carrying just the delegates assembleContext touches. */
35function mockPrisma(state: {
36  page?: { id: string; workspaceId: string; title: string } | null;
37  blocks?: Array<{ id: string; type: string; content: unknown }>;
38  database?: { id: string; workspaceId: string; title: string } | null;
39  dbProps?: Array<{ name: string; type: string; config: unknown }>;
40  records?: Array<{ position: string; properties: Array<{ property: { name: string; type: string; config: unknown }; value: unknown }> }>;
41}) {
42  return {
43    page: {
44      findFirst: vi.fn(async () => (state.page ? { id: state.page.id, title: state.page.title } : null)),
45    },
46    block: {
47      // Honors `take` like the real delegate (the DB-side cap the
48      // assembler relies on — the spec's "up to 40 blocks").
49      findMany: vi.fn(async (args: { take?: number }) => {
50        const rows = state.blocks ?? [];
51        return (args.take ? rows.slice(0, args.take) : rows).map((b) => ({ id: b.id, type: b.type, content: b.content, position: b.id }));
52      }),
53    },
54    database: {
55      findFirst: vi.fn(async () => (state.database ? { id: state.database.id, title: state.database.title } : null)),
56    },
57    databaseProperty: {
58      findMany: vi.fn(async () => state.dbProps ?? []),
59    },
60    databaseRecord: {
61      findMany: vi.fn(async (args: { take?: number }) => {
62        const rows = state.records ?? [];
63        return args.take ? rows.slice(0, args.take) : rows;
64      }),
65    },
66  } as never;
67}
68 
69const ws = "ws1";
70const user = "u1";
71 
72describe("stripHtml", () => {
73  it("removes tags, decodes entities, collapses whitespace", () => {
74    expect(stripHtml("<p>Hello&nbsp;<b>world</b></p>")).toBe("Hello world");
75    expect(stripHtml("a&amp;b&lt;c&gt;d&quot;e&quot;")).toBe('a&b<c>d"e"');
76  });
77});
78 
79describe("recordValueText", () => {
80  it("renders stored per-type JSON values to display text", () => {
81    expect(recordValueText("TITLE", { text: "Hello <b>x</b>", marks: [] })).toBe("Hello x");
82    expect(recordValueText("NUMBER", { value: 42 })).toBe("42");
83    expect(recordValueText("SELECT", { optionId: 1 }, ["a", "b", "c"])).toBe("b");
84    expect(recordValueText("MULTI_SELECT", { optionIds: [0, 2] }, ["a", "b", "c"])).toBe("a, c");
85    expect(recordValueText("CHECKBOX", { value: true })).toBe("Yes");
86    expect(recordValueText("CHECKBOX", { value: false })).toBe("No");
87    expect(recordValueText("DATE", { start: "2026-01-01", end: null })).toBe("2026-01-01");
88    expect(recordValueText("PERSON", { userIds: ["user_abcdefghij"] })).toBe("@efghij");
89    expect(recordValueText("URL", { value: "https://x.dev", title: null })).toBe("https://x.dev");
90    expect(recordValueText("SELECT", null)).toBe("");
91  });
92});
93 
94describe("assembleContext — page", () => {
95  it("includes the page title + block text (plain, HTML-stripped)", async () => {
96    lvl.mockResolvedValue("VIEW");
97    const prisma = mockPrisma({
98      page: { id: "p1", workspaceId: ws, title: "My <b>Plan</b>" },
99      blocks: [
100        { id: "b1", type: "PARAGRAPH", content: { text: "First <em>block</em>" } },
101        { id: "b2", type: "HEADING_1", content: { text: "Section" } },
102      ],
103    });
104    const out = await assembleContext(prisma, ws, user, { pageId: "p1" });
105    expect(out.dropped).toEqual([]);
106    expect(out.systemPrompt).toContain("# Page: My Plan");
107    expect(out.systemPrompt).toContain("First block"); // HTML stripped
108    expect(out.systemPrompt).not.toContain("<em>");
109    expect(out.note).toBeUndefined();
110  });
111 
112  it("caps content at 40 blocks (truncation marker)", async () => {
113    lvl.mockResolvedValue("VIEW");
114    const blocks = Array.from({ length: 60 }, (_, i) => ({
115      id: `b${i}`,
116      type: "PARAGRAPH",
117      content: { text: `block ${i}` },
118    }));
119    const prisma = mockPrisma({
120      page: { id: "p1", workspaceId: ws, title: "Big" },
121      blocks,
122    });
123    const out = await assembleContext(prisma, ws, user, { pageId: "p1" });
124    expect(out.systemPrompt).toContain("block 0");
125    expect(out.systemPrompt).toContain(`block ${MAX_CONTEXT_BLOCKS - 1}`);
126    expect(out.systemPrompt).not.toContain(`block ${MAX_CONTEXT_BLOCKS}\n`);
127    expect(out.systemPrompt).toContain(`truncated at ${MAX_CONTEXT_BLOCKS} blocks`);
128  });
129 
130  it("surfaces a selected block first (blockId)", async () => {
131    lvl.mockResolvedValue("VIEW");
132    const prisma = mockPrisma({
133      page: { id: "p1", workspaceId: ws, title: "T" },
134      blocks: [
135        { id: "b1", type: "PARAGRAPH", content: { text: "alpha" } },
136        { id: "b2", type: "PARAGRAPH", content: { text: "the selected one" } },
137      ],
138    });
139    const out = await assembleContext(prisma, ws, user, { pageId: "p1", blockId: "b2" });
140    const selectedIdx = out.systemPrompt!.indexOf("## Selected block");
141    expect(selectedIdx).toBeGreaterThan(-1);
142    expect(out.systemPrompt!.slice(selectedIdx, selectedIdx + 60)).toContain("the selected one");
143  });
144 
145  it("drops the page (content excluded) when the requester lacks VIEW", async () => {
146    lvl.mockResolvedValue("NONE");
147    const prisma = mockPrisma({
148      page: { id: "p1", workspaceId: ws, title: "Secret" },
149      blocks: [{ id: "b1", type: "PARAGRAPH", content: { text: "top-secret-content" } }],
150    });
151    const out = await assembleContext(prisma, ws, user, { pageId: "p1" });
152    expect(out.dropped).toEqual(["page"]);
153    expect(out.note).toMatch(/unavailable/);
154    // The content must NOT leak into the prompt (even as a note).
155    expect(out.systemPrompt ?? "").not.toContain("top-secret-content");
156    expect(out.note).not.toContain("top-secret-content");
157    // No prompt body at all for a fully-dropped context.
158    expect(out.systemPrompt).toBeUndefined();
159  });
160 
161  it("drops silently (note set) when the page is missing / cross-workspace", async () => {
162    lvl.mockResolvedValue("VIEW");
163    const prisma = mockPrisma({}); // findFirst scoped to workspace → null
164    const out = await assembleContext(prisma, ws, user, { pageId: "other-ws-page" });
165    expect(out.dropped).toEqual(["page"]);
166    expect(out.note).toMatch(/unavailable/);
167    expect(out.systemPrompt).toBeUndefined();
168  });
169});
170 
171describe("assembleContext — database", () => {
172  const dbProps = [
173    { name: "Name", type: "TITLE", config: {} },
174    { name: "Status", type: "SELECT", config: { options: [{ name: "Todo" }, { name: "Done" }] } },
175    { name: "Amount", type: "NUMBER", config: {} },
176  ];
177  const mkRecords = (n: number) =>
178    Array.from({ length: n }, (_, i) => ({
179      position: `a${i}`,
180      properties: [
181        { property: dbProps[0]!, value: { text: `Rec ${i}`, marks: [] } },
182        { property: dbProps[1]!, value: { optionId: i % 2 } },
183        { property: dbProps[2]!, value: { value: i * 10 } },
184      ],
185    }));
186 
187  it("includes schema + up to 20 sample rows with option labels", async () => {
188    lvl.mockResolvedValue("VIEW");
189    const prisma = mockPrisma({
190      database: { id: "db1", workspaceId: ws, title: "Tasks" },
191      dbProps,
192      records: mkRecords(5),
193    });
194    const out = await assembleContext(prisma, ws, user, { databaseId: "db1" });
195    expect(out.systemPrompt).toContain("# Database: Tasks");
196    expect(out.systemPrompt).toContain("## Schema");
197    expect(out.systemPrompt).toContain("Name (TITLE)");
198    expect(out.systemPrompt).toContain("Status (SELECT)");
199    expect(out.systemPrompt).toContain("Rec 0");
200    expect(out.systemPrompt).toContain("Status: Todo"); // optionId 0 → label
201    expect(out.systemPrompt).toContain("Status: Done"); // optionId 1 → label
202    expect(out.systemPrompt).toContain("Amount: 10");
203  });
204 
205  it("caps rows at 20 (truncation marker)", async () => {
206    lvl.mockResolvedValue("VIEW");
207    const prisma = mockPrisma({
208      database: { id: "db1", workspaceId: ws, title: "Tasks" },
209      dbProps,
210      records: mkRecords(30),
211    });
212    const out = await assembleContext(prisma, ws, user, { databaseId: "db1" });
213    expect(out.systemPrompt).toContain("Rec 0");
214    expect(out.systemPrompt).toContain(`Rec ${MAX_CONTEXT_ROWS - 1}`);
215    expect(out.systemPrompt).not.toContain(`Rec ${MAX_CONTEXT_ROWS} `);
216    expect(out.systemPrompt).toContain(`truncated at ${MAX_CONTEXT_ROWS} rows`);
217  });
218 
219  it("drops the database when the requester lacks VIEW (no row leak)", async () => {
220    lvl.mockResolvedValue("NONE");
221    const prisma = mockPrisma({
222      database: { id: "db1", workspaceId: ws, title: "Secret DB" },
223      dbProps,
224      records: mkRecords(3),
225    });
226    const out = await assembleContext(prisma, ws, user, { databaseId: "db1" });
227    expect(out.dropped).toEqual(["database"]);
228    expect(out.systemPrompt).toBeUndefined();
229    expect(out.note).toMatch(/unavailable/);
230  });
231});
232 
233describe("assembleContext — combined", () => {
234  it("combines page + database sections and reports drops per resource", async () => {
235    lvl.mockResolvedValue("VIEW");
236    const dbProps = [{ name: "Name", type: "TITLE", config: {} }];
237    const prisma = mockPrisma({
238      page: { id: "p1", workspaceId: ws, title: "Page" },
239      blocks: [{ id: "b1", type: "PARAGRAPH", content: { text: "page body" } }],
240      database: null, // missing → dropped
241      dbProps,
242    });
243    const out = await assembleContext(prisma, ws, user, { pageId: "p1", databaseId: "dbX" });
244    expect(out.dropped).toEqual(["database"]);
245    expect(out.systemPrompt).toContain("# Page: Page");
246    expect(out.systemPrompt).toContain("page body");
247    expect(out.systemPrompt).not.toContain("# Database:");
248  });
249 
250  it("returns no prompt when nothing is bound", async () => {
251    lvl.mockResolvedValue("VIEW");
252    const prisma = mockPrisma({});
253    const out = await assembleContext(prisma, ws, user, {});
254    expect(out.systemPrompt).toBeUndefined();
255    expect(out.dropped).toEqual([]);
256  });
257 
258  it("includes the base system prompt export", () => {
259    expect(BASE_SYSTEM_PROMPT).toContain("workspace assistant");
260  });
261});
262 
263// ── TASK-0126 — the generatePlan tool spec + model-quirk normalization ──────
264 
265import { validateTools, parseGeneratePlan } from "@factory/ai";
266import {
267  GENERATE_PLAN_SYSTEM_PROMPT,
268  GENERATE_PLAN_TOOL,
269  normalizeGeneratePlanArgs,
270} from "../prompts.js";
271 
272describe("GENERATE_PLAN_TOOL", () => {
273  it("passes the gateway's strict tool-schema validator", () => {
274    // The same validator the AiGateway runs before any spend — the spec must
275    // stay inside the allowed JSON-Schema subset (no oneOf/$ref/pattern).
276    expect(validateTools([GENERATE_PLAN_TOOL]).ok).toBe(true);
277  });
278 
279  it("describes the four step names + uppercase block types", () => {
280    const params = GENERATE_PLAN_TOOL.parameters as {
281      properties: { tools: { items: { properties: { name: { enum: readonly string[] } } } } };
282    };
283    expect(params.properties.tools.items.properties.name.enum).toEqual([
284      "createPage",
285      "insertBlocks",
286      "createDatabase",
287      "createRelation",
288    ]);
289    expect(GENERATE_PLAN_TOOL.description).toContain("HEADING_2");
290    expect(GENERATE_PLAN_TOOL.description).toContain("$page"); // the "never use this" warning
291  });
292});
293 
294describe("GENERATE_PLAN_SYSTEM_PROMPT", () => {
295  it("forces the tool call and the {type, content:{text}} block shape", () => {
296    expect(GENERATE_PLAN_SYSTEM_PROMPT).toContain("generatePlan tool exactly once");
297    expect(GENERATE_PLAN_SYSTEM_PROMPT).toContain("content is { text:");
298    expect(GENERATE_PLAN_SYSTEM_PROMPT).toContain("UPPERCASE");
299  });
300});
301 
302describe("normalizeGeneratePlanArgs (OSS model-quirk repair)", () => {
303  it("repairs tools double-serialized as a JSON string + $page placeholder → a VALID plan", () => {
304    // The exact shapes observed from qwen3.8-27B (20,764-char dump,
305    // /tmp/ai-tool-args.json): tools is a STRING, the first page is targeted
306    // via pageRef:"$page", block types are lowercase + `text` instead of
307    // `content:{text}` (the executor uppercases types; `text` is dropped by
308    // the schema — content defaults to {}).
309    const raw = {
310      tools: JSON.stringify([
311        { name: "createPage", args: { title: "MySQL 8.0 → 8.4 Migration Plan" } },
312        {
313          name: "insertBlocks",
314          args: {
315            pageRef: "$page",
316            blocks: [{ type: "paragraph", text: "Step one" }, { type: "HEADING_2", content: { text: "Phases" } }],
317          },
318        },
319        { name: "createDatabase", args: { ref: "tasks", pageRef: "$page", title: "Tasks", properties: [{ name: "Status", type: "SELECT", options: [{ name: "To Do" }] }, { name: "Phase", type: "SELECT", options: [{ name: "To Do" }] }] } },
320      ]),
321    };
322    const out = normalizeGeneratePlanArgs(raw) as Record<string, unknown>;
323    expect(Array.isArray(out.tools)).toBe(true);
324    const parsed = parseGeneratePlan(out);
325    expect(parsed.ok).toBe(true);
326    if (!parsed.ok) return;
327    // The $page placeholders were dropped (the executor treats a missing
328    // pageRef as "the first page"); the valid 'tasks' ref survived.
329    const insert = parsed.plan.tools[1] as { name: string; args: { pageRef?: string } };
330    expect(insert.name).toBe("insertBlocks");
331    expect(insert.args.pageRef).toBeUndefined();
332    const db = parsed.plan.tools[2] as { name: string; args: { ref: string; pageRef?: string } };
333    expect(db.args.ref).toBe("tasks");
334    expect(db.args.pageRef).toBeUndefined();
335    // The input was not mutated.
336    expect(typeof raw.tools).toBe("string");
337  });
338 
339  it("hoists block text into content.{text} (top-level text, empty content, bare-string content)", () => {
340    // The model's observed quirk: block text lives OUTSIDE content — as a
341    // top-level `text`, with `content: {}` (schema default), or as a bare
342    // string (the web editor's round-trip shape). Without the repair the
343    // schema's `content` default `{}` silently created EMPTY blocks.
344    const out = normalizeGeneratePlanArgs({
345      tools: [
346        {
347          name: "insertBlocks",
348          args: {
349            blocks: [
350              { type: "paragraph", text: "top-level text" }, // → hoist
351              { type: "HEADING_2", content: {}, text: "empty content + text" }, // → merge
352              { type: "quote", content: "bare string" }, // → { text }
353              { type: "PARAGRAPH", content: { text: "already canonical" } }, // → untouched
354              { type: "CODE", content: { code: "x", language: "sql" }, text: "structured wins" }, // → untouched
355            ],
356          },
357        },
358        { name: "createPage", args: { title: "P", blocks: [{ type: "PARAGRAPH", text: "inline" }] } },
359      ],
360    }) as { tools: Array<{ args: { blocks?: Array<Record<string, unknown>> } }> };
361    const b = out.tools[0]!.args.blocks as Array<Record<string, unknown>>;
362    expect(b[0]).toEqual({ type: "paragraph", content: { text: "top-level text" } });
363    expect(b[1]).toEqual({ type: "HEADING_2", content: { text: "empty content + text" } });
364    expect(b[2]).toEqual({ type: "quote", content: { text: "bare string" } });
365    expect(b[3]).toEqual({ type: "PARAGRAPH", content: { text: "already canonical" } });
366    expect(b[4]).toEqual({ type: "CODE", content: { code: "x", language: "sql" }, text: "structured wins" });
367    // createPage's inline blocks are repaired too.
368    expect(out.tools[1]!.args.blocks).toEqual([{ type: "PARAGRAPH", content: { text: "inline" } }]);
369  });
370 
371  it("drops known placeholder refs case-insensitively, keeps valid refs", () => {
372    const out = normalizeGeneratePlanArgs({
373      tools: [
374        { name: "insertBlocks", args: { pageRef: "$PAGE", blocks: [{ type: "PARAGRAPH", content: { text: "a" } }] } },
375        { name: "insertBlocks", args: { pageRef: "phase1", blocks: [{ type: "PARAGRAPH", content: { text: "b" } }] } },
376        { name: "createRelation", args: { databaseRef: "root", targetPageRef: "phase1", name: "links" } },
377      ],
378    }) as { tools: Array<{ args: Record<string, unknown> }> };
379    expect(out.tools[0]!.args.pageRef).toBeUndefined();
380    expect(out.tools[1]!.args.pageRef).toBe("phase1");
381    expect(out.tools[2]!.args.databaseRef).toBeUndefined();
382    expect(out.tools[2]!.args.targetPageRef).toBe("phase1");
383  });
384 
385  it("drops the SEEDED db properties (Title/Status) a createDatabase step re-declares, keeps the rest", () => {
386    // The database service always ships Title + Status; validateCreateDatabase
387    // rejects the whole plan on a name collision (observed 2025: the model
388    // emitted a Status property → createDatabase step failed). The repair
389    // drops the seeded names and keeps the user's custom columns.
390    const out = normalizeGeneratePlanArgs({
391      tools: [
392        { name: "createPage", args: { ref: "plan", title: "P" } },
393        {
394          name: "createDatabase",
395          args: {
396            ref: "tasks",
397            pageRef: "plan",
398            title: "Tasks",
399            properties: [
400              { name: "Status", type: "STATUS", options: [{ name: "To Do" }, { name: "Done" }] }, // seeded → drop
401              { name: "Title", type: "RICH_TEXT" }, // seeded → drop
402              { name: "Phase", type: "STATUS", options: [{ name: "To Do" }, { name: "Done" }] }, // custom → keep
403              { name: "Due", type: "DATE" }, // custom → keep
404            ],
405          },
406        },
407      ],
408    }) as { tools: Array<{ args: { properties?: Array<Record<string, unknown>> } }> };
409    const props = out.tools[1]!.args.properties;
410    expect(props?.map((p) => p.name)).toEqual(["Phase", "Due"]);
411    // The result still validates.
412    expect(parseGeneratePlan(out).ok).toBe(true);
413  });
414 
415  it("does not hide real schema errors (unknown tool name still fails)", () => {
416    const out = normalizeGeneratePlanArgs({
417      tools: [{ name: "deletePage", args: { ref: "root" } }],
418    });
419    expect(parseGeneratePlan(out).ok).toBe(false);
420  });
421 
422  it("leaves unparseable tools-strings untouched (the schema rejects them)", () => {
423    const raw = { tools: "{not json" };
424    const out = normalizeGeneratePlanArgs(raw) as Record<string, unknown>;
425    expect(out.tools).toBe("{not json");
426    expect(parseGeneratePlan(out).ok).toBe(false);
427  });
428 
429  it("passes non-object inputs through unchanged", () => {
430    expect(normalizeGeneratePlanArgs(undefined)).toBe(undefined);
431    expect(normalizeGeneratePlanArgs(null)).toBe(null);
432    expect(normalizeGeneratePlanArgs("str")).toBe("str");
433    expect(normalizeGeneratePlanArgs([1, 2])).toEqual([1, 2]);
434  });
435});
436