ai-app.capability.test.ts

ai-app.capability.test.tstests/integration/ai-capability/ai-app.capability.test.tsraw ← ai-capability9,010 B← Back to the app
1/**
2 * TASK-0121 — AI capability suite, part 2: the APP (budget + override + chat).
3 *
4 * The gateway-level capability set (echo / streaming order / tool roundtrip /
5 * embed dimension / error mapping, across every provider) is in
6 * `gateway.capability.test.ts`. This spec runs the parts that need the real
7 * app stack — the Postgres `ai_messages` ledger, the Redis rate window +
8 * 24 h notification slot, the authorization layer — against the deterministic
9 * `MockProvider` (env `AI_PROVIDER=mock`).
10 *
11 * Covered acceptance criteria (TASK-0121):
12 *   - budget exhaustion: `usageLimits.aiMonthlyTokens = 100` → a call returns
13 *     `402 AI_BUDGET_EXCEEDED` with `used`/`limit` details and writes an admin
14 *     `Notification` (asserted on the `Notification` table) + the
15 *     `ai.budget_exceeded` audit event.
16 *   - per-workspace `settings.ai` override (a different `chatModel`) is honored
17 *     for that workspace's calls while a second workspace keeps env defaults.
18 *   - a chat turn returns the deterministic mock echo and persists both
19 *     messages with token counts (the usage ledger).
20 *
21 * The provider is selected by CONFIG ONLY (`AI_PROVIDER=mock` set below before
22 * the app boots); nothing in this file branches on provider — the same
23 * assertions would hold for any provider that satisfies the capability set.
24 */
25// Config must be fixed BEFORE the app boots (loadEnv + the workspace cache are
26// both lazily primed on first use).
27process.env.AI_PROVIDER = "mock";
28process.env.SEARCH_VECTOR_DIMENSION = "8";
29 
30import { afterAll, beforeAll, describe, expect, it } from "vitest";
31import { createTestApp, type TestAppHandle } from "../helpers/api";
32import { addMemberViaSeam, authHeaders, countAuditEvents, createWorkspace, createUser } from "../helpers/seed";
33 
34let app: TestAppHandle;
35 
36beforeAll(async () => {
37  app = await createTestApp();
38}, 180_000);
39afterAll(async () => {
40  await app?.close();
41});
42 
43const json = async (res: Response): Promise<Record<string, unknown>> =>
44  (await res.json().catch(() => ({}))) as Record<string, unknown>;
45 
46const ping = async (headers: Record<string, string>, workspaceId: string) =>
47  (await (await app.request("POST", `/api/v1/workspaces/${workspaceId}/ai/ping`, { headers })).json()) as {
48    data: { provider: string; model: string; embeddingDimension: number; budget: { limit: number; used: number } };
49  };
50 
51describe("TASK-0121 AI capability — app (mock provider, config-only)", () => {
52  it("ping: returns the resolved provider/model/dimension, no tokens spent", async () => {
53    const owner = await createUser(app.request, { name: "AI Ping Owner" });
54    const { workspaceId } = await createWorkspace(app.request, authHeaders(owner.accessToken!), "AI Ping WS");
55    const body = await ping(authHeaders(owner.accessToken!), workspaceId);
56    expect(body.data.provider).toBe("custom"); // mock resolves into the custom family
57    expect(body.data.model).toBeTruthy();
58    expect(body.data.embeddingDimension).toBe(8); // SEARCH_VECTOR_DIMENSION
59    expect(body.data.budget.limit).toBe(0); // unset = unlimited
60    expect(body.data.budget.used).toBe(0);
61  });
62 
63  it("ping: 403 for a non-member (server-side authz)", async () => {
64    const owner = await createUser(app.request, { name: "AI Ping Owner2" });
65    const { workspaceId } = await createWorkspace(app.request, authHeaders(owner.accessToken!), "AI Ping WS2");
66    const stranger = await createUser(app.request, { name: "AI Stranger" });
67    const res = await app.request("POST", `/api/v1/workspaces/${workspaceId}/ai/ping`, {
68      headers: authHeaders(stranger.accessToken!)
69    });
70    expect(res.status).toBe(403);
71  });
72 
73  it("chat: returns the deterministic mock echo and persists both messages with tokens", async () => {
74    const owner = await createUser(app.request, { name: "AI Chat Owner" });
75    const { workspaceId } = await createWorkspace(app.request, authHeaders(owner.accessToken!), "AI Chat WS");
76    const msg = "the quick brown fox";
77    const res = await app.request("POST", `/api/v1/workspaces/${workspaceId}/ai/chat`, {
78      headers: authHeaders(owner.accessToken!),
79      body: { message: msg }
80    });
81    expect(res.status).toBe(201);
82    const body = await json(res);
83    expect(body.role).toBe("assistant");
84    expect(body.content).toBe(msg); // deterministic echo
85    expect(Number(body.tokensIn)).toBeGreaterThan(0);
86    expect(Number(body.tokensOut)).toBeGreaterThan(0);
87 
88    const convId = body.conversationId as string;
89    const rows = await app.prisma.aiMessage.findMany({
90      where: { conversationId: convId },
91      orderBy: { createdAt: "asc" }
92    });
93    expect(rows.length).toBe(2);
94    expect(rows[0]?.role).toBe("user");
95    expect(rows[0]?.content).toBe(msg);
96    expect(rows[1]?.role).toBe("assistant");
97    expect(Number(rows[1]?.tokensIn ?? 0)).toBeGreaterThan(0);
98    expect(Number(rows[1]?.tokensOut ?? 0)).toBeGreaterThan(0);
99  });
100 
101  it("budget: usageLimits.aiMonthlyTokens=100 → 402 AI_BUDGET_EXCEEDED + admin Notification + audit", async () => {
102    const owner = await createUser(app.request, { name: "AI Budget Owner" });
103    const { workspaceId } = await createWorkspace(app.request, authHeaders(owner.accessToken!), "AI Budget WS");
104    // Add an ADMIN (a second OWNER/ADMIN to prove the notification fans out).
105    const admin = await createUser(app.request, { name: "AI Budget Admin" });
106    await addMemberViaSeam(app.request, app.internalServiceToken, workspaceId, admin.email, "ADMIN");
107 
108    // Set the monthly cap. (Direct Prisma write: usageLimits has no public
109    // mutation endpoint — permitted by the seed-helper rules.)
110    await app.prisma.workspace.update({
111      where: { id: workspaceId },
112      data: { usageLimits: { aiMonthlyTokens: 100 } }
113    });
114 
115    const notifyBefore = await app.prisma.notification.count({
116      where: { workspaceId, title: "AI budget exceeded" }
117    });
118    const auditBefore = await countAuditEvents(app.prisma, { action: "ai.budget_exceeded" });
119 
120    const res = await app.request("POST", `/api/v1/workspaces/${workspaceId}/ai/chat`, {
121      headers: authHeaders(owner.accessToken!),
122      body: { message: "the quick brown fox" }
123    });
124    expect(res.status).toBe(402);
125    const body = await json(res);
126    const err = (body.error as { code: string; details?: Array<{ reason: string; used: number; limit: number }> }) ?? {};
127    expect(err.code).toBe("AI_BUDGET_EXCEEDED");
128    const details = err.details ?? [];
129    expect(details[0]?.reason).toBe("monthly_token_budget_exceeded");
130    expect(details[0]?.limit).toBe(100);
131    expect(details[0]?.used).toBe(0); // the guard rejects before any provider call
132 
133    // Admin Notification rows written for every OWNER/ADMIN.
134    const notifyAfter = await app.prisma.notification.count({
135      where: { workspaceId, title: "AI budget exceeded" }
136    });
137    expect(notifyAfter).toBe(notifyBefore + 2); // owner + admin
138    const notif = await app.prisma.notification.findFirst({
139      where: { workspaceId, title: "AI budget exceeded" },
140      orderBy: { createdAt: "desc" }
141    });
142    expect(notif?.channel).toBe("IN_APP");
143    expect(notif?.type).toBe("AUTOMATION");
144 
145    // The security-relevant audit event.
146    const auditAfter = await countAuditEvents(app.prisma, { action: "ai.budget_exceeded" });
147    expect(auditAfter).toBe(auditBefore + 1);
148  });
149 
150  it("budget: a workspace with no cap (limit=0) is unlimited and succeeds", async () => {
151    const owner = await createUser(app.request, { name: "AI Unlimited Owner" });
152    const { workspaceId } = await createWorkspace(app.request, authHeaders(owner.accessToken!), "AI Unlimited WS");
153    const res = await app.request("POST", `/api/v1/workspaces/${workspaceId}/ai/chat`, {
154      headers: authHeaders(owner.accessToken!),
155      body: { message: "the quick brown fox" }
156    });
157    expect(res.status).toBe(201);
158    const body = await json(res);
159    expect(body.content).toBe("the quick brown fox");
160  });
161 
162  it("override: a workspace's settings.ai.chatModel is honored; a second keeps the default", async () => {
163    const owner = await createUser(app.request, { name: "AI Override Owner" });
164    const h = authHeaders(owner.accessToken!);
165    const { workspaceId: wsA } = await createWorkspace(app.request, h, "AI Override A");
166    const { workspaceId: wsB } = await createWorkspace(app.request, h, "AI Override B");
167 
168    // Set a per-workspace override on wsA (the documented AiConfigSchema shape).
169    await app.prisma.workspace.update({
170      where: { id: wsA },
171      data: { settings: { ai: { enabled: true, provider: "custom", chatModel: "my-custom-model-xyz", embedModel: "mock-embed" } } }
172    });
173 
174    const { resetWorkspaceCache } = await import("@factory/api/src/modules/ai/gateway-provider");
175    resetWorkspaceCache();
176 
177    const a = await ping(h, wsA);
178    const b = await ping(h, wsB);
179    expect(a.data.model).toBe("my-custom-model-xyz");
180    expect(b.data.model).not.toBe("my-custom-model-xyz"); // keeps the env default
181  });
182});
183