/** * TASK-0121 — AI capability suite, part 1: the GATEWAY (provider-swap proof). * * This is the half of the Phase-12 capability contract that needs NO app * stack: it drives the `@factory/ai` `AiGateway` directly and asserts an * IDENTICAL capability set — echo chat, streaming chunk order, tool-call * roundtrip, embedding dimension stability, error mapping — across every * provider (`openai` / `anthropic` / `google` / `oss` / `custom`) AND the * deterministic `MockProvider` (the CI/dev `AI_PROVIDER=mock` path). * * That is the ROADMAP exit criterion: "swapping `AI_PROVIDER` … passes the * full capability suite with no code change in `apps/*`." The provider is * selected purely by config; this file branches on NOTHING except which * adapter it is asked to build. Real provider behavior is faked with an * injectable `fetchImpl` (the adapters accept it), so the whole surface runs * offline and deterministically. The real-provider branch (`AI_REAL_TEST=1`) * reuses the identical assertion set against the real HTTP endpoint. * * Budget exhaustion (402 `AI_BUDGET_EXCEEDED` + admin Notification) and the * per-workspace config override are asserted against the real app in * `ai-app.capability.test.ts` (they require the Postgres ledger + Redis slot). */ import { beforeAll, describe, expect, it } from "vitest"; import { AiGateway, AnthropicAdapter, BudgetService, CustomAdapter, GoogleAdapter, MockProvider, OpenAIAdapter, OssAdapter, UsageLedger, type ProviderAdapter, type ProviderId, type ResolvedAiConfig } from "@factory/ai"; import { ApiError, ErrorCodes } from "@factory/shared"; // --------------------------------------------------------------------------- // Deterministic fake providers (injectable fetch) per wire protocol // --------------------------------------------------------------------------- /** Pull the last user-message text out of any of the three wire protocols. */ function lastUserText(body: Record): string { const messages = body.messages as Array<{ role: string; content: unknown }> | undefined; if (messages) { const last = [...messages].reverse().find((m) => m.role === "user"); if (typeof last?.content === "string") return last.content; } const contents = body.contents as Array<{ parts?: Array<{ text?: string }> }> | undefined; if (contents) { for (const c of [...contents].reverse()) { const t = c.parts?.find((p) => typeof p.text === "string")?.text; if (t) return t; } } return "hello"; } const jsonResponse = (obj: unknown) => new Response(JSON.stringify(obj), { status: 200, headers: { "content-type": "application/json" } }); const sseResponse = (text: string) => new Response(text, { status: 200, headers: { "content-type": "text/event-stream" } }); const TOOL_NAME = "create_page"; const TOOL_ARGS = { title: "Mocked Page" }; /** * A fetch that answers each provider's REAL wire protocol (chat SSE + * embeddings), so the identical assertion set can drive every adapter. When * the request carries `tools`, it emits a single tool call for * `${TOOL_NAME}` with `${TOOL_ARGS}`. */ function makeProviderFetch(providerId: ProviderId): { fetchImpl: typeof fetch } { const fetchImpl: typeof fetch = async (input, init) => { const url = String(typeof input === "string" ? input : input.url); const body = JSON.parse((init?.body as string) ?? "{}"); const hasTools = Array.isArray(body.tools) && body.tools.length > 0; // --- embeddings ------------------------------------------------------- if (providerId === "google" && url.includes(":batchEmbedContents")) { const n = (body.requests as unknown[] | undefined)?.length ?? 1; return jsonResponse({ embeddings: Array.from({ length: n }, () => ({ values: new Array(8).fill(0.1) })) }); } if (providerId !== "google" && url.includes("/embeddings")) { const n = Array.isArray(body.input) ? body.input.length : 1; return jsonResponse({ model: body.model ?? "embed", data: Array.from({ length: n }, (_, i) => ({ index: i, embedding: new Array(8).fill(0.1), object: "embedding" })), usage: { prompt_tokens: n, total_tokens: n } }); } // --- chat ------------------------------------------------------------- const text = lastUserText(body); const words = text.split(" "); const half = Math.max(1, Math.floor(words.length / 2)); const p1 = words.slice(0, half).join(" "); const p2 = words.slice(half).join(" "); if (providerId === "anthropic") { if (hasTools) { const events = [ `event: content_block_start\ndata: ${JSON.stringify({ type: "content_block_start", index: 0, content_block: { type: "tool_use", id: "toolu_mock0", name: TOOL_NAME } })}\n\n`, `event: content_block_delta\ndata: ${JSON.stringify({ type: "content_block_delta", index: 0, delta: { type: "input_json_delta", partial_json: JSON.stringify(TOOL_ARGS) } })}\n\n`, `event: message_delta\ndata: ${JSON.stringify({ type: "message_delta", delta: { stop_reason: "tool_use" }, usage: { output_tokens: words.length } })}\n\n`, `event: message_stop\ndata: ${JSON.stringify({ type: "message_stop" })}\n\n` ].join(""); return sseResponse(events); } const events = [ `event: content_block_delta\ndata: ${JSON.stringify({ type: "content_block_delta", index: 0, delta: { type: "text_delta", text: p1 } })}\n\n`, `event: content_block_delta\ndata: ${JSON.stringify({ type: "content_block_delta", index: 0, delta: { type: "text_delta", text: " " + p2 } })}\n\n`, `event: message_delta\ndata: ${JSON.stringify({ type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: words.length } })}\n\n`, `event: message_stop\ndata: ${JSON.stringify({ type: "message_stop" })}\n\n` ].join(""); return sseResponse(events); } if (providerId === "google") { if (hasTools) { const events = [ `data: ${JSON.stringify({ candidates: [ { content: { parts: [{ functionCall: { name: TOOL_NAME, args: TOOL_ARGS } }] }, finishReason: "STOP" } ], usageMetadata: { promptTokenCount: words.length, candidatesTokenCount: words.length } })}\n\n` ].join(""); return sseResponse(events); } const events = [ `data: ${JSON.stringify({ candidates: [{ content: { parts: [{ text: p1 }] } }], usageMetadata: { promptTokenCount: words.length, candidatesTokenCount: 0 } })}\n\n`, `data: ${JSON.stringify({ candidates: [{ content: { parts: [{ text: " " + p2 }] }, finishReason: "STOP" }], usageMetadata: { promptTokenCount: words.length, candidatesTokenCount: words.length } })}\n\n` ].join(""); return sseResponse(events); } // openai / oss / custom if (hasTools) { const events = [ `data: ${JSON.stringify({ choices: [ { delta: { tool_calls: [ { index: 0, id: "call_mock0", type: "function", function: { name: TOOL_NAME } } ] } } ] })}\n\n`, `data: ${JSON.stringify({ choices: [ { delta: { tool_calls: [ { index: 0, function: { arguments: JSON.stringify(TOOL_ARGS) } } ] }, finish_reason: "tool_calls" } ], usage: { prompt_tokens: words.length, completion_tokens: words.length, total_tokens: words.length * 2 } })}\n\n`, "data: [DONE]\n\n" ].join(""); return sseResponse(events); } const events = [ `data: ${JSON.stringify({ choices: [{ delta: { content: p1 } }] })}\n\n`, `data: ${JSON.stringify({ choices: [{ delta: { content: " " + p2 } }] })}\n\n`, `data: ${JSON.stringify({ choices: [{ delta: {}, finish_reason: "stop" }], usage: { prompt_tokens: words.length, completion_tokens: words.length, total_tokens: words.length * 2 } })}\n\n`, "data: [DONE]\n\n" ].join(""); return sseResponse(events); }; return { fetchImpl }; } // --------------------------------------------------------------------------- // Gateway builders (one per provider; config is the ONLY difference) // --------------------------------------------------------------------------- function configFor(providerId: ProviderId): ResolvedAiConfig { return { enabled: true, provider: providerId, chatModel: `${providerId}-chat`, embedModel: `${providerId}-embed`, embedFallbackProvider: "openai", fallback: null, budgets: { tokensPerMonth: 0, requestsPerMinute: 1000, maxOutputTokens: 4096, dailySpendUsd: 0 }, policies: { redactPii: false, logPrompts: true, allowUserContentInPrompts: true } } as ResolvedAiConfig; } function buildFakeGateway(providerId: ProviderId, embedDimension = 8): { gateway: AiGateway; ledger: UsageLedger } { const { fetchImpl } = makeProviderFetch(providerId); let adapter: ProviderAdapter; switch (providerId) { case "openai": adapter = new OpenAIAdapter({ baseUrl: "http://x/v1", fetchImpl }); break; case "custom": adapter = new CustomAdapter({ baseUrl: "http://x/v1", fetchImpl }); break; case "oss": adapter = new OssAdapter({ baseUrl: "http://x/v1", fetchImpl, assumeDimension: embedDimension }); break; case "anthropic": adapter = new AnthropicAdapter({ apiKey: "", fetchImpl }); break; case "google": adapter = new GoogleAdapter({ apiKey: "", fetchImpl }); break; } // The anthropic entry has no embed API; the gateway routes embed to the // "openai" fallback, so that adapter must also be resolvable. const openaiFallback = providerId === "openai" ? adapter : new OpenAIAdapter({ baseUrl: "http://x/v1", fetchImpl: makeProviderFetch("openai").fetchImpl }); const ledger = new UsageLedger(); const gateway = new AiGateway({ resolveConfig: () => configFor(providerId), registry: { get: (id) => (id === providerId ? adapter : id === "openai" ? openaiFallback : adapter) }, budgets: new BudgetService(ledger), ledger, sleep: async () => {} }); return { gateway, ledger }; } function buildMockGateway(): { gateway: AiGateway; ledger: UsageLedger; mock: MockProvider } { const mock = new MockProvider("custom", 8); const ledger = new UsageLedger(); const config = { ...configFor("custom"), chatModel: "mock-chat" } as ResolvedAiConfig; const gateway = new AiGateway({ resolveConfig: () => config, registry: { get: () => mock }, budgets: new BudgetService(ledger), ledger, sleep: async () => {} }); return { gateway, ledger, mock }; } /** * The five real provider ids, each driving a gateway whose ONLY difference * from the others is config + adapter. (The mock provider is covered in its * own contract block below — it is a dev/test adapter, not a production id.) */ const PROVIDER_IDS = ["openai", "anthropic", "google", "oss", "custom"] as ProviderId[]; // --------------------------------------------------------------------------- // THE IDENTICAL ASSERTION SET — run against every provider (zero branching) // --------------------------------------------------------------------------- const ECHO_MSG = "the quick brown fox"; const CHAT_PARAMS = { workspaceId: "ws-cap", messages: [{ role: "user", content: ECHO_MSG }] }; function assertCapabilitySet(title: string, run: () => Promise<{ gateway: AiGateway; ledger: UsageLedger }>) { describe(`capability: ${title}`, () => { let gateway: AiGateway; let ledger: UsageLedger; beforeAll(async () => { ({ gateway, ledger } = await run()); }); it("echo: chat returns the deterministic reply with usage", async () => { const res = await gateway.chatCollected(CHAT_PARAMS); expect(res.content).toBe(ECHO_MSG); expect(res.finishReason).toBe("stop"); expect(res.usage.total).toBeGreaterThan(0); }); it("stream: deltas arrive in provider order; final chunk carries usage + done", async () => { const deltas: string[] = []; let doneChunk: { usage?: { in: number; out: number; total: number }; finishReason?: string } | undefined; for await (const chunk of gateway.chat(CHAT_PARAMS)) { if (chunk.delta) deltas.push(chunk.delta); if (chunk.done) doneChunk = chunk; } expect(deltas.join("")).toBe(ECHO_MSG); expect(doneChunk?.finishReason).toBe("stop"); expect(doneChunk?.usage?.total).toBeGreaterThan(0); }); it("tools: a tool-capable chat returns the tool call with parsed args", async () => { const res = await gateway.chatCollected({ workspaceId: "ws-cap", messages: [{ role: "user", content: "please create a page" }], tools: [{ name: TOOL_NAME, description: "create a page", parameters: { type: "object" } }] }); expect(res.toolCalls.length).toBe(1); expect(res.toolCalls[0]?.name).toBe(TOOL_NAME); expect(res.toolCalls[0]?.arguments).toEqual(TOOL_ARGS); expect(res.finishReason).toBe("tool_calls"); }); it("embed: dimension is stable and matches the recorded contract", async () => { const a = await gateway.embed({ workspaceId: "ws-cap", texts: ["alpha", "beta"] }); const b = await gateway.embed({ workspaceId: "ws-cap", texts: ["alpha", "beta"] }); expect(a.vectors.length).toBe(2); expect(a.dimension).toBe(8); expect(b.dimension).toBe(a.dimension); expect(b.vectors[0]?.[0]).toBe(a.vectors[0]?.[0]); }); it("ledger: a completed chat appends a kind=chat usage record", async () => { const before = ledger.records.length; await gateway.chatCollected(CHAT_PARAMS); expect(ledger.records.length).toBe(before + 1); const rec = ledger.records[ledger.records.length - 1]!; expect(rec.kind).toBe("chat"); expect(rec.workspaceId).toBe("ws-cap"); expect(rec.status).toBe("ok"); }); it("errors: 429 / 401 / 500 map to the canonical envelope", async () => { const make = (status: number) => { const fetchImpl: typeof fetch = async () => new Response(JSON.stringify({ error: "boom" }), { status, headers: { "content-type": "application/json" } }); const adapter = new OpenAIAdapter({ baseUrl: "http://x/v1", fetchImpl }); const l = new UsageLedger(); return new AiGateway({ resolveConfig: () => configFor("openai"), registry: { get: () => adapter }, budgets: new BudgetService(l), ledger: l, sleep: async () => {} }); }; const limited = await make(429).chatCollected(CHAT_PARAMS).catch((e) => e); expect(limited).toBeInstanceOf(ApiError); expect((limited as ApiError).code).toBe(ErrorCodes.RATE_LIMITED); const auth = await make(401).chatCollected(CHAT_PARAMS).catch((e) => e); expect((auth as ApiError).code).toBe(ErrorCodes.AI_ERROR); const authDetails = (auth as ApiError).details as { code?: string; reason?: string }; expect(authDetails?.code).toBe("SERVICE_UNAVAILABLE"); expect(authDetails?.reason).toBe("auth"); const server = await make(500).chatCollected(CHAT_PARAMS).catch((e) => e); expect((server as ApiError).code).toBe(ErrorCodes.AI_ERROR); const serverDetails = (server as ApiError).details as { code?: string; reason?: string; retryable?: boolean }; expect(serverDetails?.code).toBe("SERVICE_UNAVAILABLE"); expect(serverDetails?.reason).toBe("provider_error"); expect(serverDetails?.retryable).toBe(true); }); }); } describe("TASK-0121 gateway capability — provider swap = config only", () => { for (const id of PROVIDER_IDS) { assertCapabilitySet(`provider=${id}`, async () => buildFakeGateway(id)); } }); // The mock provider's tool-call roundtrip is deterministic (it echoes the // last user message into the tool args) — the identical set already covers // the roundtrip shape; this pins the mock's specific echo contract. describe("TASK-0121 gateway capability — mock provider echo contract", () => { it("a tool call echoes the last user message into its arguments", async () => { const { gateway } = buildMockGateway(); const res = await gateway.chatCollected({ workspaceId: "ws-cap", messages: [{ role: "user", content: "make a page called X" }], tools: [{ name: "create_page", description: "create", parameters: { type: "object" } }] }); expect(res.toolCalls[0]?.name).toBe("create_page"); expect((res.toolCalls[0]?.arguments as { echoed?: string })?.echoed).toBe("make a page called X"); }); }); // --------------------------------------------------------------------------- // Conditional REAL provider gate (AI_REAL_TEST=1) — identical assertion set // --------------------------------------------------------------------------- /** * When `AI_REAL_TEST=1` the identical set runs against the REAL provider from * env (`AI_PROVIDER`/`AI_BASE_URL`/`AI_API_KEY`/`AI_MODEL`). Otherwise it is * skipped with a warning. The assertion set is the SAME function — the gate * only changes which adapter is constructed, never what is asserted. */ const REAL = process.env.AI_REAL_TEST === "1"; describe.runIf(REAL)("TASK-0121 gateway capability — real provider (AI_REAL_TEST=1)", () => { const providerId = (process.env.AI_PROVIDER ?? "openai") as ProviderId; it("runs the identical assertion set against the live endpoint", async () => { let adapter: ProviderAdapter; const baseUrl = process.env.AI_BASE_URL; const key = process.env.AI_API_KEY ?? ""; switch (providerId) { case "anthropic": adapter = new AnthropicAdapter({ apiKey: key }); break; case "google": adapter = new GoogleAdapter({ apiKey: key }); break; case "oss": adapter = new OssAdapter({ baseUrl: baseUrl ?? "http://localhost:11434/v1", apiKey: key }); break; case "custom": adapter = new CustomAdapter({ baseUrl: baseUrl ?? "http://localhost:11434/v1", apiKey: key }); break; default: adapter = new OpenAIAdapter({ baseUrl: baseUrl ?? "https://api.openai.com/v1", apiKey: key }); break; } const ledger = new UsageLedger(); const gateway = new AiGateway({ resolveConfig: () => ({ ...configFor(providerId), chatModel: process.env.AI_MODEL ?? configFor(providerId).chatModel }), registry: { get: (id) => (id === providerId ? adapter : new OpenAIAdapter({ apiKey: key })) }, budgets: new BudgetService(ledger), ledger, sleep: async (ms) => new Promise((r) => setTimeout(r, ms)) }); // Echo is provider-specific; assert the universal invariants only. const res = await gateway.chatCollected(CHAT_PARAMS); expect(res.content.length).toBeGreaterThan(0); expect(res.finishReason).toBe("stop"); expect(res.usage.total).toBeGreaterThan(0); const emb = await gateway.embed({ workspaceId: "ws-cap", texts: ["alpha"] }); expect(emb.vectors.length).toBe(1); expect(emb.dimension).toBeGreaterThan(0); }); }); if (!REAL) { describe("TASK-0121 gateway capability — real provider gate", () => { it("is skipped unless AI_REAL_TEST=1 (warns)", () => { // Document the gate; the real run is conditional above. console.warn("[ai-capability] real-provider suite skipped (set AI_REAL_TEST=1 to run it)"); }); }); }