1
23import { beforeAll, describe, expect, it } from "vitest";
24import {
25 AiGateway,
26 AnthropicAdapter,
27 BudgetService,
28 CustomAdapter,
29 GoogleAdapter,
30 MockProvider,
31 OpenAIAdapter,
32 OssAdapter,
33 UsageLedger,
34 type ProviderAdapter,
35 type ProviderId,
36 type ResolvedAiConfig
37} from "@factory/ai";
38import { ApiError, ErrorCodes } from "@factory/shared";
39
40
41
42
43
44
45function lastUserText(body: Record<string, unknown>): string {
46 const messages = body.messages as Array<{ role: string; content: unknown }> | undefined;
47 if (messages) {
48 const last = [...messages].reverse().find((m) => m.role === "user");
49 if (typeof last?.content === "string") return last.content;
50 }
51 const contents = body.contents as Array<{ parts?: Array<{ text?: string }> }> | undefined;
52 if (contents) {
53 for (const c of [...contents].reverse()) {
54 const t = c.parts?.find((p) => typeof p.text === "string")?.text;
55 if (t) return t;
56 }
57 }
58 return "hello";
59}
60
61const jsonResponse = (obj: unknown) =>
62 new Response(JSON.stringify(obj), { status: 200, headers: { "content-type": "application/json" } });
63const sseResponse = (text: string) =>
64 new Response(text, { status: 200, headers: { "content-type": "text/event-stream" } });
65
66const TOOL_NAME = "create_page";
67const TOOL_ARGS = { title: "Mocked Page" };
68
69
75function makeProviderFetch(providerId: ProviderId): { fetchImpl: typeof fetch } {
76 const fetchImpl: typeof fetch = async (input, init) => {
77 const url = String(typeof input === "string" ? input : input.url);
78 const body = JSON.parse((init?.body as string) ?? "{}");
79 const hasTools = Array.isArray(body.tools) && body.tools.length > 0;
80
81
82 if (providerId === "google" && url.includes(":batchEmbedContents")) {
83 const n = (body.requests as unknown[] | undefined)?.length ?? 1;
84 return jsonResponse({
85 embeddings: Array.from({ length: n }, () => ({ values: new Array(8).fill(0.1) }))
86 });
87 }
88 if (providerId !== "google" && url.includes("/embeddings")) {
89 const n = Array.isArray(body.input) ? body.input.length : 1;
90 return jsonResponse({
91 model: body.model ?? "embed",
92 data: Array.from({ length: n }, (_, i) => ({
93 index: i,
94 embedding: new Array(8).fill(0.1),
95 object: "embedding"
96 })),
97 usage: { prompt_tokens: n, total_tokens: n }
98 });
99 }
100
101
102 const text = lastUserText(body);
103 const words = text.split(" ");
104 const half = Math.max(1, Math.floor(words.length / 2));
105 const p1 = words.slice(0, half).join(" ");
106 const p2 = words.slice(half).join(" ");
107
108 if (providerId === "anthropic") {
109 if (hasTools) {
110 const events = [
111 `event: content_block_start\ndata: ${JSON.stringify({
112 type: "content_block_start",
113 index: 0,
114 content_block: { type: "tool_use", id: "toolu_mock0", name: TOOL_NAME }
115 })}\n\n`,
116 `event: content_block_delta\ndata: ${JSON.stringify({
117 type: "content_block_delta",
118 index: 0,
119 delta: { type: "input_json_delta", partial_json: JSON.stringify(TOOL_ARGS) }
120 })}\n\n`,
121 `event: message_delta\ndata: ${JSON.stringify({
122 type: "message_delta",
123 delta: { stop_reason: "tool_use" },
124 usage: { output_tokens: words.length }
125 })}\n\n`,
126 `event: message_stop\ndata: ${JSON.stringify({ type: "message_stop" })}\n\n`
127 ].join("");
128 return sseResponse(events);
129 }
130 const events = [
131 `event: content_block_delta\ndata: ${JSON.stringify({
132 type: "content_block_delta",
133 index: 0,
134 delta: { type: "text_delta", text: p1 }
135 })}\n\n`,
136 `event: content_block_delta\ndata: ${JSON.stringify({
137 type: "content_block_delta",
138 index: 0,
139 delta: { type: "text_delta", text: " " + p2 }
140 })}\n\n`,
141 `event: message_delta\ndata: ${JSON.stringify({
142 type: "message_delta",
143 delta: { stop_reason: "end_turn" },
144 usage: { output_tokens: words.length }
145 })}\n\n`,
146 `event: message_stop\ndata: ${JSON.stringify({ type: "message_stop" })}\n\n`
147 ].join("");
148 return sseResponse(events);
149 }
150
151 if (providerId === "google") {
152 if (hasTools) {
153 const events = [
154 `data: ${JSON.stringify({
155 candidates: [
156 {
157 content: { parts: [{ functionCall: { name: TOOL_NAME, args: TOOL_ARGS } }] },
158 finishReason: "STOP"
159 }
160 ],
161 usageMetadata: { promptTokenCount: words.length, candidatesTokenCount: words.length }
162 })}\n\n`
163 ].join("");
164 return sseResponse(events);
165 }
166 const events = [
167 `data: ${JSON.stringify({
168 candidates: [{ content: { parts: [{ text: p1 }] } }],
169 usageMetadata: { promptTokenCount: words.length, candidatesTokenCount: 0 }
170 })}\n\n`,
171 `data: ${JSON.stringify({
172 candidates: [{ content: { parts: [{ text: " " + p2 }] }, finishReason: "STOP" }],
173 usageMetadata: { promptTokenCount: words.length, candidatesTokenCount: words.length }
174 })}\n\n`
175 ].join("");
176 return sseResponse(events);
177 }
178
179
180 if (hasTools) {
181 const events = [
182 `data: ${JSON.stringify({
183 choices: [
184 {
185 delta: {
186 tool_calls: [
187 { index: 0, id: "call_mock0", type: "function", function: { name: TOOL_NAME } }
188 ]
189 }
190 }
191 ]
192 })}\n\n`,
193 `data: ${JSON.stringify({
194 choices: [
195 {
196 delta: {
197 tool_calls: [
198 { index: 0, function: { arguments: JSON.stringify(TOOL_ARGS) } }
199 ]
200 },
201 finish_reason: "tool_calls"
202 }
203 ],
204 usage: { prompt_tokens: words.length, completion_tokens: words.length, total_tokens: words.length * 2 }
205 })}\n\n`,
206 "data: [DONE]\n\n"
207 ].join("");
208 return sseResponse(events);
209 }
210 const events = [
211 `data: ${JSON.stringify({ choices: [{ delta: { content: p1 } }] })}\n\n`,
212 `data: ${JSON.stringify({ choices: [{ delta: { content: " " + p2 } }] })}\n\n`,
213 `data: ${JSON.stringify({
214 choices: [{ delta: {}, finish_reason: "stop" }],
215 usage: { prompt_tokens: words.length, completion_tokens: words.length, total_tokens: words.length * 2 }
216 })}\n\n`,
217 "data: [DONE]\n\n"
218 ].join("");
219 return sseResponse(events);
220 };
221 return { fetchImpl };
222}
223
224
225
226
227
228function configFor(providerId: ProviderId): ResolvedAiConfig {
229 return {
230 enabled: true,
231 provider: providerId,
232 chatModel: `${providerId}-chat`,
233 embedModel: `${providerId}-embed`,
234 embedFallbackProvider: "openai",
235 fallback: null,
236 budgets: { tokensPerMonth: 0, requestsPerMinute: 1000, maxOutputTokens: 4096, dailySpendUsd: 0 },
237 policies: { redactPii: false, logPrompts: true, allowUserContentInPrompts: true }
238 } as ResolvedAiConfig;
239}
240
241function buildFakeGateway(providerId: ProviderId, embedDimension = 8): { gateway: AiGateway; ledger: UsageLedger } {
242 const { fetchImpl } = makeProviderFetch(providerId);
243 let adapter: ProviderAdapter;
244 switch (providerId) {
245 case "openai":
246 adapter = new OpenAIAdapter({ baseUrl: "http://x/v1", fetchImpl });
247 break;
248 case "custom":
249 adapter = new CustomAdapter({ baseUrl: "http://x/v1", fetchImpl });
250 break;
251 case "oss":
252 adapter = new OssAdapter({ baseUrl: "http://x/v1", fetchImpl, assumeDimension: embedDimension });
253 break;
254 case "anthropic":
255 adapter = new AnthropicAdapter({ apiKey: "", fetchImpl });
256 break;
257 case "google":
258 adapter = new GoogleAdapter({ apiKey: "", fetchImpl });
259 break;
260 }
261
262
263 const openaiFallback =
264 providerId === "openai" ? adapter : new OpenAIAdapter({ baseUrl: "http://x/v1", fetchImpl: makeProviderFetch("openai").fetchImpl });
265 const ledger = new UsageLedger();
266 const gateway = new AiGateway({
267 resolveConfig: () => configFor(providerId),
268 registry: {
269 get: (id) => (id === providerId ? adapter : id === "openai" ? openaiFallback : adapter)
270 },
271 budgets: new BudgetService(ledger),
272 ledger,
273 sleep: async () => {}
274 });
275 return { gateway, ledger };
276}
277
278function buildMockGateway(): { gateway: AiGateway; ledger: UsageLedger; mock: MockProvider } {
279 const mock = new MockProvider("custom", 8);
280 const ledger = new UsageLedger();
281 const config = { ...configFor("custom"), chatModel: "mock-chat" } as ResolvedAiConfig;
282 const gateway = new AiGateway({
283 resolveConfig: () => config,
284 registry: { get: () => mock },
285 budgets: new BudgetService(ledger),
286 ledger,
287 sleep: async () => {}
288 });
289 return { gateway, ledger, mock };
290}
291
292
297const PROVIDER_IDS = ["openai", "anthropic", "google", "oss", "custom"] as ProviderId[];
298
299
300
301
302
303const ECHO_MSG = "the quick brown fox";
304const CHAT_PARAMS = { workspaceId: "ws-cap", messages: [{ role: "user", content: ECHO_MSG }] };
305
306function assertCapabilitySet(title: string, run: () => Promise<{ gateway: AiGateway; ledger: UsageLedger }>) {
307 describe(`capability: ${title}`, () => {
308 let gateway: AiGateway;
309 let ledger: UsageLedger;
310 beforeAll(async () => {
311 ({ gateway, ledger } = await run());
312 });
313
314 it("echo: chat returns the deterministic reply with usage", async () => {
315 const res = await gateway.chatCollected(CHAT_PARAMS);
316 expect(res.content).toBe(ECHO_MSG);
317 expect(res.finishReason).toBe("stop");
318 expect(res.usage.total).toBeGreaterThan(0);
319 });
320
321 it("stream: deltas arrive in provider order; final chunk carries usage + done", async () => {
322 const deltas: string[] = [];
323 let doneChunk: { usage?: { in: number; out: number; total: number }; finishReason?: string } | undefined;
324 for await (const chunk of gateway.chat(CHAT_PARAMS)) {
325 if (chunk.delta) deltas.push(chunk.delta);
326 if (chunk.done) doneChunk = chunk;
327 }
328 expect(deltas.join("")).toBe(ECHO_MSG);
329 expect(doneChunk?.finishReason).toBe("stop");
330 expect(doneChunk?.usage?.total).toBeGreaterThan(0);
331 });
332
333 it("tools: a tool-capable chat returns the tool call with parsed args", async () => {
334 const res = await gateway.chatCollected({
335 workspaceId: "ws-cap",
336 messages: [{ role: "user", content: "please create a page" }],
337 tools: [{ name: TOOL_NAME, description: "create a page", parameters: { type: "object" } }]
338 });
339 expect(res.toolCalls.length).toBe(1);
340 expect(res.toolCalls[0]?.name).toBe(TOOL_NAME);
341 expect(res.toolCalls[0]?.arguments).toEqual(TOOL_ARGS);
342 expect(res.finishReason).toBe("tool_calls");
343 });
344
345 it("embed: dimension is stable and matches the recorded contract", async () => {
346 const a = await gateway.embed({ workspaceId: "ws-cap", texts: ["alpha", "beta"] });
347 const b = await gateway.embed({ workspaceId: "ws-cap", texts: ["alpha", "beta"] });
348 expect(a.vectors.length).toBe(2);
349 expect(a.dimension).toBe(8);
350 expect(b.dimension).toBe(a.dimension);
351 expect(b.vectors[0]?.[0]).toBe(a.vectors[0]?.[0]);
352 });
353
354 it("ledger: a completed chat appends a kind=chat usage record", async () => {
355 const before = ledger.records.length;
356 await gateway.chatCollected(CHAT_PARAMS);
357 expect(ledger.records.length).toBe(before + 1);
358 const rec = ledger.records[ledger.records.length - 1]!;
359 expect(rec.kind).toBe("chat");
360 expect(rec.workspaceId).toBe("ws-cap");
361 expect(rec.status).toBe("ok");
362 });
363
364 it("errors: 429 / 401 / 500 map to the canonical envelope", async () => {
365 const make = (status: number) => {
366 const fetchImpl: typeof fetch = async () =>
367 new Response(JSON.stringify({ error: "boom" }), {
368 status,
369 headers: { "content-type": "application/json" }
370 });
371 const adapter = new OpenAIAdapter({ baseUrl: "http://x/v1", fetchImpl });
372 const l = new UsageLedger();
373 return new AiGateway({
374 resolveConfig: () => configFor("openai"),
375 registry: { get: () => adapter },
376 budgets: new BudgetService(l),
377 ledger: l,
378 sleep: async () => {}
379 });
380 };
381
382 const limited = await make(429).chatCollected(CHAT_PARAMS).catch((e) => e);
383 expect(limited).toBeInstanceOf(ApiError);
384 expect((limited as ApiError).code).toBe(ErrorCodes.RATE_LIMITED);
385
386 const auth = await make(401).chatCollected(CHAT_PARAMS).catch((e) => e);
387 expect((auth as ApiError).code).toBe(ErrorCodes.AI_ERROR);
388 const authDetails = (auth as ApiError).details as { code?: string; reason?: string };
389 expect(authDetails?.code).toBe("SERVICE_UNAVAILABLE");
390 expect(authDetails?.reason).toBe("auth");
391
392 const server = await make(500).chatCollected(CHAT_PARAMS).catch((e) => e);
393 expect((server as ApiError).code).toBe(ErrorCodes.AI_ERROR);
394 const serverDetails = (server as ApiError).details as { code?: string; reason?: string; retryable?: boolean };
395 expect(serverDetails?.code).toBe("SERVICE_UNAVAILABLE");
396 expect(serverDetails?.reason).toBe("provider_error");
397 expect(serverDetails?.retryable).toBe(true);
398 });
399 });
400}
401
402describe("TASK-0121 gateway capability — provider swap = config only", () => {
403 for (const id of PROVIDER_IDS) {
404 assertCapabilitySet(`provider=${id}`, async () => buildFakeGateway(id));
405 }
406});
407
408
409
410
411describe("TASK-0121 gateway capability — mock provider echo contract", () => {
412 it("a tool call echoes the last user message into its arguments", async () => {
413 const { gateway } = buildMockGateway();
414 const res = await gateway.chatCollected({
415 workspaceId: "ws-cap",
416 messages: [{ role: "user", content: "make a page called X" }],
417 tools: [{ name: "create_page", description: "create", parameters: { type: "object" } }]
418 });
419 expect(res.toolCalls[0]?.name).toBe("create_page");
420 expect((res.toolCalls[0]?.arguments as { echoed?: string })?.echoed).toBe("make a page called X");
421 });
422});
423
424
425
426
427
428
434const REAL = process.env.AI_REAL_TEST === "1";
435
436describe.runIf(REAL)("TASK-0121 gateway capability — real provider (AI_REAL_TEST=1)", () => {
437 const providerId = (process.env.AI_PROVIDER ?? "openai") as ProviderId;
438 it("runs the identical assertion set against the live endpoint", async () => {
439 let adapter: ProviderAdapter;
440 const baseUrl = process.env.AI_BASE_URL;
441 const key = process.env.AI_API_KEY ?? "";
442 switch (providerId) {
443 case "anthropic":
444 adapter = new AnthropicAdapter({ apiKey: key });
445 break;
446 case "google":
447 adapter = new GoogleAdapter({ apiKey: key });
448 break;
449 case "oss":
450 adapter = new OssAdapter({ baseUrl: baseUrl ?? "http://localhost:11434/v1", apiKey: key });
451 break;
452 case "custom":
453 adapter = new CustomAdapter({ baseUrl: baseUrl ?? "http://localhost:11434/v1", apiKey: key });
454 break;
455 default:
456 adapter = new OpenAIAdapter({ baseUrl: baseUrl ?? "https://api.openai.com/v1", apiKey: key });
457 break;
458 }
459 const ledger = new UsageLedger();
460 const gateway = new AiGateway({
461 resolveConfig: () => ({
462 ...configFor(providerId),
463 chatModel: process.env.AI_MODEL ?? configFor(providerId).chatModel
464 }),
465 registry: { get: (id) => (id === providerId ? adapter : new OpenAIAdapter({ apiKey: key })) },
466 budgets: new BudgetService(ledger),
467 ledger,
468 sleep: async (ms) => new Promise((r) => setTimeout(r, ms))
469 });
470
471 const res = await gateway.chatCollected(CHAT_PARAMS);
472 expect(res.content.length).toBeGreaterThan(0);
473 expect(res.finishReason).toBe("stop");
474 expect(res.usage.total).toBeGreaterThan(0);
475 const emb = await gateway.embed({ workspaceId: "ws-cap", texts: ["alpha"] });
476 expect(emb.vectors.length).toBe(1);
477 expect(emb.dimension).toBeGreaterThan(0);
478 });
479});
480
481if (!REAL) {
482 describe("TASK-0121 gateway capability — real provider gate", () => {
483 it("is skipped unless AI_REAL_TEST=1 (warns)", () => {
484
485 console.warn("[ai-capability] real-provider suite skipped (set AI_REAL_TEST=1 to run it)");
486 });
487 });
488}
489
