/** * AI Software Factory — E2E: Ask AI (TASK-0122). * * Drives the REAL stack (Next web + Nest API + the deterministic mock provider) * against the acceptance criteria in docs/tasks/phase-12-ai.md. The CI e2e job * starts the API with `AI_PROVIDER=mock` + the internal test seam * (`INTERNAL_SERVICE_TOKEN` + `INTERNAL_TEST_SEAM=1`, job env — the API reads * process.env only), so these specs are deterministic and need no provider key. * * Covered: * 1. Happy path (page toolbar → panel + context badge → streaming ≥3 * incremental updates → answer persists across reload). * 2. Command palette "Ask AI" opens the same panel; create / rename / delete * a conversation round-trips (list updates + API 404 after delete). * 3. Provider outage (armed mid-stream via the seam) → retryable error banner * + Retry button → Retry re-sends WITHOUT duplicating the user row and * delivers the answer. * 4. Budget exhaustion (monthly token budget set to 1 via the seam) → * pre-stream 402 → "Monthly AI budget reached" banner, NO Retry button * (a budget 402 is not retryable). * * Send budget: the per-user AI cap is 30 messages/min (retries included) and * the demo user is shared across the 2 CI workers, so this file keeps total * sends low (~5) and the tests are serial (default within a file). * * Selector convention (tests/README.md): the panel's own data-testids — * ai-panel, ai-context-badge, ai-input, ai-send, ai-stop, ai-message-live, * ai-message-user, ai-message-assistant, ai-error-banner, ai-retry, * ai-history-item, ai-history-rename, ai-history-delete, ai-delete-confirm. */ import { test, expect } from "../fixtures"; import { armAiMockFailures, createPage, setAiBudgetLimit, seamServiceToken } from "../utils/api"; import type { APIRequestContext } from "@playwright/test"; // A short question: 5 words / 34 chars (< the mock's 40-char echo window, so // the whole question is echoed) → 5 per-word deltas + the [SYS:…] tail, // comfortably ≥3 distinct streaming frames at the CI 60 ms delta delay. const QUESTION = "Summarize the content on this page"; /** The mock echoes the last user message (first 40 chars) — here, the full question. */ const ECHO_PREFIX = QUESTION.slice(0, 40); interface ConversationRow { id: string; title: string | null; } async function listConversations(api: APIRequestContext, wsId: string): Promise { const res = await api.get(`/api/v1/workspaces/${wsId}/ai/conversations`); expect(res.status(), `listConversations(${wsId}): ${res.status()} ${await res.text()}`).toBe(200); const body = (await res.json()) as { data: ConversationRow[] }; return body.data; } /** GET /ai/conversations/:id → the conversation object (404 across the boundary). */ async function getConversation(api: APIRequestContext, id: string): Promise { const res = await api.get(`/api/v1/ai/conversations/${id}`); expect(res.status(), `getConversation(${id}): ${res.status()} ${await res.text()}`).toBe(200); const body = (await res.json()) as { data: ConversationRow }; return body.data; } async function getConversationStatus(api: APIRequestContext, id: string): Promise { return (await api.get(`/api/v1/ai/conversations/${id}`)).status(); } test.describe("Ask AI (TASK-0122)", () => { test("page toolbar → panel + context badge → streams ≥3 updates → persists across reload", async ({ authPage, api, freshWorkspace }) => { const ws = await freshWorkspace("AI Chat Space"); const page = await createPage(api, ws.id, { title: `AI Probe ${Date.now()}` }); await authPage.goto(`/w/${ws.id}/page/${page.id}`); await authPage.getByTestId("ask-ai-button").click(); // Panel opens, bound to the page (context badge = the page title). const panel = authPage.getByTestId("ai-panel"); await expect(panel).toBeVisible(); await expect(authPage.getByTestId("ai-context-badge")).toHaveText(page.title); const input = authPage.getByTestId("ai-input"); await expect(input).toBeVisible(); // Send and watch the LIVE bubble grow (≥3 distinct incremental updates). await input.fill(QUESTION); await authPage.getByTestId("ai-send").click(); const live = authPage.getByTestId("ai-message-live"); const seen = new Set(); await expect .poll( async () => { if ((await live.count()) > 0) { const t = (await live.first().textContent())?.trim() ?? ""; // The pre-first-delta "Thinking…" placeholder shares the testid. if (t && !/thinking/i.test(t)) seen.add(t); } return seen.size; }, { timeout: 20_000, intervals: [25] } ) .toBeGreaterThanOrEqual(3); // Final assistant message carries the echo (the question's first 40 chars). const assistant = authPage.getByTestId("ai-message-assistant").last(); await expect(assistant).toBeVisible({ timeout: 20_000 }); await expect(assistant).toContainText(ECHO_PREFIX); await expect(authPage.getByTestId("ai-message-user")).toContainText(QUESTION); // Persists across reload: reopen the panel and the conversation + messages // come back from the server (not client state). await authPage.reload(); await authPage.getByTestId("ask-ai-button").click(); await expect(panel).toBeVisible(); // History (open by default) shows the auto-titled conversation. const item = authPage.getByTestId("ai-history-item"); await expect(item).toBeVisible({ timeout: 15_000 }); await expect(item).toContainText(QUESTION.slice(0, 60)); // Open it → the persisted user + assistant rows reload. await item.click(); await expect(authPage.getByTestId("ai-message-user")).toContainText(QUESTION); await expect(authPage.getByTestId("ai-message-assistant").last()).toContainText(ECHO_PREFIX); }); test("command palette Ask AI → create / rename / delete round-trips", async ({ authPage, api, freshWorkspace }) => { const ws = await freshWorkspace("AI Palette Space"); const page = await createPage(api, ws.id, { title: "Palette Probe" }); await authPage.goto(`/w/${ws.id}/page/${page.id}`); // Open the palette from the topbar and run its "Ask AI" action. await authPage.getByTestId("command-palette-button").click(); const palette = authPage.getByRole("dialog", { name: "Command palette" }); await expect(palette).toBeVisible(); const paletteInput = palette.getByRole("combobox"); await expect(paletteInput).toBeVisible(); await paletteInput.fill("ask ai"); await paletteInput.press("Enter"); // The SAME panel opens (no context badge — the palette passes null context). const panel = authPage.getByTestId("ai-panel"); await expect(panel).toBeVisible(); await expect(authPage.getByTestId("ai-context-badge")).toHaveCount(0); // Create a conversation by sending a message (auto-titled from it). const input = authPage.getByTestId("ai-input"); await expect(input).toBeVisible(); await input.fill("What is this workspace about"); await authPage.getByTestId("ai-send").click(); await expect(authPage.getByTestId("ai-message-assistant").last()).toBeVisible({ timeout: 20_000 }); // It appears in the history list (auto-title = first 60 chars of the msg). const item = authPage.getByTestId("ai-history-item"); await expect(item).toBeVisible(); await expect(item).toContainText("What is this workspace about"); const [conv] = await listConversations(api, ws.id); expect(conv, "one conversation after the first send").toBeTruthy(); const convId = conv!.id; // Rename (inline edit: double-click → input → Enter). The UI PATCH is // fire-and-forget, so poll the server until the title round-trips. await item.dblclick(); const rename = authPage.getByTestId("ai-history-rename"); await expect(rename).toBeVisible(); await rename.fill("Renamed Chat"); await rename.press("Enter"); await expect(authPage.getByTestId("ai-history-item")).toContainText("Renamed Chat"); await expect .poll(async () => (await getConversation(api, convId)).title, { timeout: 10_000 }) .toBe("Renamed Chat"); // Delete (confirm dialog) → removed from the list + API 404. await authPage.getByTestId("ai-history-delete").click(); await authPage.getByTestId("ai-delete-confirm-yes").click(); await expect(authPage.getByTestId("ai-delete-confirm")).toHaveCount(0); await expect(authPage.getByTestId("ai-history-item")).toHaveCount(0); await expect .poll(async () => getConversationStatus(api, convId), { timeout: 10_000 }) .toBe(404); }); test("provider outage → retryable banner + Retry re-sends without duplicating", async ({ authPage, api, freshWorkspace }) => { const token = seamServiceToken(); const ws = await freshWorkspace("AI Outage Space"); const page = await createPage(api, ws.id, { title: "Outage Probe" }); await authPage.goto(`/w/${ws.id}/page/${page.id}`); await authPage.getByTestId("ask-ai-button").click(); const panel = authPage.getByTestId("ai-panel"); await expect(panel).toBeVisible(); const input = authPage.getByTestId("ai-input"); await expect(input).toBeVisible(); // Arm a MID-stream 500 on the next mock chat call FOR THIS WORKSPACE. // The gateway never re-streams partial output, so this surfaces as an // in-band SSE error (error delta + terminal done) → the panel's // retryable banner. Per-workspace scoping keeps it deterministic even // when Playwright's desktop + mobile projects run in parallel. await armAiMockFailures(token, ws.id, 500, 2); await input.fill(QUESTION); await authPage.getByTestId("ai-send").click(); // The retryable error banner (not budget / not rate-limit wording). const banner = authPage.getByTestId("ai-error-banner"); await expect(banner).toBeVisible({ timeout: 20_000 }); await expect(banner).toContainText("Something went wrong"); await expect(authPage.getByTestId("ai-retry")).toBeVisible(); const [conv] = await listConversations(api, ws.id); expect(conv?.id, "the failed turn created a conversation + user row").toBeTruthy(); const convId = conv!.id; // Retry → a fresh mock call (the arming is single-shot) succeeds. await authPage.getByTestId("ai-retry").click(); await expect(authPage.getByTestId("ai-message-assistant").last()).toContainText(ECHO_PREFIX, { timeout: 20_000 }); await expect(banner).toHaveCount(0); // The retry used `retryOf` — the user row was NOT duplicated (exactly one // user message in the conversation after the retry). const res = await api.get(`/api/v1/ai/conversations/${convId}/messages?limit=50`); expect(res.status()).toBe(200); const body = (await res.json()) as { data: Array<{ role: string }> }; expect(body.data.filter((m) => m.role === "user")).toHaveLength(1); }); test("budget exhaustion → pre-stream 402 banner, no Retry", async ({ authPage, api, freshWorkspace }) => { const token = seamServiceToken(); const ws = await freshWorkspace("AI Budget Space"); const page = await createPage(api, ws.id, { title: "Budget Probe" }); await authPage.goto(`/w/${ws.id}/page/${page.id}`); await authPage.getByTestId("ask-ai-button").click(); const panel = authPage.getByTestId("ai-panel"); await expect(panel).toBeVisible(); const input = authPage.getByTestId("ai-input"); await expect(input).toBeVisible(); // Set the monthly token budget to 1 (< 2×maxOutputTokens) → the pre-stream // guard 402s before any SSE byte is written (JSON envelope, not in-band). await setAiBudgetLimit(token, ws.id, 1); await input.fill(QUESTION); await authPage.getByTestId("ai-send").click(); // The budget banner (no Retry — a budget 402 is not retryable). const banner = authPage.getByTestId("ai-error-banner"); await expect(banner).toBeVisible({ timeout: 20_000 }); await expect(banner).toContainText("Monthly AI budget reached"); await expect(authPage.getByTestId("ai-retry")).toHaveCount(0); // No assistant row (the guard fired before the provider call). await expect(authPage.getByTestId("ai-message-assistant")).toHaveCount(0); }); });