ai-rag.spec.ts · Source

ai-rag.spec.tstests/e2e/specs/ai-rag.spec.tsraw ← specs6,429 B← Back to the app
1/**
2 * AI Software Factory — E2E: RAG workspace search (TASK-0125).
3 *
4 * Drives the REAL stack (Next web + Nest API + worker + OpenSearch) against
5 * the canonical ROADMAP exit check: seed a page with a unique fact, ask about
6 * it on the /ask page, and assert the streamed answer cites the fact AND a
7 * citation chip whose deep link actually works.
8 *
9 * The mock provider's RAG mode is deterministic: it returns
10 *   `According to [n] (<label>): <chunk text>`
11 * from the FIRST numbered context block the RAG service feeds it. So the
12 * seeded fact must be retrievable (worker chunks + embeds it) and the answer
13 * carries it verbatim — the assertion is exact, not fuzzy.
14 *
15 * Covered (the spec's E2E AC + the Phase-12 gate item #4 "citation link in
16 * the answer DOM"):
17 *   1. seed page with a unique fact F → `/ask` → ask "What is the launch code
18 *      word?" → streamed answer contains F.
19 *   2. a citation chip links to the seeded page; clicking it deep-links to the
20 *      page, which renders the fact (the link WORKS — not just present).
21 *   3. the Sources list renders the same citation.
22 *
23 * Indexing is the worker's job: a page write publishes a domain event the
24 * stream consumer consumes (search-index upsert + AI embed). We gate on the
25 * PAGE being searchable via the public `POST /api/v1/search` (the same
26 * permission-filtered path the RAG query uses) before asking — this proves
27 * the full pipeline (event → chunk → embed → index) ran.
28 *
29 * Selector convention (tests/README.md): this task's own data-testids —
30 *   ask-ai, ask-ai-input, ask-ai-submit, ask-ai-answer, ask-ai-sources,
31 *   ask-ai-chip-{n}, ask-ai-source-{n}.
32 *
33 * Requires OpenSearch (SEARCH_URL, default http://localhost:9200) reachable —
34 * the RAG retrieval path is index-backed. Skipped (not failed) when it is
35 * down, mirroring search.integration.test.ts's stack gate.
36 */
37import { expect, test } from "../fixtures";
38import { createBlock, createPage, createWorkspace } from "../utils/api";
39import { waitUntilSearchable } from "../utils/search-lag";
40 
41/** Is the OpenSearch at `url` (the API's SEARCH_URL default) reachable? */
42async function opensearchUp(url: string): Promise<boolean> {
43  try {
44    const res = await fetch(`${url.replace(/\/+$/, "")}/_cluster/health`, {
45      signal: AbortSignal.timeout(5_000),
46    });
47    return res.ok;
48  } catch {
49    return false;
50  }
51}
52const SEARCH_URL = process.env.SEARCH_URL ?? "http://localhost:9200";
53 
54// A globally unique fact + token: the RAG query is workspace-scoped, but the
55// search gate (public /api/v1/search) is not, so a run-unique token cannot
56// false-positive on a concurrent worker's shared index.
57const RUN = Date.now().toString(36);
58const TOKEN = `Zanzibar-${RUN}`;
59const FACT = `The launch code word is ${TOKEN}.`;
60 
61test("RAG: seeded fact is cited in the streamed answer with a working citation chip", async ({
62  api,
63  authPage,
64}) => {
65  // RAG is index-backed; skip (not fail) when OpenSearch is down, mirroring
66  // search.integration.test.ts's stack gate (CI always has it). Probed in-test
67  // (no top-level await — the Playwright runner transforms this spec to a CJS
68  // graph via require(), which rejects top-level `await`).
69  const opensearchReady = await opensearchUp(SEARCH_URL);
70  test.skip(!opensearchReady, "OpenSearch not reachable (SEARCH_URL) — RAG is index-backed");
71 
72  // Seed THROUGH the public API (real authorization layer). `api` is a Bearer
73  // context pinned to API_BASE (:4000) — `authPage.request` would resolve
74  // against the WEB origin (:3000) and 404, so the seed + search gate use
75  // `api`; only the /ask UI uses `authPage`.
76  const ws = await createWorkspace(api, `RAG ${RUN}`);
77  const page = await createPage(api, ws.id, { title: "Launch Runbook" });
78  await createBlock(api, page.id, {
79    type: "paragraph",
80    content: { text: FACT },
81    position: "1.0",
82  });
83 
84  // Gate on the worker having indexed (+ embedded) the page. The public
85  // search is the same permission-filtered path the RAG query uses, so a hit
86  // here proves the pipeline (event → chunk → embed → index) ran and the doc
87  // is a valid kNN target for the ask below. `waitUntilSearchable` throws on
88  // timeout, so resolving means the doc is searchable.
89  const lag = await waitUntilSearchable(api, TOKEN, Date.now(), { timeoutMs: 60_000 });
90  console.log(`[ai-rag] seeded page searchable after ${lag.lagMs}ms (${lag.polls} polls)`);
91 
92  // Drive the REAL /ask UI: navigate to the workspace's ask page.
93  await authPage.goto(`/w/${ws.slug}/ask`);
94  const input = authPage.getByTestId("ask-ai-input");
95  await expect(input).toBeVisible();
96  await input.fill("What is the launch code word?");
97  await authPage.getByTestId("ask-ai-submit").click();
98 
99  // The streamed (mock) answer contains the fact verbatim.
100  const answer = authPage.getByTestId("ask-ai-answer");
101  await expect(answer).toContainText(TOKEN, { timeout: 30_000 });
102 
103  // The Sources list renders the citation (asserted on the /ask page BEFORE
104  // the chip click navigates away). The mock always emits the [1] marker for
105  // the single source, so [1] is present + linked.
106  await expect(authPage.getByTestId("ask-ai-sources")).toBeVisible();
107  await expect(authPage.getByTestId("ask-ai-source-1")).toBeVisible();
108 
109  // At least one citation chip, linking to the seeded page.
110  const chip = authPage.getByTestId("ask-ai-chip-1");
111  await expect(chip).toBeVisible();
112  const href = await chip.getAttribute("href");
113  expect(href, "the citation chip must be a link").toBeTruthy();
114  expect(href, `chip href should target the seeded page (got ${href})`).toBe(
115    `/w/${ws.slug}/page/${page.id}`,
116  );
117 
118  // The citation LINK WORKS (Phase-12 gate item #4): following the citation's
119  // deep link lands on the seeded page, which renders the fact. We use a
120  // full `page.goto(href)` (a hard navigation — what happens if the link is
121  // opened fresh) rather than an in-place client-side `click()`: the point is
122  // the deep link RESOLVES and renders for the user, and a hard load is the
123  // strongest, flake-free proof of that. (A raw HTTP 200 is NOT a strong
124  // check — the Next shell returns 200 even unauthenticated — so we assert the
125  // seeded content actually renders.)
126  await authPage.goto(href!);
127  await expect(authPage).toHaveURL(new RegExp(`/w/${ws.slug}/page/`));
128  await expect(authPage.getByText(FACT)).toBeVisible();
129});
130