queries.ts · Source

queries.tspackages/search/src/queries.tsraw ← src14,684 B← Back to the app
1/**
2 * @factory/search — OpenSearch query builders (search-architecture §4–§5).
3 *
4 * Pure functions: they turn a {@link SearchQuery} + {@link ViewerContext} into
5 * the exact OpenSearch request body. The `client.ts` transport layer applies
6 * them; apps may also use them directly.
7 *
8 * Phase-8 reconciliation (recorded decision, see docs/architecture/search.md
9 * § "Layout decision"):
10 *  - **Single index** (alias `factory-content`), not one index per workspace.
11 *    Tenant isolation is the mandatory `workspaceId` filter, applied in every
12 *    builder — it is a parameter, never optional.
13 *  - **camelCase field paths** matching the stored {@link SearchDoc} (the
14 *    Phase-0 scaffold mixed snake_case / `grants.*` / `title.raw`, which did
15 *    not match the stored doc). `title.keyword` backs prefix + fuzzy.
16 *
17 * Permission model (search-architecture §5): the visibility filter is a
18 * `must`-equivalent `filter` clause — docs fail it when the viewer is not
19 * public / workspace-visible / user-granted / team-granted. The viewer's team
20 * ids are inlined (the snapshot stores team ids, not memberships, so the
21 * expansion happens here).
22 */
23 
24import { CHUNK_DOC_TYPE } from "./types.js";
25import type { SearchQuery, SearchSort, ViewerContext } from "./types.js";
26 
27/** The single content index (alias). All typed docs live here (search §2). */
28export const CONTENT_INDEX_ALIAS = "factory-content";
29 
30/**
31 * The content index to read from / write to. Currently the alias; a rollover
32 * (search §2/§7.2) swaps the backing index behind the same alias, so callers
33 * never hard-code a versioned name.
34 */
35export function contentIndex(): string {
36  return CONTENT_INDEX_ALIAS;
37}
38 
39/** Fields matched by the full-text query (title weighted 3× over body). */
40const FULLTEXT_FIELDS: Record<string, number> = {
41  title: 3,
42  body: 1,
43  createdByName: 0.5,
44};
45 
46interface BoolClause {
47  must?: unknown[];
48  must_not?: unknown[];
49  should?: unknown[];
50  filter?: unknown[];
51  minimum_should_match?: number;
52}
53 
54function bool(clause: BoolClause): { bool: BoolClause } {
55  return { bool: clause };
56}
57 
58/**
59 * The tenant clause: restrict to the caller's workspaces. Non-negotiable —
60 * every query filters on it (search §3 "tenant filter — ALWAYS present").
61 */
62export function tenantFilter(viewer: ViewerContext): unknown {
63  return { terms: { workspaceId: viewer.workspaceIds } };
64}
65 
66/** The visibility / permission filter (search-architecture §5.2). */
67export function visibilityFilter(viewer: ViewerContext): unknown {
68  // A doc is visible iff any of: a public share; the viewer's workspace grants
69  // it (non-private workspace content — the tenant filter already scopes this
70  // clause to the caller's workspaces, so `workspace: true` never over-surfaces
71  // a foreign workspace); an explicit user grant; or a team grant.
72  // TASK-0100 S5: explicit `term`/`terms` clauses. The old raw field-value
73  // form ({ "grants.public": true }) is only legal inside a `match` body, not
74  // a bool context — OpenSearch rejected the whole query ("[grants.public]
75  // query malformed, no start_object after query name"), so EVERY content
76  // search 400'd. (The field NAMES are correct — they match the doc shape
77  // the builders emit; only the clause syntax was broken.)
78  const or: unknown[] = [
79    { term: { "grants.public": true } },
80    { term: { "grants.workspace": true } },
81    { term: { "grants.users": viewer.userId } },
82  ];
83  if (viewer.teamIds.length > 0) or.push({ terms: { "grants.teams": viewer.teamIds } });
84  return bool({ should: or, minimum_should_match: 1 });
85}
86 
87/**
88 * The keyword-search chunk exclusion (TASK-0099, spec req 3): chunk docs
89 * (`type: "chunk"`) are the kNN/hybrid leg's target, never keyword hits.
90 * The public `types` filter alone CANNOT guarantee this — the live API omits
91 * the `types` clause entirely when unspecified (search.service.ts) — so the
92 * exclusion is a mandatory `must_not` in every keyword body. Keyword
93 * coverage is unaffected: a chunk's text lives in its parent's whole-doc
94 * `body`, so any keyword match a chunk would make is already matched by the
95 * parent doc.
96 */
97export function chunkExclusion(): unknown {
98  return { term: { type: CHUNK_DOC_TYPE } };
99}
100 
101/**
102 * The mandatory filter array for a content query: tenant + permission +
103 * type/chip filters. Shared by the full-text and hybrid builders so the
104 * tenant/permission clauses cannot drift between them.
105 */
106function baseFilters(viewer: ViewerContext, query: SearchQuery): unknown[] {
107  const filter: unknown[] = [tenantFilter(viewer), visibilityFilter(viewer)];
108  if (query.types && query.types.length > 0) {
109    filter.push({ terms: { type: [...query.types] } });
110  }
111  const f = query.filters;
112  if (f) {
113    if (f.teams && f.teams.length > 0) filter.push({ terms: { teamId: f.teams } });
114    if (f.createdBy) filter.push({ term: { createdBy: f.createdBy } });
115    if (f.updatedAfter) filter.push({ range: { updatedAt: { gte: f.updatedAfter } } });
116    if (f.updatedBefore) filter.push({ range: { updatedAt: { lte: f.updatedBefore } } });
117    if (f.databases && f.databases.length > 0) filter.push({ terms: { databaseId: f.databases } });
118    if (f.archived !== undefined) filter.push({ term: { archived: f.archived } });
119  }
120  // Archived docs are hidden by default (explicit `archived: true` opts in).
121  if (f?.archived !== true) filter.push({ term: { archived: false } });
122  return filter;
123}
124 
125/** Build the full-text (default mode) OpenSearch body. */
126export function buildFullTextBody(query: SearchQuery, viewer: ViewerContext): Record<string, unknown> {
127  return {
128    size: query.pageSize ?? 25,
129    track_total_hits: 10000,
130    query: bool({
131      must: [
132        {
133          multi_match: {
134            query: query.q,
135            fields: Object.entries(FULLTEXT_FIELDS).map(([f, boost]) => `${f}^${boost}`),
136            type: "best_fields",
137            operator: "or",
138          },
139        },
140      ],
141      filter: baseFilters(viewer, query),
142      // Whole-doc keyword search excludes chunk docs (TASK-0099, spec req 3).
143      must_not: [chunkExclusion()],
144    }),
145    highlight: {
146      fields: { title: {}, body: { fragment_size: 160, number_of_fragments: 1 } },
147    },
148  };
149}
150 
151/** Prefix query for autocomplete / "search as you type" (search §4.2). */
152export function buildPrefixBody(prefix: string, viewer: ViewerContext, size = 10): Record<string, unknown> {
153  const p = prefix.toLowerCase();
154  return {
155    size,
156    query: bool({
157      must: [
158        bool({
159          should: [
160            { prefix: { "title.keyword": p } },
161            // Spec §4.2 requires a CASE-INSENSITIVE prefix (the `factory_prefix`
162            // analyzer lowercases). `title.keyword` is case-sensitive, so a
163            // lowercased prefix against it misses any differently-cased title
164            // (e.g. an uppercase "Q3" title vs the lowercased "q3" prefix) →
165            // autocomplete returned [] while full-text found the same page.
166            // `match_phrase_prefix` on the analyzed (lowercased) `title` field
167            // restores the spec'd case-insensitive behavior WITHOUT rolling the
168            // index forward to add the analyzer (impossible in place — analyzer
169            // settings are immutable on the live `factory-content` index; a
170            // reindex is the recorded upgrade). Phrase semantics keep it tight:
171            // all prefix tokens must appear in order (last one as a prefix),
172            // so it does not over-match the way a plain wildcard on the text
173            // field would (the analyzer would split a hyphenated token into
174            // separate terms).
175            { match_phrase_prefix: { title: p } },
176            { prefix: { id: p } },
177          ],
178          minimum_should_match: 1,
179        }),
180      ],
181      filter: [tenantFilter(viewer), visibilityFilter(viewer)],
182      must_not: [chunkExclusion()],
183    }),
184  };
185}
186 
187/** Explicit fuzzy query ("did you mean", typo-heavy input, search §4.3). */
188export function buildFuzzyBody(
189  query: string,
190  viewer: ViewerContext,
191  options: { fuzziness?: "AUTO" | 0 | 1 | 2 } = {},
192  size = 20,
193): Record<string, unknown> {
194  return {
195    size,
196    query: bool({
197      must: [
198        {
199          fuzzy: {
200            "title.keyword": { value: query, fuzziness: options.fuzziness ?? "AUTO" },
201          },
202        },
203      ],
204      filter: [tenantFilter(viewer), visibilityFilter(viewer)],
205      must_not: [chunkExclusion()],
206    }),
207  };
208}
209 
210/**
211 * Hybrid search (search §4.5): BM25 + kNN. The client executes both paths
212 * (see the recorded two-request decision in `client.ts`); this returns the
213 * BM25 (pass 1) body. The vector leg runs as a separate `query.knn` request
214 * (`buildHybridKnnOnlyBody`) and the client fuses the two with RRF
215 * (`buildHybridFusion`, k=60).
216 *
217 * TASK-0125 (recorded deviation, verified live against 2.15): pass 1 is now
218 * full-text ONLY. The old body also carried an ES-style top-level `knn`
219 * block, which OpenSearch 2.15 rejects (`Unknown key for a START_OBJECT in
220 * [knn]` — 400 on every hybrid search); the Phase-8 scaffold shipped that
221 * shape but the kNN leg was never exercised live (Phase-8 integration only
222 * ran full-text). And the working `query.knn` clause cannot be combined with
223 * a `bool`/full-text query in one request body — it must be the SOLE top-
224 * level `query` — so the two-request + RRF design is the way to hybridize.
225 *
226 * `queryVector` is the embedded query (from `@factory/ai` or the stub).
227 */
228export function buildHybridBody(
229  query: SearchQuery,
230  viewer: ViewerContext,
231  queryVector: number[],
232  options?: { knnK?: number },
233): Record<string, unknown> {
234  void queryVector;
235  void options;
236  return buildFullTextBody(query, viewer);
237}
238 
239/**
240 * kNN-only body for the hybrid pass 2 (see `client.ts` — recorded decision:
241 * hybrid = two requests + client-side RRF, so the kNN pass needs its own body
242 * carrying only the vector clause + the same mandatory filters).
243 */
244export function buildHybridKnnOnlyBody(
245  queryVector: number[],
246  viewer: ViewerContext,
247  k = 50,
248): Record<string, unknown> {
249  return {
250    size: k,
251    query: {
252      knn: knnBlock(queryVector, viewer, k),
253    },
254  };
255}
256 
257/**
258 * The `query.knn.<field>` vector clause (OpenSearch 2.15).
259 *
260 * TASK-0125 (recorded deviation, verified live against 2.15): the query DSL
261 * is the **field-keyed `query.knn`** form — `{ "query": { "knn": { "vector":
262 * { "vector": [...], "k", "filter" } } } }` — NOT the ES-style top-level
263 * `knn: { field, query_vector }` block (400 on this build, see above). The
264 * inline `filter` is the tenant + visibility clause, applied as a
265 * **pre-filter** (FAISS engine — nmslib hard-rejects kNN filters, "Engine
266 * [NMSLIB] does not support filters") so the ANN search only considers the
267 * viewer's own docs. With the shared multi-tenant index that pre-filtering
268 * is a security/recall requirement: without it the ANN fetch would return
269 * the GLOBAL top-K (cross-tenant candidates in the fusion, and a small
270 * tenant's docs could never reach the top-50).
271 *
272 * `num_candidates` is NOT part of the `query.knn` grammar on 2.15 (`[knn]
273 * query does not support [num_candidates]`); search depth is set index-side
274 * via `knn.algo_param.ef_search` (see `contentMapping`).
275 */
276function knnBlock(
277  queryVector: number[],
278  viewer: ViewerContext,
279  k: number,
280): Record<string, unknown> {
281  return {
282    vector: {
283      vector: [...queryVector],
284      k,
285      filter: bool({ filter: [tenantFilter(viewer), visibilityFilter(viewer)] }),
286    },
287  };
288}
289 
290/** Cursor = base64 JSON of the search_after tuple (score, updated_at, id). */
291export function encodeCursor(tuple: [number, string, string]): string {
292  return Buffer.from(JSON.stringify(tuple), "utf8").toString("base64url");
293}
294 
295export function decodeCursor(cursor: string): [number, string, string] {
296  const raw = Buffer.from(cursor, "base64url").toString("utf8");
297  const parsed: unknown = JSON.parse(raw);
298  if (!Array.isArray(parsed) || parsed.length !== 3) {
299    throw new Error("invalid cursor");
300  }
301  return parsed as [number, string, string];
302}
303 
304/**
305 * Attach `search_after` to a body from the previous page's last hit
306 * (search_after is score + sort tiebreakers, per search §6). The tuple is
307 * `(score, updatedAt, id)` matching the `_score`/`updatedAt`/`id` sort order.
308 */
309export function withSearchAfter(body: Record<string, unknown>, tuple: [number, string, string]): Record<string, unknown> {
310  return {
311    ...body,
312    sort: [{ _score: "desc" }, { updatedAt: "desc" }, { id: "asc" }],
313    search_after: [...tuple],
314  };
315}
316 
317// ---------------------------------------------------------------------------
318// T-155 — first-class date sorts (spec §10: "creation date / modification
319// date"). ADDITIVE branches: the relevance path above is untouched, and the
320// two date modes use their OWN `(field, id)` cursor + sort clauses. Both date
321// fields are `type: "date"` in the index (mapping.ts), so no reindex.
322// ---------------------------------------------------------------------------
323 
324/** The indexed date field a date `sortBy` orders on (throws on unknown). */
325export function dateSortField(sort: SearchSort): "createdAt" | "updatedAt" {
326  if (sort === "createdAt" || sort === "updatedAt") return sort;
327  throw new Error(`not a date sort: ${sort}`);
328}
329 
330/**
331 * The sort clauses for a date mode: newest-first on the stored date field,
332 * `_id` ascending as a deterministic tiebreak (required for stable
333 * `search_after` paging). Kept in ONE place so the client's cursor matches the
334 * request's sort order exactly.
335 */
336export function dateSortClauses(sort: SearchSort): unknown[] {
337  return [{ [dateSortField(sort)]: "desc" }, { _id: "asc" }];
338}
339 
340/** Date-sort cursor = base64 JSON `[dateIso, id]` (the `search_after` tuple). */
341export function encodeDateCursor(tuple: [string, string]): string {
342  return Buffer.from(JSON.stringify(tuple), "utf8").toString("base64url");
343}
344 
345export function decodeDateCursor(cursor: string): [string, string] {
346  const raw = Buffer.from(cursor, "base64url").toString("utf8");
347  const parsed: unknown = JSON.parse(raw);
348  if (!Array.isArray(parsed) || parsed.length !== 2) {
349    throw new Error("invalid date-sort cursor");
350  }
351  return [parsed[0] as string, parsed[1] as string];
352}
353 
354/** Apply a date-sort + (optional) `search_after` to a body (additive). */
355export function withDateSort(
356  body: Record<string, unknown>,
357  sort: SearchSort,
358  tuple?: [string, string],
359): Record<string, unknown> {
360  const out: Record<string, unknown> = { ...body, sort: dateSortClauses(sort) };
361  if (tuple) out.search_after = [...tuple];
362  return out;
363}
364