/** * @factory/search — OpenSearch query builders (search-architecture §4–§5). * * Pure functions: they turn a {@link SearchQuery} + {@link ViewerContext} into * the exact OpenSearch request body. The `client.ts` transport layer applies * them; apps may also use them directly. * * Phase-8 reconciliation (recorded decision, see docs/architecture/search.md * § "Layout decision"): * - **Single index** (alias `factory-content`), not one index per workspace. * Tenant isolation is the mandatory `workspaceId` filter, applied in every * builder — it is a parameter, never optional. * - **camelCase field paths** matching the stored {@link SearchDoc} (the * Phase-0 scaffold mixed snake_case / `grants.*` / `title.raw`, which did * not match the stored doc). `title.keyword` backs prefix + fuzzy. * * Permission model (search-architecture §5): the visibility filter is a * `must`-equivalent `filter` clause — docs fail it when the viewer is not * public / workspace-visible / user-granted / team-granted. The viewer's team * ids are inlined (the snapshot stores team ids, not memberships, so the * expansion happens here). */ import { CHUNK_DOC_TYPE } from "./types.js"; import type { SearchQuery, SearchSort, ViewerContext } from "./types.js"; /** The single content index (alias). All typed docs live here (search §2). */ export const CONTENT_INDEX_ALIAS = "factory-content"; /** * The content index to read from / write to. Currently the alias; a rollover * (search §2/§7.2) swaps the backing index behind the same alias, so callers * never hard-code a versioned name. */ export function contentIndex(): string { return CONTENT_INDEX_ALIAS; } /** Fields matched by the full-text query (title weighted 3× over body). */ const FULLTEXT_FIELDS: Record = { title: 3, body: 1, createdByName: 0.5, }; interface BoolClause { must?: unknown[]; must_not?: unknown[]; should?: unknown[]; filter?: unknown[]; minimum_should_match?: number; } function bool(clause: BoolClause): { bool: BoolClause } { return { bool: clause }; } /** * The tenant clause: restrict to the caller's workspaces. Non-negotiable — * every query filters on it (search §3 "tenant filter — ALWAYS present"). */ export function tenantFilter(viewer: ViewerContext): unknown { return { terms: { workspaceId: viewer.workspaceIds } }; } /** The visibility / permission filter (search-architecture §5.2). */ export function visibilityFilter(viewer: ViewerContext): unknown { // A doc is visible iff any of: a public share; the viewer's workspace grants // it (non-private workspace content — the tenant filter already scopes this // clause to the caller's workspaces, so `workspace: true` never over-surfaces // a foreign workspace); an explicit user grant; or a team grant. // TASK-0100 S5: explicit `term`/`terms` clauses. The old raw field-value // form ({ "grants.public": true }) is only legal inside a `match` body, not // a bool context — OpenSearch rejected the whole query ("[grants.public] // query malformed, no start_object after query name"), so EVERY content // search 400'd. (The field NAMES are correct — they match the doc shape // the builders emit; only the clause syntax was broken.) const or: unknown[] = [ { term: { "grants.public": true } }, { term: { "grants.workspace": true } }, { term: { "grants.users": viewer.userId } }, ]; if (viewer.teamIds.length > 0) or.push({ terms: { "grants.teams": viewer.teamIds } }); return bool({ should: or, minimum_should_match: 1 }); } /** * The keyword-search chunk exclusion (TASK-0099, spec req 3): chunk docs * (`type: "chunk"`) are the kNN/hybrid leg's target, never keyword hits. * The public `types` filter alone CANNOT guarantee this — the live API omits * the `types` clause entirely when unspecified (search.service.ts) — so the * exclusion is a mandatory `must_not` in every keyword body. Keyword * coverage is unaffected: a chunk's text lives in its parent's whole-doc * `body`, so any keyword match a chunk would make is already matched by the * parent doc. */ export function chunkExclusion(): unknown { return { term: { type: CHUNK_DOC_TYPE } }; } /** * The mandatory filter array for a content query: tenant + permission + * type/chip filters. Shared by the full-text and hybrid builders so the * tenant/permission clauses cannot drift between them. */ function baseFilters(viewer: ViewerContext, query: SearchQuery): unknown[] { const filter: unknown[] = [tenantFilter(viewer), visibilityFilter(viewer)]; if (query.types && query.types.length > 0) { filter.push({ terms: { type: [...query.types] } }); } const f = query.filters; if (f) { if (f.teams && f.teams.length > 0) filter.push({ terms: { teamId: f.teams } }); if (f.createdBy) filter.push({ term: { createdBy: f.createdBy } }); if (f.updatedAfter) filter.push({ range: { updatedAt: { gte: f.updatedAfter } } }); if (f.updatedBefore) filter.push({ range: { updatedAt: { lte: f.updatedBefore } } }); if (f.databases && f.databases.length > 0) filter.push({ terms: { databaseId: f.databases } }); if (f.archived !== undefined) filter.push({ term: { archived: f.archived } }); } // Archived docs are hidden by default (explicit `archived: true` opts in). if (f?.archived !== true) filter.push({ term: { archived: false } }); return filter; } /** Build the full-text (default mode) OpenSearch body. */ export function buildFullTextBody(query: SearchQuery, viewer: ViewerContext): Record { return { size: query.pageSize ?? 25, track_total_hits: 10000, query: bool({ must: [ { multi_match: { query: query.q, fields: Object.entries(FULLTEXT_FIELDS).map(([f, boost]) => `${f}^${boost}`), type: "best_fields", operator: "or", }, }, ], filter: baseFilters(viewer, query), // Whole-doc keyword search excludes chunk docs (TASK-0099, spec req 3). must_not: [chunkExclusion()], }), highlight: { fields: { title: {}, body: { fragment_size: 160, number_of_fragments: 1 } }, }, }; } /** Prefix query for autocomplete / "search as you type" (search §4.2). */ export function buildPrefixBody(prefix: string, viewer: ViewerContext, size = 10): Record { const p = prefix.toLowerCase(); return { size, query: bool({ must: [ bool({ should: [ { prefix: { "title.keyword": p } }, // Spec §4.2 requires a CASE-INSENSITIVE prefix (the `factory_prefix` // analyzer lowercases). `title.keyword` is case-sensitive, so a // lowercased prefix against it misses any differently-cased title // (e.g. an uppercase "Q3" title vs the lowercased "q3" prefix) → // autocomplete returned [] while full-text found the same page. // `match_phrase_prefix` on the analyzed (lowercased) `title` field // restores the spec'd case-insensitive behavior WITHOUT rolling the // index forward to add the analyzer (impossible in place — analyzer // settings are immutable on the live `factory-content` index; a // reindex is the recorded upgrade). Phrase semantics keep it tight: // all prefix tokens must appear in order (last one as a prefix), // so it does not over-match the way a plain wildcard on the text // field would (the analyzer would split a hyphenated token into // separate terms). { match_phrase_prefix: { title: p } }, { prefix: { id: p } }, ], minimum_should_match: 1, }), ], filter: [tenantFilter(viewer), visibilityFilter(viewer)], must_not: [chunkExclusion()], }), }; } /** Explicit fuzzy query ("did you mean", typo-heavy input, search §4.3). */ export function buildFuzzyBody( query: string, viewer: ViewerContext, options: { fuzziness?: "AUTO" | 0 | 1 | 2 } = {}, size = 20, ): Record { return { size, query: bool({ must: [ { fuzzy: { "title.keyword": { value: query, fuzziness: options.fuzziness ?? "AUTO" }, }, }, ], filter: [tenantFilter(viewer), visibilityFilter(viewer)], must_not: [chunkExclusion()], }), }; } /** * Hybrid search (search §4.5): BM25 + kNN. The client executes both paths * (see the recorded two-request decision in `client.ts`); this returns the * BM25 (pass 1) body. The vector leg runs as a separate `query.knn` request * (`buildHybridKnnOnlyBody`) and the client fuses the two with RRF * (`buildHybridFusion`, k=60). * * TASK-0125 (recorded deviation, verified live against 2.15): pass 1 is now * full-text ONLY. The old body also carried an ES-style top-level `knn` * block, which OpenSearch 2.15 rejects (`Unknown key for a START_OBJECT in * [knn]` — 400 on every hybrid search); the Phase-8 scaffold shipped that * shape but the kNN leg was never exercised live (Phase-8 integration only * ran full-text). And the working `query.knn` clause cannot be combined with * a `bool`/full-text query in one request body — it must be the SOLE top- * level `query` — so the two-request + RRF design is the way to hybridize. * * `queryVector` is the embedded query (from `@factory/ai` or the stub). */ export function buildHybridBody( query: SearchQuery, viewer: ViewerContext, queryVector: number[], options?: { knnK?: number }, ): Record { void queryVector; void options; return buildFullTextBody(query, viewer); } /** * kNN-only body for the hybrid pass 2 (see `client.ts` — recorded decision: * hybrid = two requests + client-side RRF, so the kNN pass needs its own body * carrying only the vector clause + the same mandatory filters). */ export function buildHybridKnnOnlyBody( queryVector: number[], viewer: ViewerContext, k = 50, ): Record { return { size: k, query: { knn: knnBlock(queryVector, viewer, k), }, }; } /** * The `query.knn.` vector clause (OpenSearch 2.15). * * TASK-0125 (recorded deviation, verified live against 2.15): the query DSL * is the **field-keyed `query.knn`** form — `{ "query": { "knn": { "vector": * { "vector": [...], "k", "filter" } } } }` — NOT the ES-style top-level * `knn: { field, query_vector }` block (400 on this build, see above). The * inline `filter` is the tenant + visibility clause, applied as a * **pre-filter** (FAISS engine — nmslib hard-rejects kNN filters, "Engine * [NMSLIB] does not support filters") so the ANN search only considers the * viewer's own docs. With the shared multi-tenant index that pre-filtering * is a security/recall requirement: without it the ANN fetch would return * the GLOBAL top-K (cross-tenant candidates in the fusion, and a small * tenant's docs could never reach the top-50). * * `num_candidates` is NOT part of the `query.knn` grammar on 2.15 (`[knn] * query does not support [num_candidates]`); search depth is set index-side * via `knn.algo_param.ef_search` (see `contentMapping`). */ function knnBlock( queryVector: number[], viewer: ViewerContext, k: number, ): Record { return { vector: { vector: [...queryVector], k, filter: bool({ filter: [tenantFilter(viewer), visibilityFilter(viewer)] }), }, }; } /** Cursor = base64 JSON of the search_after tuple (score, updated_at, id). */ export function encodeCursor(tuple: [number, string, string]): string { return Buffer.from(JSON.stringify(tuple), "utf8").toString("base64url"); } export function decodeCursor(cursor: string): [number, string, string] { const raw = Buffer.from(cursor, "base64url").toString("utf8"); const parsed: unknown = JSON.parse(raw); if (!Array.isArray(parsed) || parsed.length !== 3) { throw new Error("invalid cursor"); } return parsed as [number, string, string]; } /** * Attach `search_after` to a body from the previous page's last hit * (search_after is score + sort tiebreakers, per search §6). The tuple is * `(score, updatedAt, id)` matching the `_score`/`updatedAt`/`id` sort order. */ export function withSearchAfter(body: Record, tuple: [number, string, string]): Record { return { ...body, sort: [{ _score: "desc" }, { updatedAt: "desc" }, { id: "asc" }], search_after: [...tuple], }; } // --------------------------------------------------------------------------- // T-155 — first-class date sorts (spec §10: "creation date / modification // date"). ADDITIVE branches: the relevance path above is untouched, and the // two date modes use their OWN `(field, id)` cursor + sort clauses. Both date // fields are `type: "date"` in the index (mapping.ts), so no reindex. // --------------------------------------------------------------------------- /** The indexed date field a date `sortBy` orders on (throws on unknown). */ export function dateSortField(sort: SearchSort): "createdAt" | "updatedAt" { if (sort === "createdAt" || sort === "updatedAt") return sort; throw new Error(`not a date sort: ${sort}`); } /** * The sort clauses for a date mode: newest-first on the stored date field, * `_id` ascending as a deterministic tiebreak (required for stable * `search_after` paging). Kept in ONE place so the client's cursor matches the * request's sort order exactly. */ export function dateSortClauses(sort: SearchSort): unknown[] { return [{ [dateSortField(sort)]: "desc" }, { _id: "asc" }]; } /** Date-sort cursor = base64 JSON `[dateIso, id]` (the `search_after` tuple). */ export function encodeDateCursor(tuple: [string, string]): string { return Buffer.from(JSON.stringify(tuple), "utf8").toString("base64url"); } export function decodeDateCursor(cursor: string): [string, string] { const raw = Buffer.from(cursor, "base64url").toString("utf8"); const parsed: unknown = JSON.parse(raw); if (!Array.isArray(parsed) || parsed.length !== 2) { throw new Error("invalid date-sort cursor"); } return [parsed[0] as string, parsed[1] as string]; } /** Apply a date-sort + (optional) `search_after` to a body (additive). */ export function withDateSort( body: Record, sort: SearchSort, tuple?: [string, string], ): Record { const out: Record = { ...body, sort: dateSortClauses(sort) }; if (tuple) out.search_after = [...tuple]; return out; }