/** * @factory/search — thin OpenSearch client over an injectable fetch * (mirrors the `@factory/storage` S3 client pattern: pure transport, no SDK). * * The client knows the OpenSearch HTTP API only; query shapes come from * `queries.ts`, fusion from `rrf.ts`. Failures surface as `ApiError` with * `SEARCH_UNAVAILABLE` (503) for transport/429/5xx errors. * * Hybrid search (search-architecture §4.5) is executed as TWO requests — * BM25 (`_search` with the full-text body) and kNN (`_search` with a kNN-only * body) — and the two rank lists are fused client-side with RRF (k = 60). * Recorded decision: separate requests over the combined `query + knn` body, * because the combined body's `_score` is BM25-dominated and the kNN ranking * is not recoverable from it; two requests also make the two RRF inputs exact. */ // NOTE: the API consumes this package under `node10` module resolution, // which only sees the package ROOT (no subpaths) — hence `@factory/shared` // (root re-export), not `@factory/shared/errors`. See the Node10 note in // packages/shared/src/index.ts. import { ApiError, ErrorCodes } from "@factory/shared"; import { CONTENT_INDEX_ALIAS, buildFullTextBody, buildHybridKnnOnlyBody, buildHybridBody, buildPrefixBody, contentIndex, dateSortField, decodeCursor, decodeDateCursor, encodeCursor, encodeDateCursor, withDateSort, withSearchAfter, } from "./queries.js"; import { fuseHybrid } from "./rrf.js"; import { CHUNK_DOC_TYPE, displayType } from "./types.js"; import type { AnswerCitation, SearchDoc, SearchQuery, SearchResult, SearchResponse, Suggestion, ViewerContext, } from "./types.js"; export interface SearchClientOptions { /** Base URL, e.g. `http://opensearch:9200`. */ baseUrl: string; /** Injectable fetch (defaults to global fetch) — enables offline tests. */ fetchImpl?: typeof fetch; /** Basic auth (dev); production uses IAM/signing at the network edge. */ auth?: { username: string; password: string }; /** Request timeout in ms (default 5000). */ timeoutMs?: number; now?: () => number; /** kNN list depth per hybrid pass (default 50, per search §4.5 `k: 50`). */ knnK?: number; } interface OsHit { _id: string; _score?: number; _source: SearchDoc & { highlight?: { title?: string[]; body?: string[] } }; } interface OsResponse { hits?: { total?: { value?: number }; hits?: OsHit[]; }; } export class SearchClient { readonly baseUrl: string; readonly fetchImpl: typeof fetch; private readonly headers: Record; private readonly timeoutMs: number; private readonly now: () => number; private readonly knnK: number; constructor(options: SearchClientOptions) { this.baseUrl = options.baseUrl.replace(/\/+$/, ""); this.fetchImpl = options.fetchImpl ?? fetch; this.headers = { "content-type": "application/json" }; if (options.auth) { this.headers.authorization = "Basic " + Buffer.from(`${options.auth.username}:${options.auth.password}`).toString("base64"); } this.timeoutMs = options.timeoutMs ?? 5000; this.now = options.now ?? Date.now; this.knnK = options.knnK ?? 50; } /** Index one (or many chunked) documents via `_bulk` (single content index). */ async indexDocs(docs: SearchDoc[]): Promise<{ indexed: number }> { if (docs.length === 0) return { indexed: 0 }; const lines: string[] = []; for (const doc of docs) { lines.push(JSON.stringify({ index: { _index: contentIndex(), _id: doc.docId } })); lines.push(JSON.stringify(doc)); } const res = await this.request("/_bulk", { method: "POST", body: lines.join("\n") + "\n" }); const parsed = (await res.json()) as { errors?: boolean; items?: { index?: { status?: number } }[] }; let indexed = 0; for (const item of parsed.items ?? []) { if (item.index && (item.index.status ?? 200) < 300) indexed++; } return { indexed }; } /** Delete one document by doc id (single content index). */ async deleteDoc(docId: string): Promise { await this.request(`/${contentIndex()}/_doc/${encodeURIComponent(docId)}`, { method: "DELETE" }); } /** * Delete ALL chunk docs belonging to the given parent doc id(s) (TASK-0099 §3). * Chunk docs have no `search_documents` mirror rows, so the worker cannot * enumerate them by id (a shrunk body changes the count) — it deletes them in * a parent-driven pass instead, matching on `parentDocId` AND * `type: "chunk"` (both keyword fields in the mapping). One request handles * any number of parents. A missing parent (404) is a no-op, like * {@link deleteDoc}; 0 deletes is fine (idempotent). */ async deleteChunksOf(parentDocIds: string | string[]): Promise { const ids = Array.isArray(parentDocIds) ? parentDocIds : [parentDocIds]; if (ids.length === 0) return 0; const parentClause = ids.length === 1 ? { term: { parentDocId: ids[0]! } } : { terms: { parentDocId: ids } }; const body = JSON.stringify({ query: { bool: { filter: [parentClause, { term: { type: CHUNK_DOC_TYPE } }], }, }, }); const res = await this.request(`/${contentIndex()}/_delete_by_query?conflicts=proceed`, { method: "POST", body, }); const parsed = (await res.json()) as { deleted?: number }; return parsed.deleted ?? 0; } /** * Partial merge-update of one document by doc id (e.g. the AI `vector` * field on a chunk doc — a full PUT would clobber title/body/grants). */ async updateDoc(docId: string, doc: Record): Promise { await this.request(`/${contentIndex()}/_update/${encodeURIComponent(docId)}`, { method: "POST", body: JSON.stringify({ doc }), }); } /** True when the content index/alias exists. */ async indexExists(indexOrAlias: string = contentIndex()): Promise { try { await this.request(`/${indexOrAlias}`, {}); return true; } catch { return false; } } /** * T-177 (search health) — the document count of the content index, * optionally scoped to a set of tenant workspaces. Counts ONLY whole docs * (`type != "chunk"` — chunk docs are the RAG expansion of a parent doc and * would double-count content). `size: 0` (no hits returned). A missing * index resolves to 0 (the alias is created lazily on first write — 0 is * the honest "nothing indexed yet", distinct from `indexExists` which is a * separate probe). A 5xx/network failure throws `SEARCH_UNAVAILABLE` (via * {@link request}) — the caller maps that to `healthy: false`, never to a * guessed count. */ async countDocs(workspaceIds?: string[]): Promise { const filter: Record[] = [{ bool: { must_not: [{ term: { type: CHUNK_DOC_TYPE } }] }, }]; if (workspaceIds && workspaceIds.length > 0) { filter.push({ terms: { workspaceId: workspaceIds } }); } const out = await this.osRequest( `/${contentIndex()}/_search`, { query: { bool: { filter } }, size: 0 } ); return out.hits?.total?.value ?? 0; } /** * Create the content index (behind the alias) if missing (idempotent startup * path). `mappings` is the full `{settings, mappings}` body from * `contentMapping()`. */ async createIndexIfMissing( indexName: string, body: { settings?: Record; mappings?: Record }, ): Promise { try { await this.request(`/${indexName}`, { method: "PUT", body: JSON.stringify(body) }); return true; } catch (err) { // resource_already_exists_engine_exception → already there (idempotent). if (err instanceof ApiError && /resource_already_exists/.test(err.message)) return false; throw err; } } /** * Create the content index AND point the alias at it (used by the * `scripts/opensearch-reindex.ts` rollover). `actions` swap is atomic. */ async createAlias(indexName: string, alias: string = CONTENT_INDEX_ALIAS): Promise { await this.request("/_aliases", { method: "POST", body: JSON.stringify({ actions: [ { remove: { index: indexName, alias } }, { add: { index: indexName, alias } }, ], }), }); } /** * Whether `name` is an ALIAS (as opposed to a real index). 200 → alias, * 404 → not an alias (either a real index, or nothing). Used by the * self-heal path to detect the corrupted "alias name became a real index" * state (OpenSearch was down at boot; the first `_bulk` auto-created a bare * real index behind the alias name). */ async isAlias(name: string = contentIndex()): Promise { try { await this.request(`/_alias/${name}`, {}); return true; } catch (err) { if (err instanceof ApiError && err.status === 404) return false; throw err; // 5xx / transport → treat as a real failure. } } /** Delete an index by name (the self-heal path rebuilds it with a mapping). */ async deleteIndex(name: string): Promise { await this.request(`/${name}`, { method: "DELETE" }); } /** Force a refresh (make recent writes queryable — tests / immediate lag). */ async refresh(indexOrAlias: string = contentIndex()): Promise { await this.request(`/${indexOrAlias}/_refresh`, {}); } /** The live index's `dense_vector` dimension (rollover guard, search §7.3). */ async getVectorDimension(indexOrAlias: string = contentIndex()): Promise { const res = await this.request(`/${indexOrAlias}/_mapping`, {}); const parsed = (await res.json()) as Record; for (const key of Object.keys(parsed)) { const dim = parsed[key]?.mappings?.properties?.vector?.dimension; if (typeof dim === "number") return dim; } return null; } /** * Which of `docIds` exist in the index (batched `_mget`, one request per 500). * Returns a Set of the PRESENT ids. Used by the reconcile sweep to detect * docs deleted/lost in the index (the contentHash sweep alone only catches * SOURCE drift, not index loss). */ async docExists(docIds: string[]): Promise> { const present = new Set(); if (docIds.length === 0) return present; for (let i = 0; i < docIds.length; i += 500) { const chunk = docIds.slice(i, i + 500); const res = await this.request(`/${contentIndex()}/_mget`, { method: "POST", body: JSON.stringify({ ids: chunk }), }); const parsed = (await res.json()) as { docs?: Array<{ found?: boolean; _id?: string }> }; for (const d of parsed.docs ?? []) { if (d.found && d._id) present.add(d._id); } } return present; } /** * TASK-0125 — fetch the `_source` of explicit doc ids (mget, 500-batched). * The RAG query path uses it for the FULL chunk text (the search API's * 160-char snippet is too thin for prompt context). Missing docs (deleted * between retrieval and context assembly) are silently omitted — the * caller drops their context block and keeps the answer consistent. */ async getDocs(docIds: string[]): Promise }>> { const out: Array<{ docId: string; source: Record }> = []; if (docIds.length === 0) return out; for (let i = 0; i < docIds.length; i += 500) { const chunk = docIds.slice(i, i + 500); const res = await this.request(`/${contentIndex()}/_mget`, { method: "POST", body: JSON.stringify({ ids: chunk }), }); const parsed = (await res.json()) as { docs?: Array<{ found?: boolean; _id?: string; _source?: Record }>; }; for (const d of parsed.docs ?? []) { if (d.found && d._id && d._source) out.push({ docId: d._id, source: d._source }); } } return out; } /** * Execute a search query. `fulltext` mode (default) is one request; * `hybrid` runs BM25 + kNN and fuses with RRF. The query is tenant-scoped by * the viewer's `workspaceIds` (single-index model — no index param). * * @param queryVector required for hybrid mode — the embedded query. */ async search( query: SearchQuery, viewer: ViewerContext, queryVector?: number[], ): Promise { const started = this.now(); const mode = query.mode ?? "fulltext"; if (mode === "hybrid" && !queryVector) { throw new ApiError(ErrorCodes.AI_ERROR, { message: "hybrid search requires queryVector" }); } if (mode === "hybrid") { return this.hybridSearch(query, viewer, queryVector!, started); } return this.fullTextSearch(query, viewer, started); } /** * Prefix suggestions ("search as you type", search-architecture §4.2): * prefix match on `title.keyword` (or the doc id) over the SAME * mandatory tenant + permission filters as full-text search, no highlight, * small size (default 10, cap 20). Returns minimal `{ id, type, title }` * items (the source row id, not the `docId` — the UI deep-links by row id). * * The API's `GET /search/suggest` endpoint wraps this (TASK-0097 S3). */ async suggest(prefix: string, viewer: ViewerContext, size = 10): Promise { const capped = Math.max(1, Math.min(size, 20)); const body = buildPrefixBody(prefix, viewer, capped); const os = await this.osRequest(`/${contentIndex()}/_search`, body); const hits = os.hits?.hits ?? []; const seen = new Set(); const out: Suggestion[] = []; for (const h of hits) { const s = h._source; // De-dupe by (type, id): chunk docs share the parent's title, so the // display type (chunk → parent kind) makes a chunk hit collapse into // its parent's suggestion. const type = displayType(s); if (seen.has(`${type}:${s.id}`)) continue; seen.add(`${type}:${s.id}`); out.push({ id: s.id, type, title: s.title }); if (out.length >= capped) break; } return out; } private async fullTextSearch( query: SearchQuery, viewer: ViewerContext, started: number, ): Promise { // T-155 date sorts are an ADDITIVE branch — `undefined`/`"relevance"` falls // through to the original `_score` path below, byte-for-byte unchanged. if (query.sortBy === "createdAt" || query.sortBy === "updatedAt") { return this.dateSortedSearch(query, viewer, started); } let body: Record = buildFullTextBody(query, viewer); if (query.page) body = withSearchAfter(body, decodeCursor(query.page)); const os = await this.osRequest(`/${contentIndex()}/_search`, body); const hits = os.hits?.hits ?? []; const pageSize = query.pageSize ?? 25; const results = hits.map((h) => this.mapHit(h, "bm25")); const last = hits[hits.length - 1]; const cursor = hits.length >= pageSize && last ? encodeCursor([last._score ?? 0, last._source.updatedAt, last._id]) : undefined; return { results, total: os.hits?.total?.value ?? hits.length, ...(cursor !== undefined ? { cursor } : {}), latencyMs: this.now() - started, }; } /** * T-155 — date-ordered search (`sortBy: "createdAt" | "updatedAt"`): * newest-first on the stored date field (`_id` ascending tiebreak), with its * own `(dateIso, id)` cursor. The relevance path is deliberately not touched * (see `fullTextSearch`). The same mandatory tenant + permission + type + * chip filters apply — `buildFullTextBody` builds them. */ private async dateSortedSearch( query: SearchQuery, viewer: ViewerContext, started: number, ): Promise { const field = dateSortField(query.sortBy!); let body: Record = buildFullTextBody(query, viewer); if (query.page) body = withDateSort(body, query.sortBy!, decodeDateCursor(query.page)); else body = withDateSort(body, query.sortBy!); const os = await this.osRequest(`/${contentIndex()}/_search`, body); const hits = os.hits?.hits ?? []; const pageSize = query.pageSize ?? 25; const results = hits.map((h) => this.mapHit(h, "bm25")); const last = hits[hits.length - 1]; const lastDate = last?._source[field]; const cursor = hits.length >= pageSize && last && lastDate ? encodeDateCursor([lastDate, last._id]) : undefined; return { results, total: os.hits?.total?.value ?? hits.length, ...(cursor !== undefined ? { cursor } : {}), latencyMs: this.now() - started, }; } private async hybridSearch( query: SearchQuery, viewer: ViewerContext, queryVector: number[], started: number, ): Promise { const pageSize = query.pageSize ?? 25; // Pass 1: BM25 (full-text body, sized to the fusion depth). TASK-0125: // pass 1 is BM25-only — OpenSearch 2.15's `query.knn` cannot share a // request body with a `bool`/full-text query, so the vector leg is its // own request (pass 2) and the two are fused client-side with RRF. const bm25Body: Record = { ...buildHybridBody(query, viewer, queryVector), size: this.knnK, }; // Pass 2: kNN-only (`query.knn`, pre-filtered), same mandatory filters. const knnBody = buildHybridKnnOnlyBody(queryVector, viewer, this.knnK); const [bm25Os, knnOs] = await Promise.all([ this.osRequest(`/${contentIndex()}/_search`, bm25Body), this.osRequest(`/${contentIndex()}/_search`, knnBody), ]); const bm25Hits = bm25Os.hits?.hits ?? []; const knnHits = knnOs.hits?.hits ?? []; const bm25List = bm25Hits.map((h) => h._id); const vecList = knnHits.map((h) => h._id); const fused = fuseHybrid(bm25List, vecList); const byId = new Map(); for (const h of bm25Hits) byId.set(h._id, h); for (const h of knnHits) if (!byId.has(h._id)) byId.set(h._id, h); const results: SearchResult[] = []; for (const f of fused) { const hit = byId.get(f.docId); if (!hit) continue; const source: SearchResult["source"] = f.sources.includes("bm25") && f.sources.includes("vec") ? "bm25+vec" : f.sources.includes("vec") ? "vec" : "bm25"; results.push({ ...this.mapHit(hit, source), score: f.score }); if (results.length >= pageSize) break; } return { results, total: bm25Os.hits?.total?.value ?? results.length, latencyMs: this.now() - started, }; } private mapHit(hit: OsHit, source: SearchResult["source"]): SearchResult { const s = hit._source; const hl = s.highlight; const snippet = hl?.body?.[0] ?? s.body.slice(0, 160); const result: SearchResult = { docId: hit._id, // Chunk docs surface as their parent's public kind (displayType); // whole docs map to themselves. type: displayType(s), id: s.id, title: s.title, snippet, url: s.url, updatedAt: s.updatedAt, workspaceId: s.workspaceId, score: hit._score ?? 0, source, }; if (s.icon !== undefined) result.icon = s.icon; if (s.createdByName !== undefined) result.createdByName = s.createdByName; if (s.pageId !== undefined) result.pageId = s.pageId; if (s.blockId !== undefined) result.blockId = s.blockId; if (s.databaseId !== undefined) result.databaseId = s.databaseId; if (s.recordId !== undefined) result.recordId = s.recordId; return result; } private async request(path: string, init: { method?: string; body?: string }): Promise { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), this.timeoutMs); try { const res = await this.fetchImpl(this.baseUrl + path, { method: init.method ?? "GET", headers: this.headers, body: init.body ?? null, signal: controller.signal, }); if (!res.ok) { throw new ApiError(ErrorCodes.SEARCH_UNAVAILABLE, { message: `OpenSearch ${res.status} on ${path}`, status: res.status, }); } return res; } catch (err) { if (err instanceof ApiError) throw err; throw new ApiError(ErrorCodes.SEARCH_UNAVAILABLE, { message: err instanceof Error ? err.message : "search request failed", }); } finally { clearTimeout(timer); } } private async osRequest(path: string, body: Record): Promise { const res = await this.request(path, { method: "POST", body: JSON.stringify(body) }); return (await res.json()) as T; } } /** Build RAG citations from the top search results (ai-architecture §6). */ export function citationsFromResults(results: SearchResult[], limit = 5): AnswerCitation[] { return results.slice(0, limit).map((r) => ({ docId: r.docId, title: r.title, url: r.url, quote: r.snippet ?? "", relevance: Math.min(1, r.score), })); }