/** * @factory/search — indexing pipeline helpers (the indexer worker's pure * core). `buildIndexDocs` turns an {@link IndexDocInput} into the concrete * OpenSearch documents to index: chunked docs (page/block) become one doc * per chunk; scalar docs (database/record/comment/file/user) become one doc. * * Chunked docs share the parent's grants/visibility (search §5.1: the grant * snapshot is per resource, not per chunk) and carry `parentDocId` so the * query layer can de-duplicate to the parent when a chunk hits. */ import { chunkText } from "./chunking.js"; import type { ChunkOptions, IndexDocInput, SearchDoc } from "./types.js"; /** Doc ids use the `:` convention; chunks append `#`. */ export function chunkDocId(parentDocId: string, chunkIndex: number): string { return `${parentDocId}#${chunkIndex}`; } /** * Expand one indexer input into indexable docs. Chunking is only applied to * chunked kinds (page, block); the rest index as a single doc. */ export function buildIndexDocs(input: IndexDocInput, options?: ChunkOptions): SearchDoc[] { const base: SearchDoc = { docId: input.docId, type: input.type, id: input.id, workspaceId: input.workspaceId, title: input.title, body: input.body, grants: input.grants, archived: input.archived, url: input.url, createdAt: input.createdAt, updatedAt: input.updatedAt, }; if (input.teamId !== undefined) base.teamId = input.teamId; if (input.chunkId !== undefined) base.chunkId = input.chunkId; if (input.vectorModel !== undefined) base.vectorModel = input.vectorModel; if (input.createdBy !== undefined) base.createdBy = input.createdBy; if (input.createdByName !== undefined) base.createdByName = input.createdByName; if (input.icon !== undefined) base.icon = input.icon; const chunked = input.type === "page" || input.type === "block"; if (!chunked || input.body.length === 0) { // The input's single `vector` (if any) indexes the doc as a whole. return [input.vector !== undefined ? { ...base, vector: input.vector } : base]; } const opts: Required = { maxChars: options?.maxChars ?? 800, overlapChars: options?.overlapChars ?? 120, }; const chunks = chunkText(input.body, opts); if (chunks.length <= 1) { // single chunk: index as the parent doc itself (no `#0` suffix docs) return [input.vector !== undefined ? { ...base, vector: input.vector } : base]; } return chunks.map((chunk) => ({ ...base, docId: chunkDocId(input.docId, chunk.chunkIndex), body: chunk.text, chunkIndex: chunk.chunkIndex, parentDocId: input.docId, // vectors are per-chunk; the input's single `vector` (if any) applies to // chunk 0 only — real chunks are embedded individually by the worker. ...(chunk.chunkIndex === 0 && input.vector !== undefined ? { vector: input.vector } : {}), })); } /** * De-duplicate search results to parent docs: when a chunk * (`page:pg_x#3`) and its parent (`page:pg_x`) both hit, keep the one with * the better score and remember which chunk produced it (for the snippet). */ export function dedupeToParent(docs: { docId: string; parentDocId?: string; score: number }[]): { docId: string; score: number; matchedChunk?: string; }[] { const byParent = new Map(); for (const doc of docs) { const key = doc.parentDocId ?? doc.docId; const existing = byParent.get(key); if (!existing || doc.score > existing.score) { const entry: { docId: string; score: number; matchedChunk?: string } = { docId: key, score: doc.score, }; if (doc.parentDocId) entry.matchedChunk = doc.docId; byParent.set(key, entry); } } return [...byParent.values()].sort((a, b) => b.score - a.score); }