indexer.ts · Source

indexer.tspackages/search/src/indexer.tsraw ← src3,858 B← Back to the app
1/**
2 * @factory/search — indexing pipeline helpers (the indexer worker's pure
3 * core). `buildIndexDocs` turns an {@link IndexDocInput} into the concrete
4 * OpenSearch documents to index: chunked docs (page/block) become one doc
5 * per chunk; scalar docs (database/record/comment/file/user) become one doc.
6 *
7 * Chunked docs share the parent's grants/visibility (search §5.1: the grant
8 * snapshot is per resource, not per chunk) and carry `parentDocId` so the
9 * query layer can de-duplicate to the parent when a chunk hits.
10 */
11 
12import { chunkText } from "./chunking.js";
13import type { ChunkOptions, IndexDocInput, SearchDoc } from "./types.js";
14 
15/** Doc ids use the `<type>:<id>` convention; chunks append `#<n>`. */
16export function chunkDocId(parentDocId: string, chunkIndex: number): string {
17  return `${parentDocId}#${chunkIndex}`;
18}
19 
20/**
21 * Expand one indexer input into indexable docs. Chunking is only applied to
22 * chunked kinds (page, block); the rest index as a single doc.
23 */
24export function buildIndexDocs(input: IndexDocInput, options?: ChunkOptions): SearchDoc[] {
25  const base: SearchDoc = {
26    docId: input.docId,
27    type: input.type,
28    id: input.id,
29    workspaceId: input.workspaceId,
30    title: input.title,
31    body: input.body,
32    grants: input.grants,
33    archived: input.archived,
34    url: input.url,
35    createdAt: input.createdAt,
36    updatedAt: input.updatedAt,
37  };
38  if (input.teamId !== undefined) base.teamId = input.teamId;
39  if (input.chunkId !== undefined) base.chunkId = input.chunkId;
40  if (input.vectorModel !== undefined) base.vectorModel = input.vectorModel;
41  if (input.createdBy !== undefined) base.createdBy = input.createdBy;
42  if (input.createdByName !== undefined) base.createdByName = input.createdByName;
43  if (input.icon !== undefined) base.icon = input.icon;
44 
45  const chunked = input.type === "page" || input.type === "block";
46  if (!chunked || input.body.length === 0) {
47    // The input's single `vector` (if any) indexes the doc as a whole.
48    return [input.vector !== undefined ? { ...base, vector: input.vector } : base];
49  }
50 
51  const opts: Required<ChunkOptions> = {
52    maxChars: options?.maxChars ?? 800,
53    overlapChars: options?.overlapChars ?? 120,
54  };
55  const chunks = chunkText(input.body, opts);
56  if (chunks.length <= 1) {
57    // single chunk: index as the parent doc itself (no `#0` suffix docs)
58    return [input.vector !== undefined ? { ...base, vector: input.vector } : base];
59  }
60  return chunks.map((chunk) => ({
61    ...base,
62    docId: chunkDocId(input.docId, chunk.chunkIndex),
63    body: chunk.text,
64    chunkIndex: chunk.chunkIndex,
65    parentDocId: input.docId,
66    // vectors are per-chunk; the input's single `vector` (if any) applies to
67    // chunk 0 only — real chunks are embedded individually by the worker.
68    ...(chunk.chunkIndex === 0 && input.vector !== undefined ? { vector: input.vector } : {}),
69  }));
70}
71 
72/**
73 * De-duplicate search results to parent docs: when a chunk
74 * (`page:pg_x#3`) and its parent (`page:pg_x`) both hit, keep the one with
75 * the better score and remember which chunk produced it (for the snippet).
76 */
77export function dedupeToParent(docs: { docId: string; parentDocId?: string; score: number }[]): {
78  docId: string;
79  score: number;
80  matchedChunk?: string;
81}[] {
82  const byParent = new Map<string, { docId: string; score: number; matchedChunk?: string }>();
83  for (const doc of docs) {
84    const key = doc.parentDocId ?? doc.docId;
85    const existing = byParent.get(key);
86    if (!existing || doc.score > existing.score) {
87      const entry: { docId: string; score: number; matchedChunk?: string } = {
88        docId: key,
89        score: doc.score,
90      };
91      if (doc.parentDocId) entry.matchedChunk = doc.docId;
92      byParent.set(key, entry);
93    }
94  }
95  return [...byParent.values()].sort((a, b) => b.score - a.score);
96}
97