import { describe, expect, it } from "vitest"; import { buildIndexDocs, chunkDocId, dedupeToParent } from "./indexer.js"; import type { GrantSnapshot, IndexDocInput } from "./types.js"; const grants: GrantSnapshot = { public: false, workspace: true, user: false, teams: ["t1"], users: ["u1"] }; function input(partial: Partial & Pick): IndexDocInput { return { workspaceId: "ws1", title: "T", grants, archived: false, url: "/ws1/x", createdAt: "2025-01-01T00:00:00Z", updatedAt: "2025-01-02T00:00:00Z", ...partial, }; } describe("chunkDocId", () => { it("appends #n to the parent id", () => { expect(chunkDocId("page:pg_1", 0)).toBe("page:pg_1#0"); expect(chunkDocId("page:pg_1", 7)).toBe("page:pg_1#7"); }); }); describe("buildIndexDocs", () => { it("indexes a scalar type (record) as a single doc", () => { const docs = buildIndexDocs(input({ docId: "record:r1", type: "record", id: "r1", body: "any length at all..." })); expect(docs).toHaveLength(1); expect(docs[0]!.docId).toBe("record:r1"); expect(docs[0]!.parentDocId).toBeUndefined(); }); it("indexes a short page body as the parent doc itself (no #0)", () => { const docs = buildIndexDocs(input({ docId: "page:pg_1", type: "page", id: "pg_1", body: "short body" })); expect(docs).toHaveLength(1); expect(docs[0]!.docId).toBe("page:pg_1"); }); it("expands a long page into chunked docs with parentDocId", () => { const body = Array.from({ length: 30 }, (_, i) => `Paragraph ${i} with some text. ${"word ".repeat(20).trim()}`).join( "\n\n", ); const docs = buildIndexDocs( input({ docId: "page:pg_1", type: "page", id: "pg_1", body, vector: [0.1, 0.2] }), { maxChars: 300, overlapChars: 0 }, ); expect(docs.length).toBeGreaterThan(1); for (const [i, doc] of docs.entries()) { expect(doc.docId).toBe(`page:pg_1#${i}`); expect(doc.parentDocId).toBe("page:pg_1"); expect(doc.chunkIndex).toBe(i); expect(doc.grants).toBe(grants); // shared reference — same snapshot expect(doc.body.length).toBeLessThanOrEqual(300); } // the parent's single vector applies to chunk 0 only expect(docs[0]!.vector).toEqual([0.1, 0.2]); expect(docs[1]!.vector).toBeUndefined(); }); it("keeps grants/workspace/url on every chunk", () => { const body = Array.from({ length: 30 }, (_, i) => `Para ${i} ${"filler ".repeat(15).trim()}`).join("\n\n"); const docs = buildIndexDocs(input({ docId: "page:pg_9", type: "page", id: "pg_9", body }), { maxChars: 200, overlapChars: 0, }); for (const doc of docs) { expect(doc.workspaceId).toBe("ws1"); expect(doc.url).toBe("/ws1/x"); expect(doc.grants.workspace).toBe(true); } }); }); describe("dedupeToParent", () => { it("keeps the better-scoring of chunk vs parent and records the matched chunk", () => { const out = dedupeToParent([ { docId: "page:pg_1#2", parentDocId: "page:pg_1", score: 5 }, { docId: "page:pg_1", score: 3 }, { docId: "page:pg_2#0", parentDocId: "page:pg_2", score: 9 }, ]); expect(out).toHaveLength(2); expect(out[0]).toEqual({ docId: "page:pg_2", score: 9, matchedChunk: "page:pg_2#0" }); expect(out[1]).toEqual({ docId: "page:pg_1", score: 5, matchedChunk: "page:pg_1#2" }); }); it("keeps the parent when it outranks its own chunks", () => { const out = dedupeToParent([ { docId: "page:pg_1#0", parentDocId: "page:pg_1", score: 2 }, { docId: "page:pg_1", score: 7 }, ]); expect(out).toHaveLength(1); expect(out[0]).toEqual({ docId: "page:pg_1", score: 7, matchedChunk: undefined }); }); it("sorts by descending score", () => { const out = dedupeToParent([ { docId: "page:a", score: 1 }, { docId: "page:b", score: 4 }, { docId: "page:c", score: 2 }, ]); expect(out.map((d) => d.docId)).toEqual(["page:b", "page:c", "page:a"]); }); });