mapping.ts · Source

mapping.tspackages/search/src/mapping.tsraw ← src7,792 B← Back to the app
1/**
2 * @factory/search — the OpenSearch content-index mapping (search-architecture
3 * §2–§3).
4 *
5 * Phase-8 reconciliation (recorded decision, see docs/architecture/search.md
6 * § "Layout decision"): ONE index (alias `factory-content`) holds every typed
7 * doc; tenant isolation is the `workspaceId` filter applied at query time (the
8 * Phase-0 per-workspace `factory-search-<wid>` indices are retired). The field
9 * names are **camelCase**, matching the stored {@link SearchDoc} in `types.ts`
10 * exactly — this is the single source of truth the worker indexer and the query
11 * API both emit/consume.
12 *
13 * The `vector` field uses OpenSearch 2.x `dense_vector` with an HNSW index.
14 * The arch doc shows the older 1.x `knn_vector`/`knn: true` form; the docker +
15 * CI OpenSearch is 2.x, where `dense_vector` is the current type (recorded
16 * deviation). The dimension is a parameter: `dims` is immutable per index, so
17 * a model change rolls a new index forward via the alias (TASK-0099's
18 * `scripts/opensearch-reindex.ts`), never by re-mapping in place.
19 */
20 
21import type { SearchDocType, SearchType } from "./types.js";
22 
23/** HNSW construction params (search §3; tunable via SEARCH_KNN_* at S5). */
24export interface VectorIndexOptions {
25  m?: number;
26  efConstruction?: number;
27  dimension?: number;
28}
29 
30/** The default vector dimension (matches the TASK-0099 `text-embedding-3-small` target). */
31export const DEFAULT_VECTOR_DIMENSION = 1536;
32export const DEFAULT_KNN_M = 16;
33export const DEFAULT_KNN_EF_CONSTRUCTION = 100;
34 
35/**
36 * All indexed `type` values (so a bad doc `type` is rejected by the
37 * mapping's enum). TASK-0099 adds `"chunk"`: chunk docs live in the same
38 * index and the enum must accept them — but `"chunk"` is deliberately NOT in
39 * the public {@link SearchType} / `SEARCH_TYPES` (types.ts), so the API's
40 * `types` filter can never request chunk docs (keyword search stays
41 * whole-doc — recorded reconciliation, arch doc §0).
42 */
43export const TYPE_ENUM: SearchDocType[] = [
44  "page",
45  "block",
46  "database",
47  "record",
48  "comment",
49  "file",
50  "user",
51  "chunk",
52];
53 
54/**
55 * Build the `factory-content` index mapping.
56 *
57 * @param opts vector (dense_vector / HNSW) tuning. Omit to use the defaults
58 *   (1536-dim cosine HNSW, m=16, ef_construction=100).
59 */
60export function contentMapping(opts: VectorIndexOptions = {}): Record<string, unknown> {
61  const dimension = opts.dimension ?? DEFAULT_VECTOR_DIMENSION;
62  return {
63    settings: {
64      index: {
65        refresh_interval: "1s", // lag SLA: keep 1 s (do NOT widen for the SLA — search §1.4)
66        number_of_replicas: 1,
67        knn: true,
68        "knn.algo_param.ef_search": 100,
69      },
70    },
71    mappings: {
72      properties: {
73        docId: { type: "keyword" },
74        // TASK-0100 S5: the doc type is a plain keyword. The index CANNOT carry
75        // a mapping-level `enum` — the docker/CI OpenSearch is 2.15, which
76        // rejects the `enum` param on keyword mappers ("unknown parameter
77        // [enum]"), so the content index could never be created (400). The
78        // allowed set is still enforced at the application layer: the client
79        // only indexes known SearchDocType values, and queries only accept
80        // SEARCH_TYPES (both built from TYPE_ENUM). Recorded deviation.
81        type: { type: "keyword" },
82        id: { type: "keyword" },
83        workspaceId: { type: "keyword" }, // tenant filter — ALWAYS present in every query
84        teamId: { type: "keyword" },
85        title: {
86          type: "text",
87          fields: {
88            // prefix + fuzzy target the exact keyword (search §4.2/§4.3).
89            keyword: { type: "keyword", ignore_above: 512 },
90          },
91        },
92        body: { type: "text" },
93        chunkId: { type: "keyword" }, // chunk's own id (TASK-0099); absent on whole-docs
94        chunkIndex: { type: "integer" },
95        parentDocId: { type: "keyword" },
96        vectorModel: { type: "keyword" },
97        vector: {
98          // TASK-0100 S5: the live OpenSearch (2.15) uses the 1.x `knn_vector`
99          // + `method` HNSW form — it has NO `dense_vector` handler (that is
100          // Elasticsearch's type name; `No handler for type [dense_vector]`),
101          // so the index could never be created with the mapping as written.
102          //
103          // TASK-0125 (recorded deviation, verified live against 2.15):
104          //   - engine = **faiss**, not nmslib: the query side MUST pre-filter
105          //     the kNN candidates by the caller's tenant + visibility
106          //     (search §5 permission filter). `nmslib` hard-rejects any kNN
107          //     filter ("Engine [NMSLIB] does not support filters"); only
108          //     `faiss` supports `knn.filter` pre-filtering. With the shared
109          //     multi-tenant index, a non-filtered ANN fetch returns the
110          //     GLOBAL top-K — so post-filtering would both leak cross-tenant
111          //     candidates into the fusion and drop a viewer's own docs (a
112          //     small tenant's content may never reach the global top-50).
113          //     Pre-filtering to the viewer's docs BEFORE the ANN search is
114          //     therefore a security/recall requirement, not an optimization.
115          //   - space_type = **l2** (euclidean): FAISS on 2.15 rejects
116          //     `cosinesimil` and `inner_product` (only `l2` is accepted).
117          //     This is behavior-preserving for the real provider: OpenAI's
118          //     embeddings are L2-normalized, where euclidean distance is
119          //     monotonic in cosine distance (‖a−b‖² = 2−2·cosθ), so l2
120          //     ranking == cosine ranking. The deterministic mock is
121          //     non-normalized, but its kNN leg only needs to rank the
122          //     viewer's own seeded vectors, which l2 does.
123          // The query side uses the `query.knn.<field>` DSL form with an
124          // inline `filter` (see `queries.ts` `knnBlock`) — the ES-style
125          // top-level `knn: { field, query_vector }` body 400s on this
126          // build ("Unknown key for a START_OBJECT in [knn]"), verified live.
127          type: "knn_vector",
128          dimension,
129          method: {
130            name: "hnsw",
131            space_type: "l2",
132            engine: "faiss",
133            parameters: {
134              m: opts.m ?? DEFAULT_KNN_M,
135              ef_construction: opts.efConstruction ?? DEFAULT_KNN_EF_CONSTRUCTION,
136            },
137          },
138        },
139        grants: {
140          properties: {
141            public: { type: "boolean" },
142            workspace: { type: "boolean" },
143            user: { type: "boolean" },
144            teams: { type: "keyword" },
145            users: { type: "keyword" },
146          },
147        },
148        archived: { type: "boolean" },
149        // deep-link / facet metadata (search §3)
150        pageId: { type: "keyword" },
151        blockId: { type: "keyword" },
152        databaseId: { type: "keyword" },
153        recordId: { type: "keyword" },
154        createdBy: { type: "keyword" },
155        createdByName: { type: "keyword" },
156        icon: { type: "keyword", index: false },
157        url: { type: "keyword", index: false },
158        size: { type: "long" },
159        mimeType: { type: "keyword" },
160        createdAt: { type: "date" },
161        updatedAt: { type: "date" },
162      },
163    },
164  };
165}
166 
167/**
168 * True when an existing index's `vector.dimension` already equals `dimension`
169 * (the guard used by the alias-rollover reindex — you cannot change `dims`
170 * in place, so a mismatch means "build the next version").
171 */
172export function vectorDimensionMatches(
173  existing: { mappings?: { properties?: { vector?: { dimension?: number } } } },
174  dimension: number,
175): boolean {
176  return existing.mappings?.properties?.vector?.dimension === dimension;
177}
178