/** * @factory/search — the OpenSearch content-index mapping (search-architecture * §2–§3). * * Phase-8 reconciliation (recorded decision, see docs/architecture/search.md * § "Layout decision"): ONE index (alias `factory-content`) holds every typed * doc; tenant isolation is the `workspaceId` filter applied at query time (the * Phase-0 per-workspace `factory-search-` indices are retired). The field * names are **camelCase**, matching the stored {@link SearchDoc} in `types.ts` * exactly — this is the single source of truth the worker indexer and the query * API both emit/consume. * * The `vector` field uses OpenSearch 2.x `dense_vector` with an HNSW index. * The arch doc shows the older 1.x `knn_vector`/`knn: true` form; the docker + * CI OpenSearch is 2.x, where `dense_vector` is the current type (recorded * deviation). The dimension is a parameter: `dims` is immutable per index, so * a model change rolls a new index forward via the alias (TASK-0099's * `scripts/opensearch-reindex.ts`), never by re-mapping in place. */ import type { SearchDocType, SearchType } from "./types.js"; /** HNSW construction params (search §3; tunable via SEARCH_KNN_* at S5). */ export interface VectorIndexOptions { m?: number; efConstruction?: number; dimension?: number; } /** The default vector dimension (matches the TASK-0099 `text-embedding-3-small` target). */ export const DEFAULT_VECTOR_DIMENSION = 1536; export const DEFAULT_KNN_M = 16; export const DEFAULT_KNN_EF_CONSTRUCTION = 100; /** * All indexed `type` values (so a bad doc `type` is rejected by the * mapping's enum). TASK-0099 adds `"chunk"`: chunk docs live in the same * index and the enum must accept them — but `"chunk"` is deliberately NOT in * the public {@link SearchType} / `SEARCH_TYPES` (types.ts), so the API's * `types` filter can never request chunk docs (keyword search stays * whole-doc — recorded reconciliation, arch doc §0). */ export const TYPE_ENUM: SearchDocType[] = [ "page", "block", "database", "record", "comment", "file", "user", "chunk", ]; /** * Build the `factory-content` index mapping. * * @param opts vector (dense_vector / HNSW) tuning. Omit to use the defaults * (1536-dim cosine HNSW, m=16, ef_construction=100). */ export function contentMapping(opts: VectorIndexOptions = {}): Record { const dimension = opts.dimension ?? DEFAULT_VECTOR_DIMENSION; return { settings: { index: { refresh_interval: "1s", // lag SLA: keep 1 s (do NOT widen for the SLA — search §1.4) number_of_replicas: 1, knn: true, "knn.algo_param.ef_search": 100, }, }, mappings: { properties: { docId: { type: "keyword" }, // TASK-0100 S5: the doc type is a plain keyword. The index CANNOT carry // a mapping-level `enum` — the docker/CI OpenSearch is 2.15, which // rejects the `enum` param on keyword mappers ("unknown parameter // [enum]"), so the content index could never be created (400). The // allowed set is still enforced at the application layer: the client // only indexes known SearchDocType values, and queries only accept // SEARCH_TYPES (both built from TYPE_ENUM). Recorded deviation. type: { type: "keyword" }, id: { type: "keyword" }, workspaceId: { type: "keyword" }, // tenant filter — ALWAYS present in every query teamId: { type: "keyword" }, title: { type: "text", fields: { // prefix + fuzzy target the exact keyword (search §4.2/§4.3). keyword: { type: "keyword", ignore_above: 512 }, }, }, body: { type: "text" }, chunkId: { type: "keyword" }, // chunk's own id (TASK-0099); absent on whole-docs chunkIndex: { type: "integer" }, parentDocId: { type: "keyword" }, vectorModel: { type: "keyword" }, vector: { // TASK-0100 S5: the live OpenSearch (2.15) uses the 1.x `knn_vector` // + `method` HNSW form — it has NO `dense_vector` handler (that is // Elasticsearch's type name; `No handler for type [dense_vector]`), // so the index could never be created with the mapping as written. // // TASK-0125 (recorded deviation, verified live against 2.15): // - engine = **faiss**, not nmslib: the query side MUST pre-filter // the kNN candidates by the caller's tenant + visibility // (search §5 permission filter). `nmslib` hard-rejects any kNN // filter ("Engine [NMSLIB] does not support filters"); only // `faiss` supports `knn.filter` pre-filtering. With the shared // multi-tenant index, a non-filtered ANN fetch returns the // GLOBAL top-K — so post-filtering would both leak cross-tenant // candidates into the fusion and drop a viewer's own docs (a // small tenant's content may never reach the global top-50). // Pre-filtering to the viewer's docs BEFORE the ANN search is // therefore a security/recall requirement, not an optimization. // - space_type = **l2** (euclidean): FAISS on 2.15 rejects // `cosinesimil` and `inner_product` (only `l2` is accepted). // This is behavior-preserving for the real provider: OpenAI's // embeddings are L2-normalized, where euclidean distance is // monotonic in cosine distance (‖a−b‖² = 2−2·cosθ), so l2 // ranking == cosine ranking. The deterministic mock is // non-normalized, but its kNN leg only needs to rank the // viewer's own seeded vectors, which l2 does. // The query side uses the `query.knn.` DSL form with an // inline `filter` (see `queries.ts` `knnBlock`) — the ES-style // top-level `knn: { field, query_vector }` body 400s on this // build ("Unknown key for a START_OBJECT in [knn]"), verified live. type: "knn_vector", dimension, method: { name: "hnsw", space_type: "l2", engine: "faiss", parameters: { m: opts.m ?? DEFAULT_KNN_M, ef_construction: opts.efConstruction ?? DEFAULT_KNN_EF_CONSTRUCTION, }, }, }, grants: { properties: { public: { type: "boolean" }, workspace: { type: "boolean" }, user: { type: "boolean" }, teams: { type: "keyword" }, users: { type: "keyword" }, }, }, archived: { type: "boolean" }, // deep-link / facet metadata (search §3) pageId: { type: "keyword" }, blockId: { type: "keyword" }, databaseId: { type: "keyword" }, recordId: { type: "keyword" }, createdBy: { type: "keyword" }, createdByName: { type: "keyword" }, icon: { type: "keyword", index: false }, url: { type: "keyword", index: false }, size: { type: "long" }, mimeType: { type: "keyword" }, createdAt: { type: "date" }, updatedAt: { type: "date" }, }, }, }; } /** * True when an existing index's `vector.dimension` already equals `dimension` * (the guard used by the alias-rollover reindex — you cannot change `dims` * in place, so a mismatch means "build the next version"). */ export function vectorDimensionMatches( existing: { mappings?: { properties?: { vector?: { dimension?: number } } } }, dimension: number, ): boolean { return existing.mappings?.properties?.vector?.dimension === dimension; }