embeddings.ts · Source

embeddings.tspackages/search/src/embeddings.tsraw ← src3,552 B← Back to the app
1/**
2 * @factory/search — embedding provider interface + deterministic stub
3 * (ARCHITECTURE.md §7, search-architecture §7.3).
4 *
5 * The real providers (OpenAI / Google / OSS / custom) live in `@factory/ai`
6 * and are wired into the indexer by the app layer; this package defines the
7 * contract plus a **deterministic stub** for tests and for the
8 * `vector_model = "stub"` dev path. The stub is NOT a real semantic embedding
9 * — it is a bag-of-character-n-gram hash normalized to unit length — but it
10 * is stable, dependency-free (node:crypto) and dimension-exact, which is all
11 * the pipeline needs to be testable end-to-end.
12 *
13 * Recorded decision: the stub dimension (default 64) differs from every real
14 * model (OpenAI 1536/3072, Google 768, OSS discovered). Index mappings must
15 * use the dimension of the model actually in use — the `vector_model`
16 * keyword field is the migration guard (search §7.3), never the mapping.
17 */
18 
19import { createHash } from "node:crypto";
20 
21import type { EmbeddingProvider } from "./types.js";
22 
23/** Dimension of the deterministic stub embedding. */
24export const STUB_EMBEDDING_DIMENSION = 64;
25 
26/** The vector_model tag used for stub embeddings (migration guard). */
27export const STUB_VECTOR_MODEL = "stub-v1";
28 
29/**
30 * Deterministic embedding: hash character trigrams into `dimension` buckets,
31 * L2-normalize. Pure function of (text, dimension) — no clock, no randomness.
32 */
33export function stubEmbed(text: string, dimension: number = STUB_EMBEDDING_DIMENSION): number[] {
34  const vec = new Array<number>(dimension).fill(0);
35  const s = text.toLowerCase().replace(/\s+/g, " ").trim();
36  // char trigrams (padded) + word unigrams for signal
37  for (let i = 0; i < s.length - 2; i++) {
38    const gram = s.slice(i, i + 3);
39    const h = hashBucket(gram, dimension);
40    vec[h]! += 1;
41  }
42  for (const word of s.split(" ")) {
43    if (word.length === 0) continue;
44    const h = hashBucket("w:" + word, dimension);
45    vec[h]! += 2; // words weigh more than grams
46  }
47  const norm = Math.sqrt(vec.reduce((acc, v) => acc + v * v, 0));
48  if (norm === 0) return vec; // empty text → zero vector
49  return vec.map((v) => v / norm);
50}
51 
52function hashBucket(key: string, dimension: number): number {
53  const digest = createHash("sha256").update(key, "utf8").digest();
54  let h = 0;
55  for (let i = 0; i < 8; i++) h = (h * 256 + digest[i]!) >>> 0;
56  return h % dimension;
57}
58 
59/**
60 * A deterministic, dependency-free `EmbeddingProvider` for tests, local dev
61 * and CI. Determinism guarantee: the same text always yields the same vector
62 * (byte-identical), so test fixtures can hard-code expectations.
63 */
64export class StubEmbeddingProvider implements EmbeddingProvider {
65  readonly model: string;
66  readonly dimension: number;
67  constructor(dimension: number = STUB_EMBEDDING_DIMENSION) {
68    this.dimension = dimension;
69    this.model = STUB_VECTOR_MODEL;
70  }
71  async embed(texts: string[]): Promise<number[][]> {
72    return texts.map((t) => stubEmbed(t, this.dimension));
73  }
74}
75 
76/** Cosine similarity in [−1, 1]; zero-safe (returns 0 when either is a zero vector). */
77export function cosineSimilarity(a: readonly number[], b: readonly number[]): number {
78  if (a.length !== b.length) {
79    throw new Error(`dimension mismatch: ${a.length} != ${b.length}`);
80  }
81  let dot = 0;
82  let na = 0;
83  let nb = 0;
84  for (let i = 0; i < a.length; i++) {
85    dot += a[i]! * b[i]!;
86    na += a[i]! * a[i]!;
87    nb += b[i]! * b[i]!;
88  }
89  if (na === 0 || nb === 0) return 0;
90  return dot / (Math.sqrt(na) * Math.sqrt(nb));
91}
92