metrics.ts · Source

metrics.tspackages/observability/src/metrics.tsraw ← src18,228 B← Back to the app
1/**
2 * @factory/observability — the STABLE metric registry (TASK-0135).
3 *
4 * This module is the CONFORMANCE CONTRACT: the exact metric names + label
5 * sets from the phase-15 spec (docs/tasks/phase-15-production.md, the
6 * Observability requirement — the NFR doc's metric set, reconciled per
7 * AGENTS rule 29: the task file's names win over the NFR doc's dotted
8 * names; both are listed in the mapping below). The `metrics` object is the
9 * stable registry — re-calling any accessor returns the SAME instrument
10 * while the global meter provider is unchanged (stable identity, stable
11 * series; a re-created histogram would reset its buckets). The ONE
12 * exception is a provider rebind (see below), which recreates the
13 * instruments ONCE and then keeps them stable again.
14 *
15 * When OTel is disabled (no endpoint) the instruments are the global API's
16 * no-op instruments — record calls are free no-ops (the dev/test contract).
17 *
18 * LAZY CREATION (critical, verified against @opentelemetry/api 1.9.1): the
19 * API's global meter provider starts as a STABLE NO-OP singleton and
20 * `getMeter().createX()` binds the instrument to the provider that is global
21 * AT CREATION TIME — there is no ProxyMeterProvider, so an instrument created
22 * while the global is still the no-op would NEVER record (empirically
23 * verified). The four apps all call `initOtel()` at boot BEFORE any request
24 * is served, so the instruments are created lazily (on first access) against
25 * whichever provider is global then — the SDK provider when enabled, the
26 * no-op when disabled. If a record call happens BEFORE `initOtel` (a dev
27 * script, a test), the instruments are created under the no-op and
28 * RE-CREATED (re-bound) the first time the global provider changes to the
29 * SDK one. The value sources (queue depths, ws count) are stored
30 * independently of the instruments, so nothing is lost across a rebind.
31 *
32 * NAME MAPPING (task spec → NFR doc §7.1):
33 *   http_request_duration_seconds / http_request_count ↔ http.server.request.duration (route, method, status)
34 *   db_query_duration_seconds                           ↔ db.query.duration (model, operation)
35 *   redis_command_duration_seconds                      ↔ (redis command timings, added by the task spec)
36 *   queue_depth / queue_job_duration_seconds            ↔ queue.depth / queue.process.duration
37 *   ws_active_connections / ws_messages_total           ↔ ws.connections / ws.message.*
38 *   search_query_duration_seconds                       ↔ search.query.duration (type: bm25|knn)
39 *   ai_tokens_total / ai_request_duration_seconds       ↔ ai.request.duration + token counters
40 *   storage_ops_total / storage_op_duration_seconds     ↔ (MinIO S3 ops, added by the task spec)
41 *   error_total                                         ↔ (per-service error counter, added by the task spec)
42 *
43 * Histogram buckets: every duration histogram uses the same explicit bucket
44 * set (0.005 s … 30 s) so `histogram_quantile` behaves the same across the
45 * catalog and the Grafana dashboards can share one query shape.
46 */
47import {
48  metrics as otelMetrics,
49  type Attributes,
50  type Counter,
51  type Histogram,
52  type Meter,
53  type ObservableGauge
54} from "@opentelemetry/api";
55 
56export const DURATION_BUCKETS: readonly number[] = [
57  0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30
58];
59 
60export interface HttpAttrs {
61  /** The ROUTE PATTERN (`/api/v1/pages/:id`), never the concrete id (cardinality contract). */
62  route: string;
63  method: string;
64  /** Numeric status as a string (`"200"`, `"500"`). */
65  status: string;
66}
67export interface DbAttrs {
68  model: string;
69  operation: string;
70}
71export interface RedisAttrs {
72  command: string;
73}
74export interface QueueAttrs {
75  queue: string;
76}
77export interface SearchAttrs {
78  /** `bm25` | `knn` (the spec's "BM25 vs kNN labeled"). */
79  type: string;
80}
81export interface AiTokenAttrs {
82  /** `in` | `out` (the spec's "in/out labeled per model"). */
83  direction: string;
84  model: string;
85}
86export interface AiRequestAttrs {
87  model: string;
88}
89export interface StorageAttrs {
90  op: string;
91}
92export interface ErrorAttrs {
93  service: string;
94  code: string;
95}
96 
97// ---------------------------------------------------------------------------
98// Lazy instrument creation (see the module docs — the no-op-binding gotcha).
99// Instruments are created on first access against whatever meter provider is
100// global THEN, and recreated if the global provider is later REPLACED (e.g.
101// initOtel registers the SDK provider after an early access under the no-op).
102// ---------------------------------------------------------------------------
103 
104const METER_NAME = "@factory/observability";
105 
106// Gauge value sources (process-local). Stored INDEPENDENTLY of the
107// instruments so a provider rebind (below) never loses a value that was set
108// before it; the ObservableGauge callbacks read them at collection time.
109const queueDepths = new Map<string, number>();
110let wsConnections = 0;
111let searchIndexLagSeconds = 0;
112let gaugesBound = false;
113 
114/** The provider the current instruments were created against (null = not yet). */
115let boundProvider: object | null = null;
116 
117function currentProvider(): object {
118  return otelMetrics.getMeterProvider();
119}
120 
121function assertBound(): void {
122  if (boundProvider !== currentProvider()) {
123    // The global provider changed (initOtel booted / the SDK shut down): the
124    // existing instruments are bound to the OLD provider and would never
125    // record again — recreate them (once) against the new one.
126    boundProvider = currentProvider();
127    instruments = undefined;
128    gaugesBound = false; // the gauges belong to the (new) instruments
129  }
130}
131 
132let instruments:
133  | {
134      meter: Meter;
135      httpRequestDuration: Histogram;
136      httpRequestCount: Counter;
137      dbQueryDuration: Histogram;
138      redisCommandDuration: Histogram;
139      queueDepthGauge: ObservableGauge;
140      queueJobDuration: Histogram;
141      wsActiveConnectionsGauge: ObservableGauge;
142      wsMessagesTotal: Counter;
143      searchQueryDuration: Histogram;
144      searchIndexLagGauge: ObservableGauge;
145      aiTokensTotal: Counter;
146      aiRequestDuration: Histogram;
147      storageOpsTotal: Counter;
148      storageOpDuration: Histogram;
149      errorTotal: Counter
150    }
151  | undefined;
152 
153function inst() {
154  assertBound();
155  if (!instruments) {
156    const meter = otelMetrics.getMeter(METER_NAME);
157    instruments = {
158      meter,
159      httpRequestDuration: meter.createHistogram("http_request_duration_seconds", {
160        description: "HTTP request duration in seconds.",
161        unit: "s",
162        advice: { explicitBucketBoundaries: [...DURATION_BUCKETS] }
163      }),
164      httpRequestCount: meter.createCounter("http_request_count", {
165        description: "HTTP requests, by route pattern / method / status."
166      }),
167      dbQueryDuration: meter.createHistogram("db_query_duration_seconds", {
168        description: "Prisma query duration in seconds.",
169        unit: "s",
170        advice: { explicitBucketBoundaries: [...DURATION_BUCKETS] }
171      }),
172      redisCommandDuration: meter.createHistogram("redis_command_duration_seconds", {
173        description: "Redis command duration in seconds.",
174        unit: "s",
175        advice: { explicitBucketBoundaries: [...DURATION_BUCKETS] }
176      }),
177      queueDepthGauge: meter.createObservableGauge("queue_depth", {
178        description: "BullMQ queue depth (waiting + delayed), per queue."
179      }),
180      queueJobDuration: meter.createHistogram("queue_job_duration_seconds", {
181        description: "BullMQ job processing duration in seconds, per queue.",
182        unit: "s",
183        advice: { explicitBucketBoundaries: [...DURATION_BUCKETS] }
184      }),
185      wsActiveConnectionsGauge: meter.createObservableGauge("ws_active_connections", {
186        description: "Active WebSocket connections on this realtime pod."
187      }),
188      wsMessagesTotal: meter.createCounter("ws_messages_total", {
189        description: "WebSocket frames processed, total."
190      }),
191      searchQueryDuration: meter.createHistogram("search_query_duration_seconds", {
192        description: "Search query duration in seconds, by query type.",
193        unit: "s",
194        advice: { explicitBucketBoundaries: [...DURATION_BUCKETS] }
195      }),
196      // Operational (beyond the 14-metric spec set) — backs the
197      // `SearchIndexLagHigh` alert (phase-15 alert table, warn/email-only).
198      searchIndexLagGauge: meter.createObservableGauge("search_index_lag_seconds", {
199        description:
200          "Search-index consumer lag: elapsed seconds since the oldest un-consumed stream event."
201      }),
202      aiTokensTotal: meter.createCounter("ai_tokens_total", {
203        description: "AI tokens, by direction (in|out) and model."
204      }),
205      aiRequestDuration: meter.createHistogram("ai_request_duration_seconds", {
206        description: "AI request (completion) duration in seconds, by model.",
207        unit: "s",
208        advice: { explicitBucketBoundaries: [...DURATION_BUCKETS] }
209      }),
210      storageOpsTotal: meter.createCounter("storage_ops_total", {
211        description: "MinIO/S3 object operations, by op (put|get|delete|head|sign)."
212      }),
213      storageOpDuration: meter.createHistogram("storage_op_duration_seconds", {
214        description: "MinIO/S3 object operation duration in seconds, by op.",
215        unit: "s",
216        advice: { explicitBucketBoundaries: [...DURATION_BUCKETS] }
217      }),
218      errorTotal: meter.createCounter("error_total", {
219        description: "Handled errors, by service and error code."
220      })
221    };
222  }
223  return instruments;
224}
225 
226/** Everything the four apps can record. Stable singleton — see module docs.
227 *  (Every property is a LAZY getter: first access creates the instrument
228 *  against the then-global provider; see the rebind note above.) */
229export const metrics = {
230  get meter(): Meter {
231    return inst().meter;
232  },
233  get httpRequestDuration(): Histogram {
234    return inst().httpRequestDuration;
235  },
236  get httpRequestCount(): Counter {
237    return inst().httpRequestCount;
238  },
239  get dbQueryDuration(): Histogram {
240    return inst().dbQueryDuration;
241  },
242  get redisCommandDuration(): Histogram {
243    return inst().redisCommandDuration;
244  },
245  get queueDepthGauge(): ObservableGauge {
246    return inst().queueDepthGauge;
247  },
248  get queueJobDuration(): Histogram {
249    return inst().queueJobDuration;
250  },
251  get wsActiveConnectionsGauge(): ObservableGauge {
252    return inst().wsActiveConnectionsGauge;
253  },
254  get wsMessagesTotal(): Counter {
255    return inst().wsMessagesTotal;
256  },
257  get searchQueryDuration(): Histogram {
258    return inst().searchQueryDuration;
259  },
260  get searchIndexLagGauge(): ObservableGauge {
261    return inst().searchIndexLagGauge;
262  },
263  get aiTokensTotal(): Counter {
264    return inst().aiTokensTotal;
265  },
266  get aiRequestDuration(): Histogram {
267    return inst().aiRequestDuration;
268  },
269  get storageOpsTotal(): Counter {
270    return inst().storageOpsTotal;
271  },
272  get storageOpDuration(): Histogram {
273    return inst().storageOpDuration;
274  },
275  get errorTotal(): Counter {
276    return inst().errorTotal;
277  }
278};
279 
280// ---------------------------------------------------------------------------
281// Typed recorders — the app call sites (the spec's `metrics.http.record(...)`
282// shape). Every method is a no-op when OTel is disabled (the instruments are
283// the no-op instruments), so apps call them unconditionally.
284// ---------------------------------------------------------------------------
285 
286function toAttrs(a: object): Attributes {
287  return { ...(a as Record<string, string>) };
288}
289 
290export const m = {
291  http: {
292    /** Record one request: the duration histogram + the count counter. */
293    record(attrs: HttpAttrs, seconds: number): void {
294      const a = toAttrs(attrs);
295      metrics.httpRequestDuration.record(seconds, a);
296      metrics.httpRequestCount.add(1, a);
297    },
298    count(attrs: HttpAttrs): void {
299      metrics.httpRequestCount.add(1, toAttrs(attrs));
300    }
301  },
302  db: {
303    record(attrs: DbAttrs, seconds: number): void {
304      metrics.dbQueryDuration.record(seconds, toAttrs(attrs));
305    }
306  },
307  redis: {
308    record(attrs: RedisAttrs, seconds: number): void {
309      metrics.redisCommandDuration.record(seconds, toAttrs(attrs));
310    }
311  },
312  queue: {
313    /**
314     * Set the current depth for a queue (the worker calls this on dequeue /
315     * a tick). The gauge callback reads the process-local map at collection
316     * time, so a value set once is exported on every interval.
317     */
318    setDepth: setQueueDepth,
319    recordJob(attrs: QueueAttrs, seconds: number): void {
320      metrics.queueJobDuration.record(seconds, toAttrs(attrs));
321    }
322  },
323  ws: {
324    /** Set the pod's live connection count (realtime connect/disconnect). */
325    setConnections(count: number): void {
326      wsConnections = count;
327    },
328    message(): void {
329      metrics.wsMessagesTotal.add(1);
330    }
331  },
332  search: {
333    record(attrs: SearchAttrs, seconds: number): void {
334      metrics.searchQueryDuration.record(seconds, toAttrs(attrs));
335    },
336    /**
337     * Set the search-index consumer lag in SECONDS (the worker probes the
338     * `factory:events` stream group and reports how old the tail is). No
339     * label — one consumer per cluster. Backs SearchIndexLagHigh.
340     */
341    lag(seconds: number): void {
342      setSearchIndexLag(seconds);
343    }
344  },
345  ai: {
346    tokens(attrs: AiTokenAttrs, count: number): void {
347      metrics.aiTokensTotal.add(count, toAttrs(attrs));
348    },
349    request(attrs: AiRequestAttrs, seconds: number): void {
350      metrics.aiRequestDuration.record(seconds, toAttrs(attrs));
351    }
352  },
353  storage: {
354    op(attrs: StorageAttrs, seconds: number): void {
355      metrics.storageOpsTotal.add(1, toAttrs(attrs));
356      metrics.storageOpDuration.record(seconds, toAttrs(attrs));
357    }
358  },
359  error: {
360    record(attrs: ErrorAttrs): void {
361      metrics.errorTotal.add(1, toAttrs(attrs));
362    }
363  }
364};
365 
366// ---------------------------------------------------------------------------
367// Gauge binding (the value sources live above, next to the instruments, so a
368// provider rebind can reset `gaugesBound` — the callbacks are re-attached to
369// the new instruments; a value set BEFORE the first binding is still
370// exported, and re-observing is never needed).
371// ---------------------------------------------------------------------------
372 
373export function bindGauges(): void {
374  if (gaugesBound) return;
375  gaugesBound = true;
376  metrics.queueDepthGauge.addCallback((result) => {
377    for (const [queue, depth] of queueDepths) {
378      result.observe(depth, { queue });
379    }
380  });
381  metrics.wsActiveConnectionsGauge.addCallback((result) => {
382    result.observe(wsConnections, {});
383  });
384  metrics.searchIndexLagGauge.addCallback((result) => {
385    result.observe(searchIndexLagSeconds, {});
386  });
387}
388 
389/** Record a queue depth (auto-binds the gauge callback). */
390export function setQueueDepth(queue: string, depth: number): void {
391  queueDepths.set(queue, depth);
392  bindGauges();
393}
394 
395/** Record the pod's live ws connection count (auto-binds the gauge callback). */
396export function setWsConnections(count: number): void {
397  wsConnections = count;
398  bindGauges();
399}
400 
401/** Record the search-index consumer lag in seconds (auto-binds the gauge). */
402export function setSearchIndexLag(seconds: number): void {
403  searchIndexLagSeconds = seconds;
404  bindGauges();
405}
406 
407// ---------------------------------------------------------------------------
408// Conformance surface — the unit test asserts these names against the spec.
409// ---------------------------------------------------------------------------
410 
411/**
412 * OPERATIONAL metric names BEYOND the spec's 14 (the conformance test above
413 * pins METRIC_NAMES to the spec exactly — these are additive and documented
414 * in infrastructure/observability/README.md):
415 *  - `search_index_lag_seconds` — search-index consumer lag; backs the
416 *    operational `SearchIndexLagHigh` alert (phase-15 alert table, warn +
417 *    email-only).
418 */
419export const OPERATIONAL_METRIC_NAMES = ["search_index_lag_seconds"] as const;
420 
421/** Every metric NAME this package registers (the spec's exact list). */
422export const METRIC_NAMES = [
423  "http_request_duration_seconds",
424  "http_request_count",
425  "db_query_duration_seconds",
426  "redis_command_duration_seconds",
427  "queue_depth",
428  "queue_job_duration_seconds",
429  "ws_active_connections",
430  "ws_messages_total",
431  "search_query_duration_seconds",
432  "ai_tokens_total",
433  "ai_request_duration_seconds",
434  "storage_ops_total",
435  "storage_op_duration_seconds",
436  "error_total"
437] as const;
438 
439export type MetricName = (typeof METRIC_NAMES)[number];
440 
441/**
442 * The registered instruments by name (for the conformance test + any
443 * advanced consumer). Built lazily from the live `metrics` object so the
444 * keys are guaranteed to match what actually got created.
445 */
446export function registeredInstruments(): Record<string, { instrument: unknown; kind: "counter" | "histogram" | "gauge" }> {
447  return {
448    http_request_duration_seconds: { instrument: metrics.httpRequestDuration, kind: "histogram" },
449    http_request_count: { instrument: metrics.httpRequestCount, kind: "counter" },
450    db_query_duration_seconds: { instrument: metrics.dbQueryDuration, kind: "histogram" },
451    redis_command_duration_seconds: { instrument: metrics.redisCommandDuration, kind: "histogram" },
452    queue_depth: { instrument: metrics.queueDepthGauge, kind: "gauge" },
453    queue_job_duration_seconds: { instrument: metrics.queueJobDuration, kind: "histogram" },
454    ws_active_connections: { instrument: metrics.wsActiveConnectionsGauge, kind: "gauge" },
455    ws_messages_total: { instrument: metrics.wsMessagesTotal, kind: "counter" },
456    search_query_duration_seconds: { instrument: metrics.searchQueryDuration, kind: "histogram" },
457    ai_tokens_total: { instrument: metrics.aiTokensTotal, kind: "counter" },
458    ai_request_duration_seconds: { instrument: metrics.aiRequestDuration, kind: "histogram" },
459    storage_ops_total: { instrument: metrics.storageOpsTotal, kind: "counter" },
460    storage_op_duration_seconds: { instrument: metrics.storageOpDuration, kind: "histogram" },
461    error_total: { instrument: metrics.errorTotal, kind: "counter" }
462  };
463}
464