/** * @factory/observability — the STABLE metric registry (TASK-0135). * * This module is the CONFORMANCE CONTRACT: the exact metric names + label * sets from the phase-15 spec (docs/tasks/phase-15-production.md, the * Observability requirement — the NFR doc's metric set, reconciled per * AGENTS rule 29: the task file's names win over the NFR doc's dotted * names; both are listed in the mapping below). The `metrics` object is the * stable registry — re-calling any accessor returns the SAME instrument * while the global meter provider is unchanged (stable identity, stable * series; a re-created histogram would reset its buckets). The ONE * exception is a provider rebind (see below), which recreates the * instruments ONCE and then keeps them stable again. * * When OTel is disabled (no endpoint) the instruments are the global API's * no-op instruments — record calls are free no-ops (the dev/test contract). * * LAZY CREATION (critical, verified against @opentelemetry/api 1.9.1): the * API's global meter provider starts as a STABLE NO-OP singleton and * `getMeter().createX()` binds the instrument to the provider that is global * AT CREATION TIME — there is no ProxyMeterProvider, so an instrument created * while the global is still the no-op would NEVER record (empirically * verified). The four apps all call `initOtel()` at boot BEFORE any request * is served, so the instruments are created lazily (on first access) against * whichever provider is global then — the SDK provider when enabled, the * no-op when disabled. If a record call happens BEFORE `initOtel` (a dev * script, a test), the instruments are created under the no-op and * RE-CREATED (re-bound) the first time the global provider changes to the * SDK one. The value sources (queue depths, ws count) are stored * independently of the instruments, so nothing is lost across a rebind. * * NAME MAPPING (task spec → NFR doc §7.1): * http_request_duration_seconds / http_request_count ↔ http.server.request.duration (route, method, status) * db_query_duration_seconds ↔ db.query.duration (model, operation) * redis_command_duration_seconds ↔ (redis command timings, added by the task spec) * queue_depth / queue_job_duration_seconds ↔ queue.depth / queue.process.duration * ws_active_connections / ws_messages_total ↔ ws.connections / ws.message.* * search_query_duration_seconds ↔ search.query.duration (type: bm25|knn) * ai_tokens_total / ai_request_duration_seconds ↔ ai.request.duration + token counters * storage_ops_total / storage_op_duration_seconds ↔ (MinIO S3 ops, added by the task spec) * error_total ↔ (per-service error counter, added by the task spec) * * Histogram buckets: every duration histogram uses the same explicit bucket * set (0.005 s … 30 s) so `histogram_quantile` behaves the same across the * catalog and the Grafana dashboards can share one query shape. */ import { metrics as otelMetrics, type Attributes, type Counter, type Histogram, type Meter, type ObservableGauge } from "@opentelemetry/api"; export const DURATION_BUCKETS: readonly number[] = [ 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30 ]; export interface HttpAttrs { /** The ROUTE PATTERN (`/api/v1/pages/:id`), never the concrete id (cardinality contract). */ route: string; method: string; /** Numeric status as a string (`"200"`, `"500"`). */ status: string; } export interface DbAttrs { model: string; operation: string; } export interface RedisAttrs { command: string; } export interface QueueAttrs { queue: string; } export interface SearchAttrs { /** `bm25` | `knn` (the spec's "BM25 vs kNN labeled"). */ type: string; } export interface AiTokenAttrs { /** `in` | `out` (the spec's "in/out labeled per model"). */ direction: string; model: string; } export interface AiRequestAttrs { model: string; } export interface StorageAttrs { op: string; } export interface ErrorAttrs { service: string; code: string; } // --------------------------------------------------------------------------- // Lazy instrument creation (see the module docs — the no-op-binding gotcha). // Instruments are created on first access against whatever meter provider is // global THEN, and recreated if the global provider is later REPLACED (e.g. // initOtel registers the SDK provider after an early access under the no-op). // --------------------------------------------------------------------------- const METER_NAME = "@factory/observability"; // Gauge value sources (process-local). Stored INDEPENDENTLY of the // instruments so a provider rebind (below) never loses a value that was set // before it; the ObservableGauge callbacks read them at collection time. const queueDepths = new Map(); let wsConnections = 0; let searchIndexLagSeconds = 0; let gaugesBound = false; /** The provider the current instruments were created against (null = not yet). */ let boundProvider: object | null = null; function currentProvider(): object { return otelMetrics.getMeterProvider(); } function assertBound(): void { if (boundProvider !== currentProvider()) { // The global provider changed (initOtel booted / the SDK shut down): the // existing instruments are bound to the OLD provider and would never // record again — recreate them (once) against the new one. boundProvider = currentProvider(); instruments = undefined; gaugesBound = false; // the gauges belong to the (new) instruments } } let instruments: | { meter: Meter; httpRequestDuration: Histogram; httpRequestCount: Counter; dbQueryDuration: Histogram; redisCommandDuration: Histogram; queueDepthGauge: ObservableGauge; queueJobDuration: Histogram; wsActiveConnectionsGauge: ObservableGauge; wsMessagesTotal: Counter; searchQueryDuration: Histogram; searchIndexLagGauge: ObservableGauge; aiTokensTotal: Counter; aiRequestDuration: Histogram; storageOpsTotal: Counter; storageOpDuration: Histogram; errorTotal: Counter } | undefined; function inst() { assertBound(); if (!instruments) { const meter = otelMetrics.getMeter(METER_NAME); instruments = { meter, httpRequestDuration: meter.createHistogram("http_request_duration_seconds", { description: "HTTP request duration in seconds.", unit: "s", advice: { explicitBucketBoundaries: [...DURATION_BUCKETS] } }), httpRequestCount: meter.createCounter("http_request_count", { description: "HTTP requests, by route pattern / method / status." }), dbQueryDuration: meter.createHistogram("db_query_duration_seconds", { description: "Prisma query duration in seconds.", unit: "s", advice: { explicitBucketBoundaries: [...DURATION_BUCKETS] } }), redisCommandDuration: meter.createHistogram("redis_command_duration_seconds", { description: "Redis command duration in seconds.", unit: "s", advice: { explicitBucketBoundaries: [...DURATION_BUCKETS] } }), queueDepthGauge: meter.createObservableGauge("queue_depth", { description: "BullMQ queue depth (waiting + delayed), per queue." }), queueJobDuration: meter.createHistogram("queue_job_duration_seconds", { description: "BullMQ job processing duration in seconds, per queue.", unit: "s", advice: { explicitBucketBoundaries: [...DURATION_BUCKETS] } }), wsActiveConnectionsGauge: meter.createObservableGauge("ws_active_connections", { description: "Active WebSocket connections on this realtime pod." }), wsMessagesTotal: meter.createCounter("ws_messages_total", { description: "WebSocket frames processed, total." }), searchQueryDuration: meter.createHistogram("search_query_duration_seconds", { description: "Search query duration in seconds, by query type.", unit: "s", advice: { explicitBucketBoundaries: [...DURATION_BUCKETS] } }), // Operational (beyond the 14-metric spec set) — backs the // `SearchIndexLagHigh` alert (phase-15 alert table, warn/email-only). searchIndexLagGauge: meter.createObservableGauge("search_index_lag_seconds", { description: "Search-index consumer lag: elapsed seconds since the oldest un-consumed stream event." }), aiTokensTotal: meter.createCounter("ai_tokens_total", { description: "AI tokens, by direction (in|out) and model." }), aiRequestDuration: meter.createHistogram("ai_request_duration_seconds", { description: "AI request (completion) duration in seconds, by model.", unit: "s", advice: { explicitBucketBoundaries: [...DURATION_BUCKETS] } }), storageOpsTotal: meter.createCounter("storage_ops_total", { description: "MinIO/S3 object operations, by op (put|get|delete|head|sign)." }), storageOpDuration: meter.createHistogram("storage_op_duration_seconds", { description: "MinIO/S3 object operation duration in seconds, by op.", unit: "s", advice: { explicitBucketBoundaries: [...DURATION_BUCKETS] } }), errorTotal: meter.createCounter("error_total", { description: "Handled errors, by service and error code." }) }; } return instruments; } /** Everything the four apps can record. Stable singleton — see module docs. * (Every property is a LAZY getter: first access creates the instrument * against the then-global provider; see the rebind note above.) */ export const metrics = { get meter(): Meter { return inst().meter; }, get httpRequestDuration(): Histogram { return inst().httpRequestDuration; }, get httpRequestCount(): Counter { return inst().httpRequestCount; }, get dbQueryDuration(): Histogram { return inst().dbQueryDuration; }, get redisCommandDuration(): Histogram { return inst().redisCommandDuration; }, get queueDepthGauge(): ObservableGauge { return inst().queueDepthGauge; }, get queueJobDuration(): Histogram { return inst().queueJobDuration; }, get wsActiveConnectionsGauge(): ObservableGauge { return inst().wsActiveConnectionsGauge; }, get wsMessagesTotal(): Counter { return inst().wsMessagesTotal; }, get searchQueryDuration(): Histogram { return inst().searchQueryDuration; }, get searchIndexLagGauge(): ObservableGauge { return inst().searchIndexLagGauge; }, get aiTokensTotal(): Counter { return inst().aiTokensTotal; }, get aiRequestDuration(): Histogram { return inst().aiRequestDuration; }, get storageOpsTotal(): Counter { return inst().storageOpsTotal; }, get storageOpDuration(): Histogram { return inst().storageOpDuration; }, get errorTotal(): Counter { return inst().errorTotal; } }; // --------------------------------------------------------------------------- // Typed recorders — the app call sites (the spec's `metrics.http.record(...)` // shape). Every method is a no-op when OTel is disabled (the instruments are // the no-op instruments), so apps call them unconditionally. // --------------------------------------------------------------------------- function toAttrs(a: object): Attributes { return { ...(a as Record) }; } export const m = { http: { /** Record one request: the duration histogram + the count counter. */ record(attrs: HttpAttrs, seconds: number): void { const a = toAttrs(attrs); metrics.httpRequestDuration.record(seconds, a); metrics.httpRequestCount.add(1, a); }, count(attrs: HttpAttrs): void { metrics.httpRequestCount.add(1, toAttrs(attrs)); } }, db: { record(attrs: DbAttrs, seconds: number): void { metrics.dbQueryDuration.record(seconds, toAttrs(attrs)); } }, redis: { record(attrs: RedisAttrs, seconds: number): void { metrics.redisCommandDuration.record(seconds, toAttrs(attrs)); } }, queue: { /** * Set the current depth for a queue (the worker calls this on dequeue / * a tick). The gauge callback reads the process-local map at collection * time, so a value set once is exported on every interval. */ setDepth: setQueueDepth, recordJob(attrs: QueueAttrs, seconds: number): void { metrics.queueJobDuration.record(seconds, toAttrs(attrs)); } }, ws: { /** Set the pod's live connection count (realtime connect/disconnect). */ setConnections(count: number): void { wsConnections = count; }, message(): void { metrics.wsMessagesTotal.add(1); } }, search: { record(attrs: SearchAttrs, seconds: number): void { metrics.searchQueryDuration.record(seconds, toAttrs(attrs)); }, /** * Set the search-index consumer lag in SECONDS (the worker probes the * `factory:events` stream group and reports how old the tail is). No * label — one consumer per cluster. Backs SearchIndexLagHigh. */ lag(seconds: number): void { setSearchIndexLag(seconds); } }, ai: { tokens(attrs: AiTokenAttrs, count: number): void { metrics.aiTokensTotal.add(count, toAttrs(attrs)); }, request(attrs: AiRequestAttrs, seconds: number): void { metrics.aiRequestDuration.record(seconds, toAttrs(attrs)); } }, storage: { op(attrs: StorageAttrs, seconds: number): void { metrics.storageOpsTotal.add(1, toAttrs(attrs)); metrics.storageOpDuration.record(seconds, toAttrs(attrs)); } }, error: { record(attrs: ErrorAttrs): void { metrics.errorTotal.add(1, toAttrs(attrs)); } } }; // --------------------------------------------------------------------------- // Gauge binding (the value sources live above, next to the instruments, so a // provider rebind can reset `gaugesBound` — the callbacks are re-attached to // the new instruments; a value set BEFORE the first binding is still // exported, and re-observing is never needed). // --------------------------------------------------------------------------- export function bindGauges(): void { if (gaugesBound) return; gaugesBound = true; metrics.queueDepthGauge.addCallback((result) => { for (const [queue, depth] of queueDepths) { result.observe(depth, { queue }); } }); metrics.wsActiveConnectionsGauge.addCallback((result) => { result.observe(wsConnections, {}); }); metrics.searchIndexLagGauge.addCallback((result) => { result.observe(searchIndexLagSeconds, {}); }); } /** Record a queue depth (auto-binds the gauge callback). */ export function setQueueDepth(queue: string, depth: number): void { queueDepths.set(queue, depth); bindGauges(); } /** Record the pod's live ws connection count (auto-binds the gauge callback). */ export function setWsConnections(count: number): void { wsConnections = count; bindGauges(); } /** Record the search-index consumer lag in seconds (auto-binds the gauge). */ export function setSearchIndexLag(seconds: number): void { searchIndexLagSeconds = seconds; bindGauges(); } // --------------------------------------------------------------------------- // Conformance surface — the unit test asserts these names against the spec. // --------------------------------------------------------------------------- /** * OPERATIONAL metric names BEYOND the spec's 14 (the conformance test above * pins METRIC_NAMES to the spec exactly — these are additive and documented * in infrastructure/observability/README.md): * - `search_index_lag_seconds` — search-index consumer lag; backs the * operational `SearchIndexLagHigh` alert (phase-15 alert table, warn + * email-only). */ export const OPERATIONAL_METRIC_NAMES = ["search_index_lag_seconds"] as const; /** Every metric NAME this package registers (the spec's exact list). */ export const METRIC_NAMES = [ "http_request_duration_seconds", "http_request_count", "db_query_duration_seconds", "redis_command_duration_seconds", "queue_depth", "queue_job_duration_seconds", "ws_active_connections", "ws_messages_total", "search_query_duration_seconds", "ai_tokens_total", "ai_request_duration_seconds", "storage_ops_total", "storage_op_duration_seconds", "error_total" ] as const; export type MetricName = (typeof METRIC_NAMES)[number]; /** * The registered instruments by name (for the conformance test + any * advanced consumer). Built lazily from the live `metrics` object so the * keys are guaranteed to match what actually got created. */ export function registeredInstruments(): Record { return { http_request_duration_seconds: { instrument: metrics.httpRequestDuration, kind: "histogram" }, http_request_count: { instrument: metrics.httpRequestCount, kind: "counter" }, db_query_duration_seconds: { instrument: metrics.dbQueryDuration, kind: "histogram" }, redis_command_duration_seconds: { instrument: metrics.redisCommandDuration, kind: "histogram" }, queue_depth: { instrument: metrics.queueDepthGauge, kind: "gauge" }, queue_job_duration_seconds: { instrument: metrics.queueJobDuration, kind: "histogram" }, ws_active_connections: { instrument: metrics.wsActiveConnectionsGauge, kind: "gauge" }, ws_messages_total: { instrument: metrics.wsMessagesTotal, kind: "counter" }, search_query_duration_seconds: { instrument: metrics.searchQueryDuration, kind: "histogram" }, ai_tokens_total: { instrument: metrics.aiTokensTotal, kind: "counter" }, ai_request_duration_seconds: { instrument: metrics.aiRequestDuration, kind: "histogram" }, storage_ops_total: { instrument: metrics.storageOpsTotal, kind: "counter" }, storage_op_duration_seconds: { instrument: metrics.storageOpDuration, kind: "histogram" }, error_total: { instrument: metrics.errorTotal, kind: "counter" } }; }