1/** 2 * @factory/observability — OTel SDK init (TASK-0135). 3 * 4 * `initOtel()` is the single entry the four apps call at boot. Contract: 5 * 6 * - DISABLED BY DEFAULT: when `OTEL_EXPORTER_OTLP_ENDPOINT` is unset (or 7 * empty) it is a NO-OP — dev/tests run with zero collector dependency 8 * (the NFR-H5 self-host posture: telemetry is opt-in, no data leaves the 9 * deployment). When disabled, `isOtelEnabled()` is false and every 10 * app-side helper degrades to its no-op path. 11 * - IDEMPOTENT: a second call (Nest re-init under vitest, HMR) returns the 12 * first state; it never double-registers the global providers. 13 * - Traces: BatchSpanProcessor (wrapped by the span-scrubber) → OTLP/HTTP; 14 * sampling = ParentBased over a 10% TraceIdRatio root sampler (NFR §7.1 15 * "100% errors, 10% others" — the 10% head-sample is the v1 shape; the 16 * collector-side error-always tail sampling is out of scope, recorded 17 * decision). 18 * - Metrics: PeriodicExportingMetricReader (30 s) → OTLP/HTTP; the SDK's 19 * MeterProvider is registered GLOBALLY so the singleton `metrics` 20 * registry (metrics.ts) exports to it. 21 * - Propagation: the package's W3C trace-context propagator (traceparent.ts) 22 * is registered so `captureTraceparent()`/`resumeTraceparent()` behave 23 * identically before AND after the SDK boots (the API→worker job-metadata 24 * primitive). 25 * - Context: AsyncLocalStorageContextManager (required for the 26 * API→worker traceparent primitive to work across the async queue 27 * boundary + for auto-instrumented spans to parent correctly). 28 * - Instruments: `HttpInstrumentation` (outgoing calls to OpenSearch/MinIO 29 * auto-continue the trace — the spec's "API → OpenSearch/MinIO (HTTP 30 * propagation)") + the redis instrumentations (spans for ioredis / 31 * node-redis commands). Per-Redis-command DURATIONS are recorded 32 * explicitly from the apps via the `metrics` registry regardless — 33 * deterministic, no require-order dependence (recorded decision). 34 * - Logs: v1 keeps pino → stdout (NFR §7.1: "pino → stdout (JSON)" is 35 * primary; Loki scrape is the staging convenience path). The `logs` 36 * option is accepted for spec-completeness and NOT wired (recorded gap). 37 * 38 * The SDK is imported lazily (only when enabled) so a disabled process 39 * never pays the sdk import cost and never touches the global API. 40 * 41 * OTel JS 2.x surface (T-190b — the versions pinned in package.json: SDK 42 * core 2.12.0 + exporters 0.223.0 + `@prisma/instrumentation` 6.19.3, which 43 * pairs with the 2.x train and clears the two audit exceptions carried under 44 * T-190): `NodeTracerProvider` takes its span processors via the 45 * CONSTRUCTOR `spanProcessors` option (the 1.x `addSpanProcessor()` pair is 46 * gone), resources are built with `resourceFromAttributes(...)` (the 47 * `Resource` class and `Resource.default().merge(...)` are gone), 48 * `SpanStatusCode` still comes from the API, and the OTLP/HTTP exporters 49 * still take a `url` option. 50 * 51 * SUPERSEDED (1.x pairing, recorded 2026-10): the SDK previously pinned the 52 * 1.30.x line because `@prisma/instrumentation@5.22.0` (Prisma 5.22) built 53 * its engine spans with a 1.x `new Span(...)` that crash-looped against a 54 * 2.x global tracer (`parentTracer.getSpanLimits is not a function`). That 55 * constraint dissolved when Prisma moved to 6.x — `@prisma/instrumentation` 56 * 6.x is built against the 2.x SDK, so the whole train moves to 57 * 2.12.0/0.223.0 (T-190b). Per AGENTS rule 29 the superseded decision is 58 * kept in git history and the FEATURE_TRACKER T-190b entry, not in this 59 * file's live rationale. 60 */ 61import { 62 context as otelContext, 63 metrics as metricsApi, 64 propagation, 65 SpanStatusCode, 66 trace, 67 type Context, 68 type Span 69} from "@opentelemetry/api"; 70import type { MeterProvider as SdkMeterProvider } from "@opentelemetry/sdk-metrics"; 71import { BasicTracerProvider, type ReadableSpan, type Span as SdkSpan, type SpanProcessor } from "@opentelemetry/sdk-trace-base"; 72import { buildResource, type ServiceName } from "./resource.js"; 73import { registerTraceparentPropagator, traceparentPropagator } from "./traceparent.js"; 74import { spanScrubber } from "./scrubber.js"; 75import { m } from "./metrics.js"; 76import { setPrometheusReader } from "./prometheus.js"; 77 78export interface InitOtelOptions { 79 /** One of the four factory service names (the spec's list). */ 80 serviceName: ServiceName; 81 /** OTLP/HTTP endpoint (e.g. `http://otel-collector:4318`). Unset/empty → disabled. */ 82 endpoint?: string; 83 /** See module docs: accepted, not wired in v1 (pino → stdout is primary). */ 84 logs?: boolean; 85 /** Root-span sample ratio for NEW traces (0..1). Default 0.1 (NFR §7.1). */ 86 sampleRatio?: number; 87 /** Metric export interval (ms). Default 30_000. */ 88 metricExportIntervalMs?: number; 89 /** `deployment.environment` attribute (defaults to NODE_ENV). */ 90 environment?: string; 91} 92 93export interface OtelState { 94 /** False when the endpoint was unset — every consumer must no-op. */ 95 enabled: boolean; 96 serviceName: ServiceName; 97 tracerProvider?: BasicTracerProvider; 98 meterProvider?: SdkMeterProvider; 99} 100 101let state: OtelState | undefined; 102 103/** Whether telemetry is live (the endpoint was set at init time). */ 104export function isOtelEnabled(): boolean { 105 return state?.enabled === true; 106} 107 108/** The current init state (undefined = never initialized → disabled). */ 109export function getOtelState(): OtelState | undefined { 110 return state; 111} 112 113/** 114 * SpanProcessor wrapper that scrubs PII + secrets off every span's 115 * attributes at `onEnd` — BEFORE the BatchSpanProcessor queues it for 116 * export. This is the central scrubbing point: it covers spans this package 117 * creates AND the auto-instrumented ones (HTTP, ioredis, Prisma — the 118 * Prisma span carries `db.*` attributes the scrubber knows to redact). 119 * 120 * `onStart` receives the SDK `Span`, `onEnd` receives the `ReadableSpan`; 121 * both are handled via the shared inner processor. 122 */ 123class ScrubbingSpanProcessor implements SpanProcessor { 124 constructor(private readonly inner: SpanProcessor) {} 125 126 forceFlush(): Promise<void> { 127 return this.inner.forceFlush(); 128 } 129 130 shutdown(): Promise<void> { 131 return this.inner.shutdown(); 132 } 133 134 onStart(span: SdkSpan, parentContext: Context): void { 135 this.inner.onStart(span, parentContext); 136 } 137 138 onEnd(span: ReadableSpan): void { 139 if (span.attributes) spanScrubber(span.attributes as unknown as Record<string, unknown>); 140 this.inner.onEnd(span); 141 } 142} 143 144/** 145 * Central db/redis duration collector (TASK-0135). Records, on `onEnd`, the 146 * two spec histograms that would otherwise need an edit at EVERY Prisma / 147 * ioredis call site: 148 * 149 * - Prisma operation spans (from `@prisma/instrumentation`, active only when 150 * the schema has `previewFeatures = ["tracing"]`): span name 151 * `prisma:client:operation`, attributes `model` + `method` (= the Prisma 152 * action) → `db_query_duration_seconds{model, operation}`. 153 * - ioredis command spans: span name = the command, attribute 154 * `db.system.name = "redis"` → `redis_command_duration_seconds{command}`. 155 * 156 * Duration comes from `span.duration` (HrTime `[sec, nanos]`) — no 157 * manual start/end math. The instruments are the singleton registry's 158 * (metrics.ts) — no-op when OTel is disabled, so a span ending can never 159 * throw here. Order in the chain: AFTER the scrubber (attributes are already 160 * redacted; we only READ model/operation/command, never the query text). 161 */ 162export class DbRedisMetricsProcessor implements SpanProcessor { 163 forceFlush(): Promise<void> { 164 return Promise.resolve(); 165 } 166 167 shutdown(): Promise<void> { 168 return Promise.resolve(); 169 } 170 171 onStart(): void { 172 /* metrics are recorded at end only */ 173 } 174 175 onEnd(span: ReadableSpan): void { 176 const seconds = hrTimeToSeconds(span.duration); 177 const attrs = (span.attributes ?? {}) as Record<string, unknown>; 178 if (span.name === "prisma:client:operation") { 179 const model = typeof attrs["model"] === "string" ? (attrs["model"] as string) : "unknown"; 180 const operation = typeof attrs["method"] === "string" ? (attrs["method"] as string) : "unknown"; 181 m.db.record({ model, operation }, seconds); 182 return; 183 } 184 if (attrs["db.system.name"] === "redis") { 185 const command = typeof attrs["db.operation.name"] === "string" ? (attrs["db.operation.name"] as string) : span.name; 186 m.redis.record({ command }, seconds); 187 } 188 } 189} 190 191/** HrTime `[seconds, nanos]` → fractional seconds (`ReadableSpan.duration`, same shape in 1.x). */ 192function hrTimeToSeconds(hrTime: readonly [number, number]): number { 193 return hrTime[0] + hrTime[1] / 1_000_000_000; 194} 195 196/** 197 * Initialize OTel for one app (module docs for the contract). Returns the 198 * shared state; disabled when no endpoint is configured. 199 */ 200export async function initOtel(opts: InitOtelOptions): Promise<OtelState> { 201 if (state) return state; // idempotent — first init wins 202 // The propagator registers even when disabled: capture/resume must behave 203 // (as no-ops) identically in dev and prod. 204 registerTraceparentPropagator(); 205 206 const endpoint = (opts.endpoint ?? process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "").trim(); 207 if (!endpoint) { 208 state = { enabled: false, serviceName: opts.serviceName }; 209 return state; 210 } 211 212 // Lazy import: only an ENABLED process loads the SDK. 213 const [ 214 sdkTraceBase, 215 { NodeTracerProvider }, 216 { OTLPTraceExporter }, 217 sdkMetrics, 218 { OTLPMetricExporter }, 219 { PrometheusExporter }, 220 { defaultResource }, 221 { AsyncLocalStorageContextManager }, 222 httpInst, 223 ioredisInst, 224 redisInst, 225 prismaInst 226 ] = await Promise.all([ 227 import("@opentelemetry/sdk-trace-base"), 228 import("@opentelemetry/sdk-trace-node"), 229 import("@opentelemetry/exporter-trace-otlp-http"), 230 import("@opentelemetry/sdk-metrics"), 231 import("@opentelemetry/exporter-metrics-otlp-http"), 232 import("@opentelemetry/exporter-prometheus"), 233 import("@opentelemetry/resources"), 234 import("@opentelemetry/context-async-hooks"), 235 import("@opentelemetry/instrumentation-http"), 236 import("@opentelemetry/instrumentation-ioredis"), 237 import("@opentelemetry/instrumentation-redis"), 238 import("@prisma/instrumentation") 239 ]); 240 241 const { BatchSpanProcessor, ParentBasedSampler, TraceIdRatioBasedSampler } = sdkTraceBase; 242 const { MeterProvider, PeriodicExportingMetricReader } = sdkMetrics; 243 244 // 2.x: resources are plain objects via resourceFromAttributes(); the 245 // defaultResource() factory replaces Resource.default(). Merge our stable 246 // attributes onto the SDK default (telemetry.sdk.* + detected platform 247 // attrs). 248 const resource = defaultResource().merge( 249 buildResource(opts.serviceName, opts.environment !== undefined ? { environment: opts.environment } : {}) 250 ); 251 const sampler = new ParentBasedSampler({ 252 root: new TraceIdRatioBasedSampler(opts.sampleRatio ?? 0.1) 253 }); 254 255 const contextManager = new AsyncLocalStorageContextManager(); 256 // 2.x: span processors attach via the CONSTRUCTOR `spanProcessors` option 257 // (the 1.x addSpanProcessor() pair is gone). Order preserved: (1) the 258 // central db/redis metric recorder, (2) the scrubber wrapping the batched 259 // OTLP exporter (scrub before export). 260 const tracerProvider = new NodeTracerProvider({ 261 resource, 262 sampler, 263 spanProcessors: [ 264 new DbRedisMetricsProcessor(), 265 new ScrubbingSpanProcessor(new BatchSpanProcessor(new OTLPTraceExporter({ url: endpoint + "/v1/traces" }))) 266 ] 267 }); 268 contextManager.enable(); 269 // Global registration (guarded above — one init per process). The 270 // context manager + propagator are set explicitly (register() would also 271 // set them, but the constructor already owns the processor chain). 272 trace.setGlobalTracerProvider(tracerProvider); 273 otelContext.setGlobalContextManager(contextManager); 274 propagation.setGlobalPropagator(traceparentPropagator); 275 276 // The Prometheus reader runs SERVER-LESS (preventServerStart) — each app 277 // exposes it on its OWN port's /metrics via renderPrometheus() (the scrape 278 // config targets the app ports, not a second port). The serializer is 279 // constructed directly in renderPrometheus() anyway (no exporter-level 280 // withoutScopeInfo/withoutTargetInfo flags used here). 281 const promReader = new PrometheusExporter({ preventServerStart: true }); 282 setPrometheusReader(promReader); 283 284 const meterProvider: SdkMeterProvider = new MeterProvider({ 285 resource, 286 readers: [ 287 new PeriodicExportingMetricReader({ 288 exporter: new OTLPMetricExporter({ url: endpoint + "/v1/metrics" }), 289 exportIntervalMillis: opts.metricExportIntervalMs ?? 30_000 290 }), 291 promReader 292 ] 293 }); 294 metricsApi.setGlobalMeterProvider(meterProvider); 295 296 // Outgoing HTTP (OpenSearch/MinIO/AI-provider calls continue the trace). 297 new httpInst.HttpInstrumentation().enable(); 298 // Redis spans (ioredis is the client of record; node-redis kept for the 299 // redis-common shared path). Failures are non-fatal — the apps record 300 // redis durations explicitly via the `metrics` registry either way. 301 try { 302 new ioredisInst.IORedisInstrumentation().enable(); 303 } catch { 304 /* client not loaded yet — durations are still recorded explicitly */ 305 } 306 try { 307 new redisInst.RedisInstrumentation().enable(); 308 } catch { 309 /* node-redis is not an app dependency — no-op */ 310 } 311 // Prisma: enable() sets globalThis.PRISMA_INSTRUMENTATION = { helper }; the 312 // generated client reads that global LAZILY at query time (verified in 313 // @prisma/client 6.x), so this works even though every app's PrismaClient 314 // is constructed AFTER initOtel. In 6.x the tracing helper is always built 315 // (the `tracing` preview feature was promoted to GA in 6.1) — Spans → 316 // DbRedisMetricsProcessor (db_query_duration_seconds) + ScrubbingSpanProcessor 317 // (redacted export). 318 try { 319 new prismaInst.PrismaInstrumentation().enable(); 320 } catch { 321 /* never fatal */ 322 } 323 324 state = { enabled: true, serviceName: opts.serviceName, tracerProvider, meterProvider }; 325 return state; 326} 327 328/** Gracefully shut the SDK down (flush pending spans + metrics). */ 329export async function shutdownOtel(): Promise<void> { 330 if (!state?.enabled) return; 331 try { 332 await state.tracerProvider?.shutdown(); 333 await state.meterProvider?.shutdown(); 334 } finally { 335 state = undefined; 336 } 337} 338 339function scrubAttrs(attributes: Record<string, unknown>): Record<string, string | number | boolean> { 340 return spanScrubber({ ...attributes }) as Record<string, string | number | boolean>; 341} 342 343/** 344 * Start a span with scrubbed attributes, run `fn` UNDER it (the active span 345 * is set on the ambient context so anything `fn` calls — including 346 * auto-instrumented spans — parents correctly), and end the span when `fn` 347 * settles. Errors mark the span ERROR + rethrow. When OTel is disabled the 348 * global tracer is the no-op tracer and `context.with` runs ambient — the 349 * no-op contract. 350 * 351 * Attributes are scrubbed at CREATION (this helper) AND at `onEnd` (the 352 * ScrubbingSpanProcessor) — defense in depth against attributes added by 353 * auto-instrumentation mid-span. 354 */ 355export function startSpan<T>(name: string, attributes: Record<string, unknown>, fn: (span: SpanHandle) => T): T { 356 const span = trace.getTracer("factory").startSpan(name, { attributes: scrubAttrs(attributes) }); 357 const handle = makeSpanHandle(span); 358 try { 359 return otelContext.with(trace.setSpan(otelContext.active(), span), () => fn(handle)); 360 } catch (err) { 361 markError(span, err); 362 throw err; 363 } finally { 364 span.end(); 365 } 366} 367 368/** 369 * Async variant of {@link startSpan} for `async` callbacks (the worker 370 * processor path) — the span is ENDED when the promise settles, which the 371 * sync helper cannot know. 372 */ 373export async function startAsyncSpan<T>( 374 name: string, 375 attributes: Record<string, unknown>, 376 fn: (span: SpanHandle) => Promise<T> 377): Promise<T> { 378 const span = trace.getTracer("factory").startSpan(name, { attributes: scrubAttrs(attributes) }); 379 const handle = makeSpanHandle(span); 380 try { 381 return await otelContext.with(trace.setSpan(otelContext.active(), span), () => fn(handle)); 382 } catch (err) { 383 markError(span, err); 384 throw err; 385 } finally { 386 span.end(); 387 } 388} 389 390function markError(span: Span, err: unknown): void { 391 try { 392 span.recordException(err instanceof Error ? err : new Error(String(err))); 393 span.setStatus({ code: SpanStatusCode.ERROR, message: err instanceof Error ? err.message : String(err) }); 394 } catch { 395 /* the no-op span never throws — keep the rethrow clean */ 396 } 397} 398 399function makeSpanHandle(span: Span): SpanHandle { 400 return { 401 attributes: {} as Record<string, unknown>, 402 setAttribute: (k, v) => { 403 if (v === undefined) return; 404 span.setAttribute(k, v); 405 }, 406 setStatus: (status, message) => 407 span.setStatus({ 408 code: status === "ok" ? SpanStatusCode.OK : SpanStatusCode.ERROR, 409 ...(message !== undefined ? { message } : {}) 410 }) 411 }; 412} 413 414/** 415 * The handle passed to a `startSpan` callback: an attribute setter (values 416 * are written straight onto the live span; the onEnd scrub re-runs over the 417 * final map) + a status setter. (The raw `Span` is intentionally NOT 418 * exposed — call sites that need `span.end()` timing should use 419 * `startAsyncSpan`.) 420 */ 421export interface SpanHandle { 422 /** 423 * The span's attribute map as of creation (already scrubbed). The live 424 * span is the source of truth for anything set via `setAttribute`. 425 */ 426 attributes: Record<string, unknown>; 427 setAttribute(key: string, value: string | number | boolean): void; 428 setStatus(status: "ok" | "error", message?: string): void; 429} 430