dashboard-completeness.mjs

dashboard-completeness.mjsinfrastructure/observability/scripts/dashboard-completeness.mjsraw ← scripts8,559 B← Back to the app
1#!/usr/bin/env node
2/**
3 * Dashboard-completeness check (TASK-0135, phase-15).
4 *
5 * The spec's dashboard requirement: "every emitted metric must appear in at
6 * least one panel, OR be explicitly listed as unpaneled in README.md with a
7 * reason." This script makes that check mechanical (and re-runnable in CI):
8 *
9 *   1. parse every panel target's `expr` in the six dashboards and extract
10 *      the metric families it actually READS — a token only counts if it
11 *      sits in a metric position (immediately before `{` or `[`, or a bare
12 *      metric reference);
13 *   2. assert every family the panels read is in the emitted-metric catalog
14 *      (catches typos in dashboards);
15 *   3. assert every catalog family (the 15 OTel instruments incl. the
16 *      operational search_index_lag_seconds, the 10 realtime-local series,
17 *      target_info, and the 2 expected-external families) is read by some
18 *      panel OR listed in the README's "Unpaneled / gap" section.
19 *
20 * Run from anywhere (paths are resolved relative to this file):
21 *   node infrastructure/observability/scripts/dashboard-completeness.mjs
22 * Exit 0 = complete, 1 = gap or typo.
23 */
24import { readFileSync, readdirSync } from "node:fs";
25import { dirname, join } from "node:path";
26import { fileURLToPath } from "node:url";
27 
28const here = dirname(fileURLToPath(import.meta.url));
29const dashboardsDir = join(here, "..", "grafana", "dashboards");
30const readmePath = join(here, "..", "README.md");
31 
32/** The emitted-metric catalog (by source). EXPOSED names — classic text
33 *  format, so the http counter is `http_request_count_total`. */
34const CATALOG = [
35  // packages/observability instruments (15 — 14 spec + 1 operational)
36  "http_request_duration_seconds",
37  "http_request_count_total",
38  "db_query_duration_seconds",
39  "redis_command_duration_seconds",
40  "queue_depth",
41  "queue_job_duration_seconds",
42  "ws_active_connections",
43  "ws_messages_total",
44  "search_query_duration_seconds",
45  "ai_tokens_total",
46  "ai_request_duration_seconds",
47  "storage_ops_total",
48  "storage_op_duration_seconds",
49  "error_total",
50  "search_index_lag_seconds",
51  // realtime app-local registry (10 — merged into the same exposition)
52  "uptime_seconds",
53  "connected_clients",
54  "events_processed_total",
55  "ws_handshake_failures_total",
56  "ws_connections",
57  "ws_drops_total",
58  "ws_room_peers",
59  "ws_update_log_size",
60  "ws_rooms",
61  // OTel exporter always emits this on every target
62  "target_info",
63  // expected-present external exporters (the alerts read them; the Infra
64  // panels read the node ones directly)
65  "node_filesystem_usage",
66  "node_cpu_seconds_total",
67  "node_memory_MemAvailable_bytes",
68  "node_memory_MemTotal_bytes",
69  "probe_ssl_earliest_cert_expiry"
70];
71 
72/** Recording-rule names the dashboards may legitimately read (derived, not
73 *  raw emitted families — allowed as panel sources). */
74const RECORDING_RULES = new Set([
75  "factory_api:http_request_duration_seconds:p50",
76  "factory_api:http_request_duration_seconds:p95",
77  "factory_api:http_request_duration_seconds:p99",
78  "factory_api:http_requests:rate5m",
79  "factory_api:http_request_error_rate:5m",
80  "factory_worker:queue_depth",
81  "factory_worker:queue_jobs:rate5m",
82  "factory_worker:queue_job_duration_seconds:p95",
83  "factory_worker:search_index_lag_seconds",
84  "factory_realtime:ws_connections",
85  "factory_realtime:ws_messages:rate5m",
86  "factory_api:search_query_duration_seconds:p95",
87  "factory_api:ai_tokens:rate5m",
88  "factory_api:ai_request_duration_seconds:p95",
89  "factory_api:db_query_duration_seconds:p95"
90]);
91 
92const catalogSet = new Set(CATALOG);
93 
94function collectExprs() {
95  const files = readdirSync(dashboardsDir).filter((f) => f.endsWith(".json")).sort();
96  const out = [];
97  for (const f of files) {
98    const d = JSON.parse(readFileSync(join(dashboardsDir, f), "utf8"));
99    for (const p of d.panels ?? []) {
100      for (const t of p.targets ?? []) {
101        if (typeof t.expr === "string" && t.expr.trim()) out.push({ file: f, panel: p.title ?? `panel ${p.id}`, expr: t.expr });
102      }
103    }
104  }
105  return out;
106}
107 
108/** Histogram derived-series suffixes (map back to the base family). */
109const HIST_SUFFIXES = ["_bucket", "_sum", "_count", "_total"];
110 
111/** Extract the metric references an expr touches. A token counts as a
112 *  metric family iff it is immediately followed by `{` (labels) or `[`
113 *  (range vector), OR is a bare metric (at an expression start / after an
114 *  operator, not a keyword). `by (label)` group-by clauses are stripped
115 *  first so their label names are never mistaken for metrics; histogram
116 *  derived series (foo_bucket/foo_sum/foo_count/foo_total) map to `foo`. */
117function familiesInExpr(expr) {
118  const found = new Set();
119  // strip quoted label values (e.g. mountpoint="/var/lib") and group-by
120  // clauses (their contents are label names, not metrics)
121  const stripped = expr.replace(/"[^"]*"/g, '""').replace(/\bby\s*\(([^)]*)\)/g, " by () ");
122  for (const m of stripped.matchAll(/([a-zA-Z_][a-zA-Z0-9_:]*)(\s*\{|\s*\[)/g)) {
123    found.add(resolveFamily(m[1]));
124  }
125  for (const m of stripped.matchAll(/(?:^|[+\-*/()\s]\s*)([a-zA-Z_][a-zA-Z0-9_]*)\s*(?=[+\-*/<>!]=(?!=)|[)\s]|$)/g)) {
126    const tok = m[1];
127    if (isBareMetricTok(tok)) found.add(resolveFamily(tok));
128  }
129  return found;
130}
131 
132/** Map a series name onto its catalog family: strip histogram suffixes, and
133 *  for recording rules (namespace:family:record) use the middle segment. */
134function resolveFamily(tok) {
135  let base = tok;
136  if (tok.includes(":") && tok.startsWith("factory")) {
137    const parts = tok.split(":");
138    if (parts.length >= 3) base = parts.slice(1, -1).join(":");
139  }
140  if (catalogSet.has(base)) return base;
141  for (const s of HIST_SUFFIXES) {
142    if (base.endsWith(s) && catalogSet.has(base.slice(0, -s.length))) return base.slice(0, -s.length);
143  }
144  return tok;
145}
146 
147const BARE_NON_METRIC = new Set([
148  "by", "on", "ignoring", "group_left", "group_right", "bool", "or", "and",
149  "unless", "offset", "sum", "avg", "min", "max", "count", "rate", "irate",
150  "increase", "histogram_quantile", "avg_over_time", "max_over_time",
151  "min_over_time", "sum_over_time", "last_over_time", "label_replace",
152  "label_join", "time", "label_values", "topk", "bottomk", "sort", "absent",
153  "changes", "resets", "deriv", "idelta", "clamp", "clamp_max", "clamp_min",
154  "vector", "scalar", "timestamp", "predict_linear", "le", "quantile", "if",
155  "with", "true", "false", "null", "instance", "job", "mode", "idle"
156]);
157function isBareMetricTok(tok) {
158  return !BARE_NON_METRIC.has(tok) && /^[a-z][a-zA-Z0-9_]*$/.test(tok);
159}
160 
161function main() {
162  const exprs = collectExprs();
163  if (!exprs.length) {
164    console.error("FAIL: no dashboard exprs found");
165    process.exit(1);
166  }
167  const readme = readFileSync(readmePath, "utf8");
168  const gapSection = readme.includes("## Unpaneled / gap")
169    ? readme.split("## Unpaneled / gap")[1].split(/\n## /)[0]
170    : "";
171 
172  const paneld = new Set();
173  const violations = [];
174 
175  for (const e of exprs) {
176    for (const fam of familiesInExpr(e.expr)) {
177      if (catalogSet.has(fam)) {
178        paneld.add(fam);
179      } else if (RECORDING_RULES.has(fam) || fam.includes(":")) {
180        // derived series — fine
181      } else if (isBareMetricTok(fam) && !/[A-Z]/.test(fam)) {
182        // a lowercase bare token that is NOT a catalog family, NOT a
183        // recording rule, NOT a known keyword → probable typo
184        violations.push(`TYPO? ${e.file} panel "${e.panel}" reads unknown family "${fam}" — expr: ${e.expr.trim().slice(0, 80)}`);
185      }
186    }
187  }
188 
189  const unpaneld = CATALOG.filter((f) => !paneld.has(f));
190  for (const fam of unpaneld) {
191    if (!gapSection.includes(fam)) {
192      violations.push(`UNPANELLED: catalog family "${fam}" is read by no panel and is NOT listed in README "Unpaneled / gap"`);
193    }
194  }
195 
196  console.log(`dashboards : ${readdirSync(dashboardsDir).filter((f) => f.endsWith(".json")).length}`);
197  console.log(`exprs      : ${exprs.length}`);
198  console.log(`catalog    : ${CATALOG.length} families (15 OTel + 10 realtime-local + target_info + 5 external)`);
199  console.log(`paneld     : ${CATALOG.filter((f) => paneld.has(f)).length}/${CATALOG.length}`);
200  if (unpaneld.length) console.log(`unpaneld   : ${unpaneld.join(", ")}`);
201 
202  if (violations.length) {
203    console.error("\nDASHBOARD-COMPLETENESS VIOLATIONS:");
204    for (const v of violations) console.error("  - " + v);
205    process.exit(1);
206  }
207  console.log("\nOK: every emitted metric is paneld or listed unpaneled in README.md");
208  process.exit(0);
209}
210 
211main();
212