alerts.yaml · Source

alerts.yamlinfrastructure/observability/prometheus/alerts.yamlraw ← prometheus7,463 B← Back to the app
1# =============================================================================
2# Alert rules (TASK-0135, phase-15 — the six alerts, exact expressions)
3#
4# Loaded by the monitoring-namespace Prometheus (rule_files). Names/labels
5# follow the spec table; EXPOSED metric names are used (classic-text counters
6# carry `_total`: http_request_count → http_request_count_total — the only
7# deviation from the spec's literal expressions, forced by the exporter).
8#
9# Routing: the `severity` label drives alertmanager (page → webhook + email,
10# warn → email-only). Webhook posts to the platform's own incoming-webhook
11# automation (self-dogfood); email via the ops mailbox (see alertmanager.yaml).
12#
13# Expected-but-external metrics (the alerts are defined per spec regardless;
14# they stay silent until the producing infra exists):
15#   - node_filesystem_usage   → node-exporter daemonset (expected-present)
16#   - probe_ssl_earliest_cert_expiry → blackbox exporter probing the ingress
17#   - factory_backup_last_success_timestamp_seconds /
18#     factory_wal_last_ship_timestamp_seconds → the postgres backup stack
19#     (coordinator-owned postgres/ workstream exports them from the wal-g
20#     sidecar/CronJob; see the BackupStale notes below)
21# =============================================================================
22groups:
23  - name: factory.core
24    rules:
25      # 1/6 — API 5xx rate above 1% over 5 m (page). Division by zero (no
26      # traffic) yields an empty result → no alert, which is the desired
27      # behavior for a freshly-restarted / idle service.
28      - alert: ApiErrorRateHigh
29        expr: |
30          sum(rate(http_request_count_total{service_name="factory-api", status=~"5.."}[5m]))
31            / sum(rate(http_request_count_total{service_name="factory-api"}[5m]))
32            > 0.01
33        for: 5m
34        labels:
35          severity: page
36        annotations:
37          summary: "API error rate above 1% (5 m window)"
38          description: "factory-api 5xx ratio is {{ $value | humanizePercentage }} — over 1% for 5 minutes."
39          runbook: "Check the API Overview dashboard (5xx by route_group) + recent deploys."
40 
41      # 2/6 — API p95 above 500 ms over 10 m (page).
42      - alert: ApiP95Slow
43        expr: |
44          histogram_quantile(0.95,
45            sum(rate(http_request_duration_seconds_bucket{service_name="factory-api"}[10m]))
46              by (le, route_group)
47          ) > 0.5
48        for: 10m
49        labels:
50          severity: page
51        annotations:
52          summary: "API p95 latency above 500 ms (10 m window)"
53          description: "factory-api p95 is {{ $value | humanizeDuration }} — over 500 ms for 10 minutes."
54          runbook: "Check API Overview → p95 by route_group; correlate with db_query_duration_seconds + queue depth."
55 
56      # 3/6 — any BullMQ queue above 10k jobs (page).
57      - alert: QueueDepthHigh
58        expr: queue_depth{service_name="factory-worker"} > 10000
59        for: 5m
60        labels:
61          severity: page
62        annotations:
63          summary: "Queue depth above 10k ({{ $labels.queue }})"
64          description: "factory-worker queue {{ $labels.queue }} holds {{ $value | humanize }} jobs — over 10k for 5 minutes."
65          runbook: "Check Workers/Queues dashboard; the chaos drill (stop a worker) is the 5-min validation for this alert."
66 
67      # 4/6 — realtime connections dropped >30% vs the 1 h baseline (page).
68      # (0 < 0.7*0 is false → no alert when the service has zero connections;
69      # a fully-down pod is caught by k8s liveness instead.)
70      - alert: WsConnectionsDrop
71        expr: |
72          ws_active_connections{service_name="factory-realtime"}
73            < 0.7 * avg_over_time(ws_active_connections{service_name="factory-realtime"}[1h])
74        for: 15m
75        labels:
76          severity: page
77        annotations:
78          summary: "WebSocket connections dropped > 30% vs 1 h baseline"
79          description: "factory-realtime has {{ $value | humanize }} live connections; the 1 h average was higher by >30% for 15 minutes."
80          runbook: "Check the Realtime dashboard (connections + drops by cause) + the realtime Deployment rollout."
81 
82      # 5/6 — disk usage above 80% on /var/lib (warn — email-only).
83      # Reads the node-exporter (expected-present; silent until installed).
84      - alert: DiskUsageHigh
85        expr: node_filesystem_usage{mountpoint="/var/lib"} > 0.8
86        for: 10m
87        labels:
88          severity: warn
89        annotations:
90          summary: "Disk usage above 80% on {{ $labels.instance }}"
91          description: "/var/lib on {{ $labels.instance }} is {{ $value | humanizePercentage }} full for 10 minutes."
92          runbook: "Check the Infra dashboard (disk); rotate/clear logs or expand the volume."
93 
94      # 6/6 — ingress TLS cert expiring within 14 days (page).
95      # Reads the blackbox exporter's probe (expected-present: the ingress
96      # cert checker; silent until a blackbox probe job is deployed).
97      - alert: CertExpiringSoon
98        expr: probe_ssl_earliest_cert_expiry - time() < 14 * 86400
99        for: 1h
100        labels:
101          severity: page
102        annotations:
103          summary: "Ingress certificate expires within 14 days ({{ $labels.instance }})"
104          description: "The TLS cert on {{ $labels.instance }} expires in {{ $value | humanizeDuration }}."
105          runbook: "Renew the ingress cert (cert-manager) and re-deploy."
106 
107  # --- operational (warn, email-only — the spec's two extra alerts) ----------
108  - name: factory.operational
109    rules:
110      # BackupStale — the release pipeline's stage-9 conditions as a
111      # CONTINUOUS alert: last base backup < 26 h, last WAL ship < 10 min,
112      # last OpenSearch snapshot < 26 h. The gauges are exported by the
113      # postgres backup stack (wal-g sidecar / CronJob, coordinator-owned
114      # postgres/ workstream) as Unix timestamps; until that exporter is
115      # wired the alert is empty (silent) by design.
116      - alert: BackupStale
117        expr: |
118          time() - factory_backup_last_success_timestamp_seconds{backup_kind="base"} > 26 * 3600
119            or
120          time() - factory_wal_last_ship_timestamp_seconds > 10 * 60
121            or
122          time() - factory_backup_last_success_timestamp_seconds{backup_kind="opensearch"} > 26 * 3600
123        for: 15m
124        labels:
125          severity: warn
126        annotations:
127          summary: "Backup or WAL shipping is stale"
128          description: "A backup/WAL-ship gauge is older than its threshold (base 26 h / WAL 10 min / snapshot 26 h) on {{ $labels.instance }}."
129          runbook: "Check the backup CronJobs (postgres/cronjob-backup.yaml, cronjob-opensearch-snapshot.yaml) + the S3 bucket; run the restore drill if stale > 26 h."
130 
131      # SearchIndexLagHigh — indexer lag above 30 s (the Phase 8 SLO),
132      # from the worker's search_index_lag_seconds gauge (TASK-0135).
133      - alert: SearchIndexLagHigh
134        expr: search_index_lag_seconds{service_name="factory-worker"} > 30
135        for: 15m
136        labels:
137          severity: warn
138        annotations:
139          summary: "Search index consumer lag above 30 s"
140          description: "The search-index consumer is {{ $value | humanizeDuration }} behind the factory:events stream (SLO is 30 s)."
141          runbook: "Check the Search dashboard (index lag + query duration) + the worker's search-index queue depth; scale workers if the lag persists."
142