# =============================================================================
# Alert rules (TASK-0135, phase-15 — the six alerts, exact expressions)
#
# Loaded by the monitoring-namespace Prometheus (rule_files). Names/labels
# follow the spec table; EXPOSED metric names are used (classic-text counters
# carry `_total`: http_request_count → http_request_count_total — the only
# deviation from the spec's literal expressions, forced by the exporter).
#
# Routing: the `severity` label drives alertmanager (page → webhook + email,
# warn → email-only). Webhook posts to the platform's own incoming-webhook
# automation (self-dogfood); email via the ops mailbox (see alertmanager.yaml).
#
# Expected-but-external metrics (the alerts are defined per spec regardless;
# they stay silent until the producing infra exists):
#   - node_filesystem_usage   → node-exporter daemonset (expected-present)
#   - probe_ssl_earliest_cert_expiry → blackbox exporter probing the ingress
#   - factory_backup_last_success_timestamp_seconds /
#     factory_wal_last_ship_timestamp_seconds → the postgres backup stack
#     (coordinator-owned postgres/ workstream exports them from the wal-g
#     sidecar/CronJob; see the BackupStale notes below)
# =============================================================================
groups:
  - name: factory.core
    rules:
      # 1/6 — API 5xx rate above 1% over 5 m (page). Division by zero (no
      # traffic) yields an empty result → no alert, which is the desired
      # behavior for a freshly-restarted / idle service.
      - alert: ApiErrorRateHigh
        expr: |
          sum(rate(http_request_count_total{service_name="factory-api", status=~"5.."}[5m]))
            / sum(rate(http_request_count_total{service_name="factory-api"}[5m]))
            > 0.01
        for: 5m
        labels:
          severity: page
        annotations:
          summary: "API error rate above 1% (5 m window)"
          description: "factory-api 5xx ratio is {{ $value | humanizePercentage }} — over 1% for 5 minutes."
          runbook: "Check the API Overview dashboard (5xx by route_group) + recent deploys."

      # 2/6 — API p95 above 500 ms over 10 m (page).
      - alert: ApiP95Slow
        expr: |
          histogram_quantile(0.95,
            sum(rate(http_request_duration_seconds_bucket{service_name="factory-api"}[10m]))
              by (le, route_group)
          ) > 0.5
        for: 10m
        labels:
          severity: page
        annotations:
          summary: "API p95 latency above 500 ms (10 m window)"
          description: "factory-api p95 is {{ $value | humanizeDuration }} — over 500 ms for 10 minutes."
          runbook: "Check API Overview → p95 by route_group; correlate with db_query_duration_seconds + queue depth."

      # 3/6 — any BullMQ queue above 10k jobs (page).
      - alert: QueueDepthHigh
        expr: queue_depth{service_name="factory-worker"} > 10000
        for: 5m
        labels:
          severity: page
        annotations:
          summary: "Queue depth above 10k ({{ $labels.queue }})"
          description: "factory-worker queue {{ $labels.queue }} holds {{ $value | humanize }} jobs — over 10k for 5 minutes."
          runbook: "Check Workers/Queues dashboard; the chaos drill (stop a worker) is the 5-min validation for this alert."

      # 4/6 — realtime connections dropped >30% vs the 1 h baseline (page).
      # (0 < 0.7*0 is false → no alert when the service has zero connections;
      # a fully-down pod is caught by k8s liveness instead.)
      - alert: WsConnectionsDrop
        expr: |
          ws_active_connections{service_name="factory-realtime"}
            < 0.7 * avg_over_time(ws_active_connections{service_name="factory-realtime"}[1h])
        for: 15m
        labels:
          severity: page
        annotations:
          summary: "WebSocket connections dropped > 30% vs 1 h baseline"
          description: "factory-realtime has {{ $value | humanize }} live connections; the 1 h average was higher by >30% for 15 minutes."
          runbook: "Check the Realtime dashboard (connections + drops by cause) + the realtime Deployment rollout."

      # 5/6 — disk usage above 80% on /var/lib (warn — email-only).
      # Reads the node-exporter (expected-present; silent until installed).
      - alert: DiskUsageHigh
        expr: node_filesystem_usage{mountpoint="/var/lib"} > 0.8
        for: 10m
        labels:
          severity: warn
        annotations:
          summary: "Disk usage above 80% on {{ $labels.instance }}"
          description: "/var/lib on {{ $labels.instance }} is {{ $value | humanizePercentage }} full for 10 minutes."
          runbook: "Check the Infra dashboard (disk); rotate/clear logs or expand the volume."

      # 6/6 — ingress TLS cert expiring within 14 days (page).
      # Reads the blackbox exporter's probe (expected-present: the ingress
      # cert checker; silent until a blackbox probe job is deployed).
      - alert: CertExpiringSoon
        expr: probe_ssl_earliest_cert_expiry - time() < 14 * 86400
        for: 1h
        labels:
          severity: page
        annotations:
          summary: "Ingress certificate expires within 14 days ({{ $labels.instance }})"
          description: "The TLS cert on {{ $labels.instance }} expires in {{ $value | humanizeDuration }}."
          runbook: "Renew the ingress cert (cert-manager) and re-deploy."

  # --- operational (warn, email-only — the spec's two extra alerts) ----------
  - name: factory.operational
    rules:
      # BackupStale — the release pipeline's stage-9 conditions as a
      # CONTINUOUS alert: last base backup < 26 h, last WAL ship < 10 min,
      # last OpenSearch snapshot < 26 h. The gauges are exported by the
      # postgres backup stack (wal-g sidecar / CronJob, coordinator-owned
      # postgres/ workstream) as Unix timestamps; until that exporter is
      # wired the alert is empty (silent) by design.
      - alert: BackupStale
        expr: |
          time() - factory_backup_last_success_timestamp_seconds{backup_kind="base"} > 26 * 3600
            or
          time() - factory_wal_last_ship_timestamp_seconds > 10 * 60
            or
          time() - factory_backup_last_success_timestamp_seconds{backup_kind="opensearch"} > 26 * 3600
        for: 15m
        labels:
          severity: warn
        annotations:
          summary: "Backup or WAL shipping is stale"
          description: "A backup/WAL-ship gauge is older than its threshold (base 26 h / WAL 10 min / snapshot 26 h) on {{ $labels.instance }}."
          runbook: "Check the backup CronJobs (postgres/cronjob-backup.yaml, cronjob-opensearch-snapshot.yaml) + the S3 bucket; run the restore drill if stale > 26 h."

      # SearchIndexLagHigh — indexer lag above 30 s (the Phase 8 SLO),
      # from the worker's search_index_lag_seconds gauge (TASK-0135).
      - alert: SearchIndexLagHigh
        expr: search_index_lag_seconds{service_name="factory-worker"} > 30
        for: 15m
        labels:
          severity: warn
        annotations:
          summary: "Search index consumer lag above 30 s"
          description: "The search-index consumer is {{ $value | humanizeDuration }} behind the factory:events stream (SLO is 30 s)."
          runbook: "Check the Search dashboard (index lag + query duration) + the worker's search-index queue depth; scale workers if the lag persists."