1# ============================================================================= 2# Prometheus main config (prometheus.yml) — TASK-0135, phase-15 §Observability 3# 4# Source of truth. The monitoring-namespace ConfigMap 5# (infrastructure/kubernetes/observability/kustomization.yaml, 6# configMapGenerator key `prometheus.yml`) is generated FROM this file — do 7# not hand-edit the rendered ConfigMap. 8# 9# Targets — every app serves /metrics (the worker's is its ONLY port, 4003): 10# factory-api api.factory.svc.cluster.local:4000/metrics 11# factory-worker worker pods:4003/metrics (pod SD — the worker has no 12# ClusterIP Service; its health server binds the pod IP) 13# factory-realtime realtime.factory.svc.cluster.local:4001/metrics 14# (the app's local ws_* registry + OTel text, merged) 15# factory-web web.factory.svc.cluster.local:3000/api/metrics 16# (Next.js route — the path differs from the others) 17# node-exporter expected-present daemonset (pod SD by container name; 18# the DiskUsageHigh alert reads it — no data until a 19# node exporter is installed in the cluster) 20# 21# Label strategy: each job sets a `service_name` static label, so EVERY 22# series from that target carries service_name="factory-<app>" (including the 23# app-local realtime ws_* series, which carry no label of their own). The 24# `target_info{service_name=...}` gauge (always emitted by the OTel exporter) 25# is the belt-and-braces proof of identity. 26# 27# Naming contract (classic text format, exporter-prometheus 0.57.0): 28# counters expose with a `_total` suffix that is NOT in the instrument name — 29# http_request_count → http_request_count_total; ws_messages_total / 30# storage_ops_total / ai_tokens_total / error_total already end in _total and 31# are unchanged. Recording rules + alerts use the EXPOSED names. 32# ============================================================================= 33global: 34 scrape_interval: 15s 35 evaluation_interval: 15s 36 37rule_files: 38 # sibling files, mounted into /etc/prometheus/rules/ by the Deployment 39 - /etc/prometheus/rules/recording-rules.yaml 40 - /etc/prometheus/rules/alerts.yaml 41 42alerting: 43 alertmanagers: 44 - static_configs: 45 - targets: ["alertmanager.monitoring.svc.cluster.local:9093"] 46 47scrape_configs: 48 # --- the four factory apps -------------------------------------------------- 49 - job_name: factory-api 50 metrics_path: /metrics 51 static_configs: 52 - targets: ["api.factory.svc.cluster.local:4000"] 53 labels: 54 service_name: factory-api 55 56 # The worker exposes /metrics only on its pod IP (no ClusterIP Service in 57 # worker.yaml) → pod SD, keep the worker container, pin port 4003. 58 - job_name: factory-worker 59 metrics_path: /metrics 60 kubernetes_sd_configs: 61 - role: pod 62 relabel_configs: 63 - source_labels: [__meta_kubernetes_namespace] 64 regex: "factory" 65 action: keep 66 - source_labels: [__meta_kubernetes_pod_container_name] 67 regex: "worker" 68 action: keep 69 - source_labels: [__meta_kubernetes_pod_ip] 70 target_label: __address__ 71 replacement: "$1:4003" 72 - target_label: service_name 73 replacement: factory-worker 74 75 - job_name: factory-realtime 76 metrics_path: /metrics 77 static_configs: 78 - targets: ["realtime.factory.svc.cluster.local:4001"] 79 labels: 80 service_name: factory-realtime 81 82 # web serves the exposition at /api/metrics (Next.js app route). 83 - job_name: factory-web 84 metrics_path: /api/metrics 85 static_configs: 86 - targets: ["web.factory.svc.cluster.local:3000"] 87 labels: 88 service_name: factory-web 89 90 # --- node exporter (expected-present; see header) --------------------------- 91 - job_name: node-exporter 92 kubernetes_sd_configs: 93 - role: pod 94 relabel_configs: 95 - source_labels: [__meta_kubernetes_pod_container_name] 96 regex: "node-exporter" 97 action: keep 98 - source_labels: [__meta_kubernetes_pod_node_name] 99 target_label: node 100 - target_label: service_name 101 replacement: node-exporter 102 103 # --- self ------------------------------------------------------------------- 104 - job_name: prometheus 105 static_configs: 106 - targets: ["localhost:9090"] 107