1# ============================================================================= 2# Recording rules (TASK-0135, phase-15 §Observability) 3# 4# The spec's recordings: p95 per route-group, error rate (5m), queue depth 5# per queue, ws connections. EXPOSED metric names are used (classic-text 6# counters carry `_total` — see scrape.yaml header). 7# 8# route_group derivation: the api span records the ROUTE PATTERN in `route`. 9# A nested label_replace chain materializes a bounded `route_group` label 10# (innermost = default "other"; outermost = highest priority, it wins the 11# overwrite). Nine groups cover the 29 route controllers + the default 12# (pages/…) controller; unmatched patterns (404s, /metrics, internal probes) 13# aggregate in `other` so a typo storm cannot explode cardinality. 14# 15# search /api/v1/search 16# ai /api/v1/workspaces/:id/ai/** 17# files /api/v1/files/** 18# account auth|sessions|oauth|saml|sso|users|notifications|webhooks/incoming|scim/v2 19# content pages|blocks|comments|databases (top-level) 20# workspaces workspaces|orgs (incl. their sub-routes, except ai) 21# system health|docs|internal 22# other everything else 23# ============================================================================= 24groups: 25 # --- HTTP: latency quantiles + rates per route-group ------------------------ 26 - name: factory.api.route-group 27 interval: 15s 28 rules: 29 - record: factory_api:http_request_duration_seconds:p95 30 expr: | 31 histogram_quantile(0.95, 32 sum by (le, route_group) ( 33 rate( 34 label_replace( 35 label_replace( 36 label_replace( 37 label_replace( 38 label_replace( 39 label_replace( 40 label_replace( 41 http_request_duration_seconds_bucket{service_name="factory-api"}, 42 "route_group", "other", "__name__", ".*" 43 ), 44 "route_group", "workspaces", "route", "^/api/v1/(workspaces|orgs)" 45 ), 46 "route_group", "content", "route", "^/api/v1/(pages|blocks|comments|databases)" 47 ), 48 "route_group", "account", "route", "^/api/v1/(auth|sessions|oauth|saml|sso|users|notifications|webhooks/incoming|scim/v2)" 49 ), 50 "route_group", "files", "route", "^/api/v1/files" 51 ), 52 "route_group", "search", "route", "^/api/v1/search" 53 ), 54 "route_group", "ai", "route", "^/api/v1/workspaces/[^/]+/ai" 55 ), 56 "route_group", "system", "route", "^/api/v1/(health|docs|internal)" 57 )[5m] 58 ) 59 ) 60 labels: 61 quantile: "0.95" 62 - record: factory_api:http_request_duration_seconds:p50 63 expr: | 64 histogram_quantile(0.5, 65 sum by (le, route_group) ( 66 rate( 67 label_replace( 68 label_replace( 69 label_replace( 70 label_replace( 71 label_replace( 72 label_replace( 73 label_replace( 74 http_request_duration_seconds_bucket{service_name="factory-api"}, 75 "route_group", "other", "__name__", ".*" 76 ), 77 "route_group", "workspaces", "route", "^/api/v1/(workspaces|orgs)" 78 ), 79 "route_group", "content", "route", "^/api/v1/(pages|blocks|comments|databases)" 80 ), 81 "route_group", "account", "route", "^/api/v1/(auth|sessions|oauth|saml|sso|users|notifications|webhooks/incoming|scim/v2)" 82 ), 83 "route_group", "files", "route", "^/api/v1/files" 84 ), 85 "route_group", "search", "route", "^/api/v1/search" 86 ), 87 "route_group", "ai", "route", "^/api/v1/workspaces/[^/]+/ai" 88 ), 89 "route_group", "system", "route", "^/api/v1/(health|docs|internal)" 90 )[5m] 91 ) 92 ) 93 labels: 94 quantile: "0.5" 95 - record: factory_api:http_request_duration_seconds:p99 96 expr: | 97 histogram_quantile(0.99, 98 sum by (le, route_group) ( 99 rate( 100 label_replace( 101 label_replace( 102 label_replace( 103 label_replace( 104 label_replace( 105 label_replace( 106 label_replace( 107 http_request_duration_seconds_bucket{service_name="factory-api"}, 108 "route_group", "other", "__name__", ".*" 109 ), 110 "route_group", "workspaces", "route", "^/api/v1/(workspaces|orgs)" 111 ), 112 "route_group", "content", "route", "^/api/v1/(pages|blocks|comments|databases)" 113 ), 114 "route_group", "account", "route", "^/api/v1/(auth|sessions|oauth|saml|sso|users|notifications|webhooks/incoming|scim/v2)" 115 ), 116 "route_group", "files", "route", "^/api/v1/files" 117 ), 118 "route_group", "search", "route", "^/api/v1/search" 119 ), 120 "route_group", "ai", "route", "^/api/v1/workspaces/[^/]+/ai" 121 ), 122 "route_group", "system", "route", "^/api/v1/(health|docs|internal)" 123 )[5m] 124 ) 125 ) 126 labels: 127 quantile: "0.99" 128 - record: factory_api:http_requests:rate5m 129 expr: | 130 sum by (route_group) ( 131 rate( 132 label_replace( 133 label_replace( 134 label_replace( 135 label_replace( 136 label_replace( 137 label_replace( 138 label_replace( 139 http_request_count_total{service_name="factory-api"}, 140 "route_group", "other", "__name__", ".*" 141 ), 142 "route_group", "workspaces", "route", "^/api/v1/(workspaces|orgs)" 143 ), 144 "route_group", "content", "route", "^/api/v1/(pages|blocks|comments|databases)" 145 ), 146 "route_group", "account", "route", "^/api/v1/(auth|sessions|oauth|saml|sso|users|notifications|webhooks/incoming|scim/v2)" 147 ), 148 "route_group", "files", "route", "^/api/v1/files" 149 ), 150 "route_group", "search", "route", "^/api/v1/search" 151 ), 152 "route_group", "ai", "route", "^/api/v1/workspaces/[^/]+/ai" 153 ), 154 "route_group", "system", "route", "^/api/v1/(health|docs|internal)" 155 )[5m] 156 ) 157 # --- error rate (5 m) — the spec's ApiErrorRateHigh window -------------- 158 - record: factory_api:http_request_error_rate:5m 159 expr: | 160 sum(rate(http_request_count_total{service_name="factory-api", status=~"5.."}[5m])) 161 / ignoring (le) group_left() 162 (sum(rate(http_request_count_total{service_name="factory-api"}[5m])) > 0) 163 164 # --- queue depth per queue + job throughput --------------------------------- 165 - name: factory.queues 166 interval: 15s 167 rules: 168 - record: factory_worker:queue_depth 169 expr: queue_depth{service_name="factory-worker"} 170 - record: factory_worker:queue_jobs:rate5m 171 expr: sum by (queue) (rate(queue_job_duration_seconds_count{service_name="factory-worker"}[5m])) 172 - record: factory_worker:queue_job_duration_seconds:p95 173 expr: | 174 histogram_quantile(0.95, 175 sum by (le, queue) (rate(queue_job_duration_seconds_bucket{service_name="factory-worker"}[10m])) 176 ) 177 labels: 178 quantile: "0.95" 179 180 # --- realtime (the ws-connections recording backs the drop alert) ----------- 181 - name: factory.realtime 182 interval: 15s 183 rules: 184 - record: factory_realtime:ws_connections 185 expr: sum by (pod) (ws_active_connections{service_name="factory-realtime"}) 186 - record: factory_realtime:ws_messages:rate5m 187 expr: sum(rate(ws_messages_total{service_name="factory-realtime"}[5m])) 188 # search-index consumer lag (operational metric; the SearchIndexLagHigh 189 # alert reads it directly). 190 - record: factory_worker:search_index_lag_seconds 191 expr: search_index_lag_seconds{service_name="factory-worker"} 192 193 # --- search / ai / storage / db convenience series --------------------------- 194 - name: factory.search.ai 195 interval: 15s 196 rules: 197 - record: factory_api:search_query_duration_seconds:p95 198 expr: | 199 histogram_quantile(0.95, 200 sum by (le, type) (rate(search_query_duration_seconds_bucket{service_name="factory-api"}[10m])) 201 ) 202 labels: 203 quantile: "0.95" 204 - record: factory_api:ai_tokens:rate5m 205 expr: sum by (direction, model) (rate(ai_tokens_total{service_name="factory-api"}[5m])) 206 - record: factory_api:ai_request_duration_seconds:p95 207 expr: | 208 histogram_quantile(0.95, 209 sum by (le, model) (rate(ai_request_duration_seconds_bucket{service_name="factory-api"}[10m])) 210 ) 211 labels: 212 quantile: "0.95" 213 - record: factory_api:db_query_duration_seconds:p95 214 expr: | 215 histogram_quantile(0.95, 216 sum by (le, model) (rate(db_query_duration_seconds_bucket{service_name=~"factory-(api|worker)"}[10m])) 217 ) 218 labels: 219 quantile: "0.95" 220