# =============================================================================
# Recording rules (TASK-0135, phase-15 §Observability)
#
# The spec's recordings: p95 per route-group, error rate (5m), queue depth
# per queue, ws connections. EXPOSED metric names are used (classic-text
# counters carry `_total` — see scrape.yaml header).
#
# route_group derivation: the api span records the ROUTE PATTERN in `route`.
# A nested label_replace chain materializes a bounded `route_group` label
# (innermost = default "other"; outermost = highest priority, it wins the
# overwrite). Nine groups cover the 29 route controllers + the default
# (pages/…) controller; unmatched patterns (404s, /metrics, internal probes)
# aggregate in `other` so a typo storm cannot explode cardinality.
#
#   search     /api/v1/search
#   ai         /api/v1/workspaces/:id/ai/**
#   files      /api/v1/files/**
#   account    auth|sessions|oauth|saml|sso|users|notifications|webhooks/incoming|scim/v2
#   content    pages|blocks|comments|databases (top-level)
#   workspaces workspaces|orgs (incl. their sub-routes, except ai)
#   system     health|docs|internal
#   other      everything else
# =============================================================================
groups:
  # --- HTTP: latency quantiles + rates per route-group ------------------------
  - name: factory.api.route-group
    interval: 15s
    rules:
      - record: factory_api:http_request_duration_seconds:p95
        expr: |
          histogram_quantile(0.95,
            sum by (le, route_group) (
              rate(
                label_replace(
                  label_replace(
                    label_replace(
                      label_replace(
                        label_replace(
                          label_replace(
                            label_replace(
                              http_request_duration_seconds_bucket{service_name="factory-api"},
                              "route_group", "other",      "__name__", ".*"
                            ),
                            "route_group", "workspaces", "route", "^/api/v1/(workspaces|orgs)"
                          ),
                          "route_group", "content",    "route", "^/api/v1/(pages|blocks|comments|databases)"
                        ),
                        "route_group", "account",    "route", "^/api/v1/(auth|sessions|oauth|saml|sso|users|notifications|webhooks/incoming|scim/v2)"
                      ),
                      "route_group", "files",        "route", "^/api/v1/files"
                    ),
                    "route_group", "search",       "route", "^/api/v1/search"
                  ),
                  "route_group", "ai",             "route", "^/api/v1/workspaces/[^/]+/ai"
                ),
                "route_group", "system",        "route", "^/api/v1/(health|docs|internal)"
              )[5m]
            )
          )
        labels:
          quantile: "0.95"
      - record: factory_api:http_request_duration_seconds:p50
        expr: |
          histogram_quantile(0.5,
            sum by (le, route_group) (
              rate(
                label_replace(
                  label_replace(
                    label_replace(
                      label_replace(
                        label_replace(
                          label_replace(
                            label_replace(
                              http_request_duration_seconds_bucket{service_name="factory-api"},
                              "route_group", "other",      "__name__", ".*"
                            ),
                            "route_group", "workspaces", "route", "^/api/v1/(workspaces|orgs)"
                          ),
                          "route_group", "content",    "route", "^/api/v1/(pages|blocks|comments|databases)"
                        ),
                        "route_group", "account",    "route", "^/api/v1/(auth|sessions|oauth|saml|sso|users|notifications|webhooks/incoming|scim/v2)"
                      ),
                      "route_group", "files",        "route", "^/api/v1/files"
                    ),
                    "route_group", "search",       "route", "^/api/v1/search"
                  ),
                  "route_group", "ai",             "route", "^/api/v1/workspaces/[^/]+/ai"
                ),
                "route_group", "system",        "route", "^/api/v1/(health|docs|internal)"
              )[5m]
            )
          )
        labels:
          quantile: "0.5"
      - record: factory_api:http_request_duration_seconds:p99
        expr: |
          histogram_quantile(0.99,
            sum by (le, route_group) (
              rate(
                label_replace(
                  label_replace(
                    label_replace(
                      label_replace(
                        label_replace(
                          label_replace(
                            label_replace(
                              http_request_duration_seconds_bucket{service_name="factory-api"},
                              "route_group", "other",      "__name__", ".*"
                            ),
                            "route_group", "workspaces", "route", "^/api/v1/(workspaces|orgs)"
                          ),
                          "route_group", "content",    "route", "^/api/v1/(pages|blocks|comments|databases)"
                        ),
                        "route_group", "account",    "route", "^/api/v1/(auth|sessions|oauth|saml|sso|users|notifications|webhooks/incoming|scim/v2)"
                      ),
                      "route_group", "files",        "route", "^/api/v1/files"
                    ),
                    "route_group", "search",       "route", "^/api/v1/search"
                  ),
                  "route_group", "ai",             "route", "^/api/v1/workspaces/[^/]+/ai"
                ),
                "route_group", "system",        "route", "^/api/v1/(health|docs|internal)"
              )[5m]
            )
          )
        labels:
          quantile: "0.99"
      - record: factory_api:http_requests:rate5m
        expr: |
          sum by (route_group) (
            rate(
              label_replace(
                label_replace(
                  label_replace(
                    label_replace(
                      label_replace(
                        label_replace(
                          label_replace(
                            http_request_count_total{service_name="factory-api"},
                            "route_group", "other",      "__name__", ".*"
                          ),
                          "route_group", "workspaces", "route", "^/api/v1/(workspaces|orgs)"
                        ),
                        "route_group", "content",    "route", "^/api/v1/(pages|blocks|comments|databases)"
                      ),
                      "route_group", "account",    "route", "^/api/v1/(auth|sessions|oauth|saml|sso|users|notifications|webhooks/incoming|scim/v2)"
                    ),
                    "route_group", "files",        "route", "^/api/v1/files"
                  ),
                  "route_group", "search",       "route", "^/api/v1/search"
                ),
                "route_group", "ai",             "route", "^/api/v1/workspaces/[^/]+/ai"
              ),
              "route_group", "system",        "route", "^/api/v1/(health|docs|internal)"
            )[5m]
          )
      # --- error rate (5 m) — the spec's ApiErrorRateHigh window --------------
      - record: factory_api:http_request_error_rate:5m
        expr: |
          sum(rate(http_request_count_total{service_name="factory-api", status=~"5.."}[5m]))
            / ignoring (le) group_left()
            (sum(rate(http_request_count_total{service_name="factory-api"}[5m])) > 0)

  # --- queue depth per queue + job throughput ---------------------------------
  - name: factory.queues
    interval: 15s
    rules:
      - record: factory_worker:queue_depth
        expr: queue_depth{service_name="factory-worker"}
      - record: factory_worker:queue_jobs:rate5m
        expr: sum by (queue) (rate(queue_job_duration_seconds_count{service_name="factory-worker"}[5m]))
      - record: factory_worker:queue_job_duration_seconds:p95
        expr: |
          histogram_quantile(0.95,
            sum by (le, queue) (rate(queue_job_duration_seconds_bucket{service_name="factory-worker"}[10m]))
          )
        labels:
          quantile: "0.95"

  # --- realtime (the ws-connections recording backs the drop alert) -----------
  - name: factory.realtime
    interval: 15s
    rules:
      - record: factory_realtime:ws_connections
        expr: sum by (pod) (ws_active_connections{service_name="factory-realtime"})
      - record: factory_realtime:ws_messages:rate5m
        expr: sum(rate(ws_messages_total{service_name="factory-realtime"}[5m]))
      # search-index consumer lag (operational metric; the SearchIndexLagHigh
      # alert reads it directly).
      - record: factory_worker:search_index_lag_seconds
        expr: search_index_lag_seconds{service_name="factory-worker"}

  # --- search / ai / storage / db convenience series ---------------------------
  - name: factory.search.ai
    interval: 15s
    rules:
      - record: factory_api:search_query_duration_seconds:p95
        expr: |
          histogram_quantile(0.95,
            sum by (le, type) (rate(search_query_duration_seconds_bucket{service_name="factory-api"}[10m]))
          )
        labels:
          quantile: "0.95"
      - record: factory_api:ai_tokens:rate5m
        expr: sum by (direction, model) (rate(ai_tokens_total{service_name="factory-api"}[5m]))
      - record: factory_api:ai_request_duration_seconds:p95
        expr: |
          histogram_quantile(0.95,
            sum by (le, model) (rate(ai_request_duration_seconds_bucket{service_name="factory-api"}[10m]))
          )
        labels:
          quantile: "0.95"
      - record: factory_api:db_query_duration_seconds:p95
        expr: |
          histogram_quantile(0.95,
            sum by (le, model) (rate(db_query_duration_seconds_bucket{service_name=~"factory-(api|worker)"}[10m]))
          )
        labels:
          quantile: "0.95"