worker.yaml · Source

worker.yamlinfrastructure/kubernetes/worker.yamlraw ← kubernetes3,967 B← Back to the app
1# =============================================================================
2# AI Software Factory — Worker (BullMQ) Deployment + HPA
3#
4# Consumes job queues from Redis (search indexing, file pipeline, automation
5# execution, email, webhooks). 2 replicas by default.
6#
7# HPA NOTE — custom queue-depth metric:
8#   The autoscaler below scales on the custom metric `queue_depth` (the
9#   EXPOSED name of the OTel gauge the worker itself emits — TASK-0135:
10#   packages/observability `queue_depth{queue}`, scraped by Prometheus at
11#   /metrics:4003 per queue every 15 s).
12#   For the metric to reach the autoscaler you need:
13#     1. The Prometheus scrape target (infrastructure/observability/prometheus/
14#        scrape.yaml, job `factory-worker`) — shipped with this task.
15#     2. The Prometheus Adapter (k8s) with an `APIService` mapping the custom
16#        metric, e.g. configmap adapter rules:
17#        ---
18#        rules:
19#          - seriesQuery: queue_depth{queue=~".+"}
20#            resources: {service: worker}
21#            name: {as: queueDepth}
22#            metricsQuery: |
23#              sum by (queue) (queue_depth{namespace="{{.Namespace}}",service_name="factory-worker"})
24#        (adjust the metricsQuery to your exporter's labels)
25#   Without the adapter, replace the metric with a CPU target (commented at
26#   the bottom) so the HPA still functions.
27# =============================================================================
28apiVersion: apps/v1
29kind: Deployment
30metadata:
31  name: worker
32  labels:
33    app.kubernetes.io/name: worker
34    app.kubernetes.io/component: jobs
35spec:
36  replicas: 2
37  selector:
38    matchLabels:
39      app.kubernetes.io/name: worker
40  strategy:
41    type: RollingUpdate
42    rollingUpdate:
43      maxUnavailable: 0
44      maxSurge: 1
45  template:
46    metadata:
47      labels:
48        app.kubernetes.io/name: worker
49      annotations:
50        checksum/config: "factory-config"
51    spec:
52      containers:
53        - name: worker
54          image: ghcr.io/ai-software-factory/worker:TAG
55          imagePullPolicy: IfNotPresent
56          # Workers have no HTTP port; liveness is a process heartbeat file
57          # maintained by the worker (written each loop tick). Adjust the
58          # path if your worker uses a different heartbeat mechanism.
59          envFrom:
60            - configMapRef:
61                name: factory-config
62            - secretRef:
63                name: factory-secrets
64          resources:
65            requests:
66              cpu: 250m
67              memory: 512Mi
68            limits:
69              cpu: "1"
70              memory: 1Gi
71          livenessProbe:
72            exec:
73              command:
74                - sh
75                - -c
76                - "test -f /tmp/worker.heartbeat && [ $(( $(date +%s) - $(cat /tmp/worker.heartbeat) )) -lt 90 ]"
77            initialDelaySeconds: 30
78            periodSeconds: 30
79            timeoutSeconds: 5
80            failureThreshold: 3
81          securityContext:
82            allowPrivilegeEscalation: false
83            runAsNonRoot: true
84            runAsUser: 1000
85            runAsGroup: 1000
86            seccompProfile:
87              type: RuntimeDefault
88      terminationGracePeriodSeconds: 60 # let in-flight jobs finish
89---
90apiVersion: autoscaling/v2
91kind: HorizontalPodAutoscaler
92metadata:
93  name: worker
94spec:
95  scaleTargetRef:
96    apiVersion: apps/v1
97    kind: Deployment
98    name: worker
99  minReplicas: 2
100  maxReplicas: 8
101  metrics:
102    # Preferred: scale on backlog. Requires the Prometheus Adapter setup
103    # documented in the header of this file.
104    - type: Pods
105      pods:
106        metric:
107          name: queue_depth
108        target:
109          type: AverageValue
110          averageValue: "50" # 50 pending jobs per worker pod → scale up
111  # Fallback (uncomment if the custom metric is not wired yet):
112  # - type: Resource
113  #   resource:
114  #     name: cpu
115  #     target:
116  #       type: Utilization
117  #       averageUtilization: 70
118