1# =============================================================================
2# AI Software Factory — OpenSearch daily snapshot (CronJob, 05:30 UTC)
3#
4# TASK-0135 backup/DR deliverable. Documented decision (Phase-15 task): the
5# OpenSearch snapshot is a k8s CronJob that calls the OpenSearch REST API
6# directly — not a sidecar, not the worker (the worker only runs the
7# *verification* job after backups).
8#
9# Each run (30 5 * * * UTC — 30 min after the PG base backup, so a transient
10# outage is more likely to have recovered by snapshot time; the two are
11# gated independently by scripts/backup-health.ts):
12#
13# 1. Ensure the S3 repository `factory-s3` exists (idempotent PUT). The
14# repo points at the SAME S3 endpoint the app uses (factory-config
15# S3_ENDPOINT) with the storage credentials (factory-secrets
16# S3_ACCESS_KEY/S3_SECRET_KEY), base_path `opensearch/snapshots`. All
17# backups live in ONE bucket (`factory-backups`, versioning ON — a
18# bucket-level setting, runbook §8.6) under distinct prefixes:
19# s3:
20# s3:
21# s3:
22# 2. Create snapshot `factory-YYYYmmddTHHMM` of the content indices,
23# TARGETED BY ALIAS NOT PHYSICAL NAME: `factory-content` (alias →
24# factory-content-vN — the reindexer rolls the suffix on each mapping
25# change, see scripts/opensearch-reindex.ts, so hardcoding
26# `factory-content-v1` would silently skip later versions), plus the
27# stable names `factory-docs`, `ai-chunks`. `ignore_unavailable: true`
28# tolerates indices that don't exist in a fresh cluster.
29# VERIFIED against a live OpenSearch 2.15 (2026-10, drill): snapshot of
30# the `factory-content` alias + a missing index SUCCEEDED and recorded
31# the concrete index it expanded to.
32# 3. Wait for completion (`?wait_for_completion=true`) and assert
33# `state == SUCCESS` (curl image has no jq — grep/sed on the small
34# response body).
35#
36# Health: scripts/backup-health.ts reads the newest successful snapshot
37# start_time from `GET /_snapshot/factory-s3/_all` directly from OpenSearch
38# (no marker file to keep in sync — the snapshot IS the marker).
39#
40# Restore (runbook §8.3): `POST /_snapshot/factory-s3/<snap>/_restore`.
41# VERIFIED against live OS 2.15: restoring under the ORIGINAL index name
42# auto-restores the ALIAS as well (temp index + alias: delete → alias gone →
43# restore → doc AND alias back). OS 2.15 rejects the ES8-style
44# `index_rename_*` params, so a scratch restore = restore the original name
45# on a SCRATCH cluster (the `restore-drill` namespace); on the live cluster
46# a snapshot restore is a cutover.
47#
48# Image: curlimages/curl:latest (verified: curl 8.22.0 + POSIX sh; no
49# jq/bash/envsubst, hence grep/sed parsing).
50# =============================================================================
51apiVersion: batch/v1
52kind: CronJob
53metadata:
54 name: opensearch-snapshot
55 labels:
56 app.kubernetes.io/name: opensearch
57 app.kubernetes.io/component: backup
58 app.kubernetes.io/part-of: ai-software-factory
59spec:
60 schedule: "30 5 * * *"
61 timeZone: "UTC"
62 concurrencyPolicy: Forbid
63 successfulJobsHistoryLimit: 3
64 failedJobsHistoryLimit: 5
65 jobTemplate:
66 spec:
67 backoffLimit: 1
68 activeDeadlineSeconds: 3600
69 template:
70 metadata:
71 labels:
72 app.kubernetes.io/name: opensearch-snapshot
73 app.kubernetes.io/component: backup
74 app.kubernetes.io/part-of: ai-software-factory
75 spec:
76 restartPolicy: Never
77 containers:
78 - name: snapshot
79 image: curlimages/curl:latest
80 command:
81 - sh
82 - -c
83 - |
84 set -eu
85 SNAP="factory-$(date -u +%Y%m%dT%H%M)"
86 REPO="factory-s3"
87
88 # 0) wait for OpenSearch to answer (up to ~5 min)
89 i=0
90 until curl -sf -m 5 "$SEARCH_URL/_cluster/health" >/dev/null 2>&1; do
91 i=$((i+1)); [ "$i" -ge 60 ] && { echo "[os-snapshot] OpenSearch not reachable at ${SEARCH_URL} after 300s"; exit 1; }
92 sleep 5
93 done
94
95 # 1) ensure the S3 repo exists (idempotent; an identical
96 # re-PUT is a no-op in OpenSearch)
97 curl -sf -X PUT "$SEARCH_URL/_snapshot/$REPO" \
98 -H 'content-type: application/json' \
99 -d "{\"type\":\"s3\",\"settings\":{\"bucket\":\"$S3_BUCKET\",\"base_path\":\"$S3_BASE_PATH\",\"endpoint\":\"$S3_ENDPOINT\",\"region\":\"$S3_REGION\",\"access_key_id\":\"$S3_ACCESS_KEY\",\"secret_access_key\":\"$S3_SECRET_KEY\"}}" \
100 >/dev/null \
101 || echo "[os-snapshot] repo PUT non-zero (exists or transient) — continuing"
102
103 # 2) create the daily snapshot (alias-based indices;
104 # ignore_unavailable tolerates fresh/missing indices)
105 curl -sf -X PUT "$SEARCH_URL/_snapshot/$REPO/$SNAP?wait_for_completion=true" \
106 -H 'content-type: application/json' \
107 -d "{\"indices\":\"$SNAPSHOT_INDICES\",\"ignore_unavailable\":true}" \
108 > /tmp/snap.json
109
110 # 3) assert SUCCESS (no jq in the image: grep/sed)
111 state=$(grep -o '"state":"[A-Z]*"' /tmp/snap.json | head -1 | sed 's/.*:"//;s/"//')
112 indices=$(grep -o '"indices":\[[^]]*\]' /tmp/snap.json | head -1)
113 if [ "$state" != "SUCCESS" ]; then
114 echo "[os-snapshot] FAIL: snapshot $SNAP state=${state:-UNKNOWN} (indices: $indices)"
115 head -c 2000 /tmp/snap.json
116 exit 1
117 fi
118 echo "[os-snapshot] OK: snapshot $SNAP state=SUCCESS (indices: $indices)"
119 env:
120 # --- OpenSearch target (app config; k8s service DNS) ---
121 - name: SEARCH_URL
122 valueFrom: { configMapKeyRef: { name: factory-config, key: SEARCH_URL } }
123 - name: SNAPSHOT_INDICES
124 value: "factory-content,factory-docs,ai-chunks"
125 # --- S3 repo target (the shared backup bucket) ---
126 - name: S3_BUCKET
127 value: "factory-backups"
128 - name: S3_BASE_PATH
129 value: "opensearch/snapshots"
130 - name: S3_ENDPOINT
131 valueFrom: { configMapKeyRef: { name: factory-config, key: S3_ENDPOINT } }
132 - name: S3_REGION
133 valueFrom: { configMapKeyRef: { name: factory-config, key: S3_REGION } }
134 - name: S3_ACCESS_KEY
135 valueFrom: { secretKeyRef: { name: factory-secrets, key: S3_ACCESS_KEY } }
136 - name: S3_SECRET_KEY
137 valueFrom: { secretKeyRef: { name: factory-secrets, key: S3_SECRET_KEY } }
138
