# ============================================================================= # AI Software Factory — OpenSearch daily snapshot (CronJob, 05:30 UTC) # # TASK-0135 backup/DR deliverable. Documented decision (Phase-15 task): the # OpenSearch snapshot is a k8s CronJob that calls the OpenSearch REST API # directly — not a sidecar, not the worker (the worker only runs the # *verification* job after backups). # # Each run (30 5 * * * UTC — 30 min after the PG base backup, so a transient # outage is more likely to have recovered by snapshot time; the two are # gated independently by scripts/backup-health.ts): # # 1. Ensure the S3 repository `factory-s3` exists (idempotent PUT). The # repo points at the SAME S3 endpoint the app uses (factory-config # S3_ENDPOINT) with the storage credentials (factory-secrets # S3_ACCESS_KEY/S3_SECRET_KEY), base_path `opensearch/snapshots`. All # backups live in ONE bucket (`factory-backups`, versioning ON — a # bucket-level setting, runbook §8.6) under distinct prefixes: # s3://factory-backups/pg/basebackups_005/… PG base (wal-g) # s3://factory-backups/pg/wal_005/… PG WAL (wal-g) # s3://factory-backups/opensearch/snapshots/… OS snapshots # 2. Create snapshot `factory-YYYYmmddTHHMM` of the content indices, # TARGETED BY ALIAS NOT PHYSICAL NAME: `factory-content` (alias → # factory-content-vN — the reindexer rolls the suffix on each mapping # change, see scripts/opensearch-reindex.ts, so hardcoding # `factory-content-v1` would silently skip later versions), plus the # stable names `factory-docs`, `ai-chunks`. `ignore_unavailable: true` # tolerates indices that don't exist in a fresh cluster. # VERIFIED against a live OpenSearch 2.15 (2026-10, drill): snapshot of # the `factory-content` alias + a missing index SUCCEEDED and recorded # the concrete index it expanded to. # 3. Wait for completion (`?wait_for_completion=true`) and assert # `state == SUCCESS` (curl image has no jq — grep/sed on the small # response body). # # Health: scripts/backup-health.ts reads the newest successful snapshot # start_time from `GET /_snapshot/factory-s3/_all` directly from OpenSearch # (no marker file to keep in sync — the snapshot IS the marker). # # Restore (runbook §8.3): `POST /_snapshot/factory-s3//_restore`. # VERIFIED against live OS 2.15: restoring under the ORIGINAL index name # auto-restores the ALIAS as well (temp index + alias: delete → alias gone → # restore → doc AND alias back). OS 2.15 rejects the ES8-style # `index_rename_*` params, so a scratch restore = restore the original name # on a SCRATCH cluster (the `restore-drill` namespace); on the live cluster # a snapshot restore is a cutover. # # Image: curlimages/curl:latest (verified: curl 8.22.0 + POSIX sh; no # jq/bash/envsubst, hence grep/sed parsing). # ============================================================================= apiVersion: batch/v1 kind: CronJob metadata: name: opensearch-snapshot labels: app.kubernetes.io/name: opensearch app.kubernetes.io/component: backup app.kubernetes.io/part-of: ai-software-factory spec: schedule: "30 5 * * *" timeZone: "UTC" concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 5 jobTemplate: spec: backoffLimit: 1 activeDeadlineSeconds: 3600 template: metadata: labels: app.kubernetes.io/name: opensearch-snapshot app.kubernetes.io/component: backup app.kubernetes.io/part-of: ai-software-factory spec: restartPolicy: Never containers: - name: snapshot image: curlimages/curl:latest command: - sh - -c - | set -eu SNAP="factory-$(date -u +%Y%m%dT%H%M)" REPO="factory-s3" # 0) wait for OpenSearch to answer (up to ~5 min) i=0 until curl -sf -m 5 "$SEARCH_URL/_cluster/health" >/dev/null 2>&1; do i=$((i+1)); [ "$i" -ge 60 ] && { echo "[os-snapshot] OpenSearch not reachable at ${SEARCH_URL} after 300s"; exit 1; } sleep 5 done # 1) ensure the S3 repo exists (idempotent; an identical # re-PUT is a no-op in OpenSearch) curl -sf -X PUT "$SEARCH_URL/_snapshot/$REPO" \ -H 'content-type: application/json' \ -d "{\"type\":\"s3\",\"settings\":{\"bucket\":\"$S3_BUCKET\",\"base_path\":\"$S3_BASE_PATH\",\"endpoint\":\"$S3_ENDPOINT\",\"region\":\"$S3_REGION\",\"access_key_id\":\"$S3_ACCESS_KEY\",\"secret_access_key\":\"$S3_SECRET_KEY\"}}" \ >/dev/null \ || echo "[os-snapshot] repo PUT non-zero (exists or transient) — continuing" # 2) create the daily snapshot (alias-based indices; # ignore_unavailable tolerates fresh/missing indices) curl -sf -X PUT "$SEARCH_URL/_snapshot/$REPO/$SNAP?wait_for_completion=true" \ -H 'content-type: application/json' \ -d "{\"indices\":\"$SNAPSHOT_INDICES\",\"ignore_unavailable\":true}" \ > /tmp/snap.json # 3) assert SUCCESS (no jq in the image: grep/sed) state=$(grep -o '"state":"[A-Z]*"' /tmp/snap.json | head -1 | sed 's/.*:"//;s/"//') indices=$(grep -o '"indices":\[[^]]*\]' /tmp/snap.json | head -1) if [ "$state" != "SUCCESS" ]; then echo "[os-snapshot] FAIL: snapshot $SNAP state=${state:-UNKNOWN} (indices: $indices)" head -c 2000 /tmp/snap.json exit 1 fi echo "[os-snapshot] OK: snapshot $SNAP state=SUCCESS (indices: $indices)" env: # --- OpenSearch target (app config; k8s service DNS) --- - name: SEARCH_URL valueFrom: { configMapKeyRef: { name: factory-config, key: SEARCH_URL } } - name: SNAPSHOT_INDICES value: "factory-content,factory-docs,ai-chunks" # --- S3 repo target (the shared backup bucket) --- - name: S3_BUCKET value: "factory-backups" - name: S3_BASE_PATH value: "opensearch/snapshots" - name: S3_ENDPOINT valueFrom: { configMapKeyRef: { name: factory-config, key: S3_ENDPOINT } } - name: S3_REGION valueFrom: { configMapKeyRef: { name: factory-config, key: S3_REGION } } - name: S3_ACCESS_KEY valueFrom: { secretKeyRef: { name: factory-secrets, key: S3_ACCESS_KEY } } - name: S3_SECRET_KEY valueFrom: { secretKeyRef: { name: factory-secrets, key: S3_SECRET_KEY } }