/** * AI Software Factory — backup/DR health gate (TASK-0135, release pipeline * stage 9). * * Asserts the three "last-success" markers are fresh, using the REAL backup * artifacts (no fake status objects): * * 1. Postgres base backup — newest object LastModified under * `pg/basebackups_005/` in the backup S3 bucket (wal-g v3.0.7 layout, * verified from source: utility.BaseBackupPath = "basebackups_005/"). * Threshold: < 26 h. * 2. Postgres WAL ship — newest object LastModified under `pg/wal_005/` * (utility.WalPath = "wal_005/"). Threshold: < 10 min. * 3. OpenSearch snapshot — newest SUCCESS snapshot start_time from * `GET /_snapshot//_all`. Threshold: < 26 h. * * The base-backup CronJob (postgres/cronjob-backup.yaml) and the WAL * archive_command (postgres/configmap.yaml) produce those S3 objects; the * OpenSearch snapshot CronJob (postgres/cronjob-opensearch-snapshot.yaml) * produces the snapshot. This script only READS them — it is the independent * verifier the release pipeline runs as a hard gate. * * Behavior (documented decision): * - A REACHABLE-but-STALE marker ALWAYS fails (exit 1) — in any mode, a * backup that stopped working is a real problem, not a dev nicety. * - An UNREACHABLE S3/OS endpoint (a dev laptop with no MinIO / no * cluster): by default it is a LOUD WARNING + report-only SKIP (exit 0) * so a local `pnpm backup:health` never hard-fails a developer who * simply has no backup stack running. In STRICT mode (staging/release — * `BACKUP_HEALTH_STRICT=1` or `DEPLOY_STAGING=true`) unreachable ALSO * fails, because in staging the backup stack is expected to be up. * * Runs from the repo root: * * pnpm backup:health # dev: unreachable = loud warning * BACKUP_HEALTH_STRICT=1 pnpm backup:health # staging: hard gate * * Env (all optional, defaults match local dev; a root `.env` is loaded): * S3_ENDPOINT (http://localhost:9000) * S3_REGION (us-east-1) * S3_ACCESS_KEY / S3_SECRET_KEY (minioadmin) * BACKUP_S3_BUCKET (factory-backups) — the backup bucket (NOT the * app's factory-files bucket) * BACKUP_S3_PG_PREFIX (pg) — wal-g storage root prefix * OS_URL (http://localhost:9200) * OS_SNAPSHOT_REPO (factory-s3) * BASE_MAX_HOURS (26) WAL_MAX_MINUTES (10) OS_MAX_HOURS (26) * BACKUP_HEALTH_STRICT (0) or DEPLOY_STAGING (false) */ import { readFileSync, existsSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { signS3Request, type S3Config } from "@factory/storage"; // The PURE decision logic (verdict / auth / unavailable / age) lives in a // side-effect-free sibling module so the release-gate's decision matrix is // unit-testable (`tests/security/backup-health.test.ts`) — the script owns // only the live S3/OpenSearch checks + env + I/O. import { ageMsSince, fmtAge, recordUnavailable, verdictFor, type CheckResult } from "./backup-health-decisions.js"; // ── env ───────────────────────────────────────────────────────────────────── const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); /** Minimal `.env` loader (KEY=VALUE, `#` comments) — mirrors minio-init.ts. */ function loadDotEnv(): Record { const out: Record = {}; const path = join(REPO_ROOT, ".env"); if (!existsSync(path)) return out; for (const line of readFileSync(path, "utf8").split("\n")) { const t = line.trim(); if (t === "" || t.startsWith("#")) continue; const eq = t.indexOf("="); if (eq <= 0) continue; out[t.slice(0, eq).trim()] = t.slice(eq + 1).trim(); } return out; } const fileEnv = loadDotEnv(); const env = (key: string, fallback: string): string => process.env[key] ?? fileEnv[key] ?? fallback; function safeOrigin(raw: string, name: string): string { let u: URL; try { u = new URL(raw); } catch { throw new Error(`unsafe value for ${name}: ${JSON.stringify(raw)}`); } if (u.protocol !== "http:" && u.protocol !== "https:" || u.pathname !== "/" || u.search !== "") { throw new Error(`unsafe value for ${name} (bare http/https origin only): ${JSON.stringify(raw)}`); } return `${u.protocol}//${u.host}`; } const config: S3Config = { endpoint: safeOrigin(env("S3_ENDPOINT", "http://localhost:9000"), "S3_ENDPOINT"), region: env("S3_REGION", "us-east-1"), bucket: "", // set per-request accessKeyId: env("S3_ACCESS_KEY", "minioadmin"), secretAccessKey: env("S3_SECRET_KEY", "minioadmin"), forcePathStyle: true, // MinIO & most self-hosted S3 }; // Bucket/prefix/repo/URL come from the (untrusted) environment: validate them // so a crafted value can't turn the fixed request shapes into other paths. function requireSafeToken(value: string, name: string, re: RegExp): string { if (!re.test(value)) { throw new Error(`unsafe value for ${name}: ${JSON.stringify(value)}`); } return value; } const BUCKET = requireSafeToken(env("BACKUP_S3_BUCKET", "factory-backups"), "BACKUP_S3_BUCKET", /^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/); const PG_PREFIX = requireSafeToken(env("BACKUP_S3_PG_PREFIX", "pg"), "BACKUP_S3_PG_PREFIX", /^[a-z0-9][a-z0-9._-]*$/).replace(/\/+$/, ""); const OS_REPO = requireSafeToken(env("OS_SNAPSHOT_REPO", "factory-s3"), "OS_SNAPSHOT_REPO", /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/); const OS_URL = ((): string => { const raw = env("OS_URL", "http://localhost:9200"); let u: URL; try { u = new URL(raw); } catch { throw new Error(`unsafe value for OS_URL: ${JSON.stringify(raw)}`); } if (u.protocol !== "http:" && u.protocol !== "https:") { throw new Error(`unsafe value for OS_URL (http/https only): ${JSON.stringify(raw)}`); } return `${u.protocol}//${u.host}`; // drop any path the env var might have carried })(); const BASE_MAX_HOURS = Number(env("BASE_MAX_HOURS", "26")); const WAL_MAX_MINUTES = Number(env("WAL_MAX_MINUTES", "10")); const OS_MAX_HOURS = Number(env("OS_MAX_HOURS", "26")); const STRICT = env("BACKUP_HEALTH_STRICT", "0") === "1" || env("DEPLOY_STAGING", "false") === "true"; // ── signed S3 ListObjectsV2 (raw fetch; S3Client.listObjects drops // LastModified, which is exactly what this gate needs) ──────────────────── const EMPTY_SHA = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; interface S3Listing { /** newest LastModified (ISO) under the prefix, or null if no objects. */ newestLastModified: string | null; count: number; truncated: boolean; } /** * List all objects under `prefix` (paged, capped) and return the newest * LastModified. ListObjectsV2 returns keys in lexicographic order (NOT by * time), so we must scan to find the newest timestamp — for a busy WAL * folder this is the last few keys, but we page through honestly rather than * assume the max key is the newest. */ async function newestUnderPrefix(prefix: string): Promise { let newest: string | null = null; let count = 0; let continuationToken: string | undefined; let truncated = false; const MAX_PAGES = 100; // 100 * 1000 = 100k objects — far beyond any sane let page = 0; do { const query: Record = { "list-type": "2", prefix, "max-keys": "1000", }; if (continuationToken) query["continuation-token"] = continuationToken; const signed = signS3Request({ method: "GET", path: `/${BUCKET}`, // bucket-level op, path-style query, headers: {}, payloadHash: EMPTY_SHA, config: { ...config, bucket: BUCKET }, now: new Date(), alwaysSignContentSha: true, }); const qs = new URLSearchParams(query).toString(); const res = await fetch(`${config.endpoint}/${BUCKET}?${qs}`, { method: "GET", headers: signed.headers, }); if (res.status !== 200) { throw new Error(`ListObjectsV2 ${prefix} → HTTP ${res.status}: ${(await res.text()).slice(0, 200)}`); } const xml = await res.text(); const lm = /([\s\S]*?)<\/LastModified>/g; let m: RegExpExecArray | null; while ((m = lm.exec(xml)) !== null) { count += 1; const t = m[1]!.trim(); if (newest === null || t > newest) newest = t; // ISO-8601 Z strings sort lexicographically } truncated = /true<\/IsTruncated>/.test(xml); continuationToken = /([\s\S]*?)<\/NextContinuationToken>/.exec(xml)?.[1] ?? undefined; page += 1; } while (truncated && continuationToken !== undefined && page < MAX_PAGES); if (truncated && page >= MAX_PAGES) { console.warn(` ! prefix ${prefix} truncated at ${MAX_PAGES} pages (count >= ${count}); newest may be an under-estimate`); } return { newestLastModified: newest, count, truncated: page >= MAX_PAGES && truncated }; } // ── OpenSearch newest snapshot ────────────────────────────────────────────── interface OsSnapshot { name: string | null; state: string | null; startMillis: number | null; } async function newestOsSnapshot(): Promise { const res = await fetch(`${OS_URL}/_snapshot/${OS_REPO}/_all`); if (res.status === 404) { // repo missing → no snapshots at all return { name: null, state: null, startMillis: null }; } if (!res.ok) { throw new Error(`OS snapshot list → HTTP ${res.status}: ${(await res.text()).slice(0, 200)}`); } const body = (await res.json()) as { snapshots?: Array<{ snapshot: string; state?: string; start_time_in_millis?: number }>; }; const snaps = body.snapshots ?? []; let best: OsSnapshot = { name: null, state: null, startMillis: null }; for (const s of snaps) { if ((s.state ?? "SUCCESS") !== "SUCCESS") continue; const start = s.start_time_in_millis ?? null; if (start !== null && (best.startMillis === null || start > best.startMillis)) { best = { name: s.snapshot, state: s.state ?? "SUCCESS", startMillis: start }; } } return best; } // ── verdict helpers ───────────────────────────────────────────────────────── // ── main ──────────────────────────────────────────────────────────────────── async function main(): Promise { const now = new Date(); console.log(`backup-health: ${now.toISOString()} (strict=${STRICT ? "yes" : "no (dev report-only)"})`); console.log(` S3=${config.endpoint} bucket=${BUCKET} prefix=${PG_PREFIX}/ OS=${OS_URL} repo=${OS_REPO}`); const results: CheckResult[] = []; // ---- 1) Postgres base backup (< 26 h) ------------------------------------ try { const listing = await newestUnderPrefix(`${PG_PREFIX}/basebackups_005/`); if (listing.newestLastModified === null) { throw new Error(`no base backups found under ${PG_PREFIX}/basebackups_005/ (0 objects)`); } const age = ageMsSince(listing.newestLastModified, now); const limit = BASE_MAX_HOURS * 3600000; const pass = age <= limit; results.push({ label: "Postgres base backup", status: pass ? "PASS" : "FAIL", ageMs: age, detail: `newest=${listing.newestLastModified} (${fmtAge(age)} old, ${listing.count} objects) limit=${BASE_MAX_HOURS}h ${pass ? "✓" : "✗ STALE"}`, }); } catch (err) { recordUnavailable(results, "Postgres base backup", err, `${BASE_MAX_HOURS}h`, STRICT); } // ---- 2) Postgres WAL ship (< 10 min) ------------------------------------- try { const listing = await newestUnderPrefix(`${PG_PREFIX}/wal_005/`); if (listing.newestLastModified === null) { throw new Error(`no WAL archived under ${PG_PREFIX}/wal_005/ (0 objects) — archive_command may not be running`); } const age = ageMsSince(listing.newestLastModified, now); const limit = WAL_MAX_MINUTES * 60000; const pass = age <= limit; results.push({ label: "Postgres WAL ship", status: pass ? "PASS" : "FAIL", ageMs: age, detail: `newest=${listing.newestLastModified} (${fmtAge(age)} old, ${listing.count} segments) limit=${WAL_MAX_MINUTES}m ${pass ? "✓" : "✗ STALE"}`, }); } catch (err) { recordUnavailable(results, "Postgres WAL ship", err, `${WAL_MAX_MINUTES}m`, STRICT); } // ---- 3) OpenSearch snapshot (< 26 h) ------------------------------------- try { const snap = await newestOsSnapshot(); if (snap.startMillis === null) { throw new Error(`no successful snapshot found in repo ${OS_REPO} (repo may not exist or never succeeded)`); } const age = now.getTime() - snap.startMillis; const limit = OS_MAX_HOURS * 3600000; const pass = age <= limit; results.push({ label: "OpenSearch snapshot", status: pass ? "PASS" : "FAIL", ageMs: age, detail: `newest=${snap.name} state=${snap.state} started=${new Date(snap.startMillis).toISOString()} (${fmtAge(age)} old) limit=${OS_MAX_HOURS}h ${pass ? "✓" : "✗ STALE"}`, }); } catch (err) { recordUnavailable(results, "OpenSearch snapshot", err, `${OS_MAX_HOURS}h`, STRICT); } // ---- verdict (single source of truth: backup-health-decisions.verdictFor) -- console.log(""); for (const r of results) { const mark = r.status === "PASS" ? "✓" : r.status === "FAIL" ? "✗" : "⚠"; console.log(` ${mark} [${r.status}] ${r.label}: ${r.detail}`); } const verdict = verdictFor(results); console.log(""); if (verdict.exitCode !== 0) { console.error(`backup-health: ${verdict.message}`); } else if (verdict.skips.length > 0) { console.warn(`backup-health: ${verdict.message}`); } else { console.log(`backup-health: ${verdict.message}`); } process.exit(verdict.exitCode); } main().catch((err) => { console.error(`backup-health crashed: ${(err as Error).stack ?? err}`); process.exit(1); });