1
50
51import { readFileSync, existsSync } from "node:fs";
52import { dirname, join } from "node:path";
53import { fileURLToPath } from "node:url";
54
55import { signS3Request, type S3Config } from "@factory/storage";
56
57
58
59
60import {
61 ageMsSince,
62 fmtAge,
63 recordUnavailable,
64 verdictFor,
65 type CheckResult
66} from "./backup-health-decisions.js";
67
68
69
70const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..");
71
72
73function loadDotEnv(): Record<string, string> {
74 const out: Record<string, string> = {};
75 const path = join(REPO_ROOT, ".env");
76 if (!existsSync(path)) return out;
77 for (const line of readFileSync(path, "utf8").split("\n")) {
78 const t = line.trim();
79 if (t === "" || t.startsWith("#")) continue;
80 const eq = t.indexOf("=");
81 if (eq <= 0) continue;
82 out[t.slice(0, eq).trim()] = t.slice(eq + 1).trim();
83 }
84 return out;
85}
86
87const fileEnv = loadDotEnv();
88const env = (key: string, fallback: string): string =>
89 process.env[key] ?? fileEnv[key] ?? fallback;
90
91function safeOrigin(raw: string, name: string): string {
92 let u: URL;
93 try {
94 u = new URL(raw);
95 } catch {
96 throw new Error(`unsafe value for ${name}: ${JSON.stringify(raw)}`);
97 }
98 if (u.protocol !== "http:" && u.protocol !== "https:" || u.pathname !== "/" || u.search !== "") {
99 throw new Error(`unsafe value for ${name} (bare http/https origin only): ${JSON.stringify(raw)}`);
100 }
101 return `${u.protocol}//${u.host}`;
102}
103const config: S3Config = {
104 endpoint: safeOrigin(env("S3_ENDPOINT", "http://localhost:9000"), "S3_ENDPOINT"),
105 region: env("S3_REGION", "us-east-1"),
106 bucket: "",
107 accessKeyId: env("S3_ACCESS_KEY", "minioadmin"),
108 secretAccessKey: env("S3_SECRET_KEY", "minioadmin"),
109 forcePathStyle: true,
110};
111
112
113function requireSafeToken(value: string, name: string, re: RegExp): string {
114 if (!re.test(value)) {
115 throw new Error(`unsafe value for ${name}: ${JSON.stringify(value)}`);
116 }
117 return value;
118}
119const BUCKET = requireSafeToken(env("BACKUP_S3_BUCKET", "factory-backups"), "BACKUP_S3_BUCKET", /^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/);
120const PG_PREFIX = requireSafeToken(env("BACKUP_S3_PG_PREFIX", "pg"), "BACKUP_S3_PG_PREFIX", /^[a-z0-9][a-z0-9._-]*$/).replace(/\/+$/, "");
121const OS_REPO = requireSafeToken(env("OS_SNAPSHOT_REPO", "factory-s3"), "OS_SNAPSHOT_REPO", /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/);
122const OS_URL = ((): string => {
123 const raw = env("OS_URL", "http://localhost:9200");
124 let u: URL;
125 try {
126 u = new URL(raw);
127 } catch {
128 throw new Error(`unsafe value for OS_URL: ${JSON.stringify(raw)}`);
129 }
130 if (u.protocol !== "http:" && u.protocol !== "https:") {
131 throw new Error(`unsafe value for OS_URL (http/https only): ${JSON.stringify(raw)}`);
132 }
133 return `${u.protocol}//${u.host}`;
134})();
135const BASE_MAX_HOURS = Number(env("BASE_MAX_HOURS", "26"));
136const WAL_MAX_MINUTES = Number(env("WAL_MAX_MINUTES", "10"));
137const OS_MAX_HOURS = Number(env("OS_MAX_HOURS", "26"));
138const STRICT =
139 env("BACKUP_HEALTH_STRICT", "0") === "1" || env("DEPLOY_STAGING", "false") === "true";
140
141
142
143
144const EMPTY_SHA =
145 "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
146
147interface S3Listing {
148
149 newestLastModified: string | null;
150 count: number;
151 truncated: boolean;
152}
153
154
161async function newestUnderPrefix(prefix: string): Promise<S3Listing> {
162 let newest: string | null = null;
163 let count = 0;
164 let continuationToken: string | undefined;
165 let truncated = false;
166 const MAX_PAGES = 100;
167 let page = 0;
168
169 do {
170 const query: Record<string, string> = {
171 "list-type": "2",
172 prefix,
173 "max-keys": "1000",
174 };
175 if (continuationToken) query["continuation-token"] = continuationToken;
176
177 const signed = signS3Request({
178 method: "GET",
179 path: `/${BUCKET}`,
180 query,
181 headers: {},
182 payloadHash: EMPTY_SHA,
183 config: { ...config, bucket: BUCKET },
184 now: new Date(),
185 alwaysSignContentSha: true,
186 });
187 const qs = new URLSearchParams(query).toString();
188 const res = await fetch(`${config.endpoint}/${BUCKET}?${qs}`, {
189 method: "GET",
190 headers: signed.headers,
191 });
192 if (res.status !== 200) {
193 throw new Error(`ListObjectsV2 ${prefix} → HTTP ${res.status}: ${(await res.text()).slice(0, 200)}`);
194 }
195 const xml = await res.text();
196
197 const lm = /<LastModified>([\s\S]*?)<\/LastModified>/g;
198 let m: RegExpExecArray | null;
199 while ((m = lm.exec(xml)) !== null) {
200 count += 1;
201 const t = m[1]!.trim();
202 if (newest === null || t > newest) newest = t;
203 }
204 truncated = /<IsTruncated>true<\/IsTruncated>/.test(xml);
205 continuationToken =
206 /<NextContinuationToken>([\s\S]*?)<\/NextContinuationToken>/.exec(xml)?.[1] ?? undefined;
207 page += 1;
208 } while (truncated && continuationToken !== undefined && page < MAX_PAGES);
209
210 if (truncated && page >= MAX_PAGES) {
211 console.warn(` ! prefix ${prefix} truncated at ${MAX_PAGES} pages (count >= ${count}); newest may be an under-estimate`);
212 }
213 return { newestLastModified: newest, count, truncated: page >= MAX_PAGES && truncated };
214}
215
216
217
218interface OsSnapshot {
219 name: string | null;
220 state: string | null;
221 startMillis: number | null;
222}
223
224async function newestOsSnapshot(): Promise<OsSnapshot> {
225 const res = await fetch(`${OS_URL}/_snapshot/${OS_REPO}/_all`);
226 if (res.status === 404) {
227
228 return { name: null, state: null, startMillis: null };
229 }
230 if (!res.ok) {
231 throw new Error(`OS snapshot list → HTTP ${res.status}: ${(await res.text()).slice(0, 200)}`);
232 }
233 const body = (await res.json()) as {
234 snapshots?: Array<{ snapshot: string; state?: string; start_time_in_millis?: number }>;
235 };
236 const snaps = body.snapshots ?? [];
237 let best: OsSnapshot = { name: null, state: null, startMillis: null };
238 for (const s of snaps) {
239 if ((s.state ?? "SUCCESS") !== "SUCCESS") continue;
240 const start = s.start_time_in_millis ?? null;
241 if (start !== null && (best.startMillis === null || start > best.startMillis)) {
242 best = { name: s.snapshot, state: s.state ?? "SUCCESS", startMillis: start };
243 }
244 }
245 return best;
246}
247
248
249
250
251
252async function main(): Promise<void> {
253 const now = new Date();
254 console.log(`backup-health: ${now.toISOString()} (strict=${STRICT ? "yes" : "no (dev report-only)"})`);
255 console.log(` S3=${config.endpoint} bucket=${BUCKET} prefix=${PG_PREFIX}/ OS=${OS_URL} repo=${OS_REPO}`);
256
257 const results: CheckResult[] = [];
258
259
260 try {
261 const listing = await newestUnderPrefix(`${PG_PREFIX}/basebackups_005/`);
262 if (listing.newestLastModified === null) {
263 throw new Error(`no base backups found under ${PG_PREFIX}/basebackups_005/ (0 objects)`);
264 }
265 const age = ageMsSince(listing.newestLastModified, now);
266 const limit = BASE_MAX_HOURS * 3600000;
267 const pass = age <= limit;
268 results.push({
269 label: "Postgres base backup",
270 status: pass ? "PASS" : "FAIL",
271 ageMs: age,
272 detail: `newest=${listing.newestLastModified} (${fmtAge(age)} old, ${listing.count} objects) limit=${BASE_MAX_HOURS}h ${pass ? "✓" : "✗ STALE"}`,
273 });
274 } catch (err) {
275 recordUnavailable(results, "Postgres base backup", err, `${BASE_MAX_HOURS}h`, STRICT);
276 }
277
278
279 try {
280 const listing = await newestUnderPrefix(`${PG_PREFIX}/wal_005/`);
281 if (listing.newestLastModified === null) {
282 throw new Error(`no WAL archived under ${PG_PREFIX}/wal_005/ (0 objects) — archive_command may not be running`);
283 }
284 const age = ageMsSince(listing.newestLastModified, now);
285 const limit = WAL_MAX_MINUTES * 60000;
286 const pass = age <= limit;
287 results.push({
288 label: "Postgres WAL ship",
289 status: pass ? "PASS" : "FAIL",
290 ageMs: age,
291 detail: `newest=${listing.newestLastModified} (${fmtAge(age)} old, ${listing.count} segments) limit=${WAL_MAX_MINUTES}m ${pass ? "✓" : "✗ STALE"}`,
292 });
293 } catch (err) {
294 recordUnavailable(results, "Postgres WAL ship", err, `${WAL_MAX_MINUTES}m`, STRICT);
295 }
296
297
298 try {
299 const snap = await newestOsSnapshot();
300 if (snap.startMillis === null) {
301 throw new Error(`no successful snapshot found in repo ${OS_REPO} (repo may not exist or never succeeded)`);
302 }
303 const age = now.getTime() - snap.startMillis;
304 const limit = OS_MAX_HOURS * 3600000;
305 const pass = age <= limit;
306 results.push({
307 label: "OpenSearch snapshot",
308 status: pass ? "PASS" : "FAIL",
309 ageMs: age,
310 detail: `newest=${snap.name} state=${snap.state} started=${new Date(snap.startMillis).toISOString()} (${fmtAge(age)} old) limit=${OS_MAX_HOURS}h ${pass ? "✓" : "✗ STALE"}`,
311 });
312 } catch (err) {
313 recordUnavailable(results, "OpenSearch snapshot", err, `${OS_MAX_HOURS}h`, STRICT);
314 }
315
316
317 console.log("");
318 for (const r of results) {
319 const mark = r.status === "PASS" ? "✓" : r.status === "FAIL" ? "✗" : "⚠";
320 console.log(` ${mark} [${r.status}] ${r.label}: ${r.detail}`);
321 }
322 const verdict = verdictFor(results);
323 console.log("");
324 if (verdict.exitCode !== 0) {
325 console.error(`backup-health: ${verdict.message}`);
326 } else if (verdict.skips.length > 0) {
327 console.warn(`backup-health: ${verdict.message}`);
328 } else {
329 console.log(`backup-health: ${verdict.message}`);
330 }
331 process.exit(verdict.exitCode);
332}
333
334main().catch((err) => {
335 console.error(`backup-health crashed: ${(err as Error).stack ?? err}`);
336 process.exit(1);
337});
338
