/** * AI Software Factory — PITR drill: VERIFY the restore (TASK-0135 * deliverable #7). Run against the RESTORED (scratch-namespace) stack after * the restore runbook (deployment.md §8.1/§8.3) has cutover the drill DB. * * The three verification queries (the Phase-15 acceptance criteria): * * 1. PG page — the page created ~10 min before the PITR target is * present WITH ITS BLOCKS (page row + every seeded block * row, content intact). * 2. PG record — the database record created ~2 min before the target is * present (record row + its record-page + database row). * 3. OS doc — an OpenSearch query returns the seeded content document * (title match on the `factory-content` index/alias). * * Marker rows come from `seed-drill.mjs` (state file) — or, for a verify * against a pre-existing restore without the seed state, pass * `--marker ` and it discovers the rows by marker title. * * It also prints MEASUREMENTS for the drill log (runbook §8.8): * - verify duration (the verify-side RTO slice) * - PITR target → row commit times (the RPO window actually preserved: * rows committed before T survived) * * Env (all overridable — the scratch namespace points the SAME env names at * the restored stack): * DATABASE_URL (default: local dev DB) * SEARCH_URL (default: http://localhost:9200) * SEARCH_INDEX (default: factory-content) * DRILL_STATE_FILE (default: /tmp/pitr-drill-state.json) * PITR_TARGET_TIME (optional ISO-8601 — recorded/used for the RPO window) * * Usage (repo root): * node tests/integration/drill/verify-restore.mjs [--marker PITR-DRILL-…] * * Exit 0 = all three checks pass; exit 1 = any failure (the drill log must * record the failure + remediation task). */ import { PrismaClient } from "@prisma/client"; import { existsSync, readFileSync } from "node:fs"; // ── env ───────────────────────────────────────────────────────────────────── const DATABASE_URL = process.env.DATABASE_URL ?? "postgresql://factory:factory@localhost:5432/factory"; const SEARCH_URL = (process.env.SEARCH_URL ?? "http://localhost:9200").replace(/\/+$/, ""); const SEARCH_INDEX = process.env.SEARCH_INDEX ?? "factory-content"; const STATE_FILE = process.env.DRILL_STATE_FILE ?? "/tmp/pitr-drill-state.json"; const markerArgIdx = process.argv.indexOf("--marker"); const markerArg = markerArgIdx >= 0 ? process.argv[markerArgIdx + 1] : null; const VERIFY_START = new Date(); const prisma = new PrismaClient({ datasources: { db: { url: DATABASE_URL } } }); // ── state / marker resolution ─────────────────────────────────────────────── async function resolveMarker() { if (markerArg) return { marker: markerArg, state: null }; if (existsSync(STATE_FILE)) { const state = JSON.parse(readFileSync(STATE_FILE, "utf8")); return { marker: state.marker, state }; } // Discover the most recent marker by title. const page = await prisma.page.findFirst({ where: { title: { startsWith: "PITR-DRILL-" } }, orderBy: { createdAt: "desc" }, }); if (!page) { throw new Error( "no drill marker found (no --marker, no state file, no PITR-DRILL- page). " + "Run seed-drill.mjs first or pass --marker PITR-DRILL-…", ); } const marker = page.title.replace(/ page$/, ""); const recordPage = await prisma.page.findFirst({ where: { workspaceId: page.workspaceId, title: { startsWith: marker } }, orderBy: { createdAt: "desc" }, }); return { marker, state: { marker, workspaceId: page.workspaceId, pageId: page.id, recordPageId: recordPage?.id, }, }; } // ── the three checks ──────────────────────────────────────────────────────── async function checkPgPage(marker, state) { const page = state?.pageId ? await prisma.page.findUnique({ where: { id: state.pageId } }) : await prisma.page.findFirst({ where: { title: `${marker} page` } }); if (!page) { return { name: "1. PG page (created ~10 min before target)", pass: false, detail: "page row NOT present after restore", measurement: { pageFound: false }, }; } const blocks = await prisma.block.findMany({ where: { pageId: page.id } }); const contentOk = blocks.some((b) => JSON.stringify(b.content).includes(marker)); const pass = blocks.length >= 2 && contentOk; return { name: "1. PG page (created ~10 min before target)", pass, detail: pass ? `page ${page.id} present with ${blocks.length} blocks, marker content intact` : `page ${page.id} present but ${blocks.length} blocks (expected ≥2), marker content intact=${contentOk}`, measurement: { pageFound: true, pageId: page.id, pageCreatedAt: page.createdAt.toISOString(), blockCount: blocks.length, blockContentIntact: contentOk, }, }; } async function checkPgRecord(marker, state) { // The record IS a page (app invariant): find the marker's record-page in // the workspace, then the record row pointing at it. const workspaceId = state?.workspaceId ?? (await prisma.page.findFirst({ where: { title: `${marker} page` } }))?.workspaceId; if (!workspaceId) { return { name: "2. PG record (created ~2 min before target)", pass: false, detail: "could not resolve workspace (marker page absent)", measurement: { recordFound: false }, }; } const recordPage = state?.recordPageId ? await prisma.page.findUnique({ where: { id: state.recordPageId } }) : await prisma.page.findFirst({ where: { workspaceId, title: { startsWith: marker } }, orderBy: { createdAt: "desc" }, }); if (!recordPage) { return { name: "2. PG record (created ~2 min before target)", pass: false, detail: "record-page NOT present after restore", measurement: { recordFound: false }, }; } const record = await prisma.databaseRecord.findFirst({ where: { workspaceId, pageId: recordPage.id }, include: { database: true }, }); if (!record) { return { name: "2. PG record (created ~2 min before target)", pass: false, detail: `record-page ${recordPage.id} present but no DatabaseRecord row`, measurement: { recordFound: false, recordPageId: recordPage.id }, }; } return { name: "2. PG record (created ~2 min before target)", pass: true, detail: `record ${record.id} present (database ${record.database.id}, page ${record.pageId})`, measurement: { recordFound: true, recordId: record.id, recordCreatedAt: record.createdAt.toISOString(), databaseId: record.database.id, recordPageId: record.pageId, }, }; } async function checkOsDoc(marker) { // Query by the marker title — a real search, not a doc-id fetch (the drill // proves the search leg of the restore, not just object storage). const res = await fetch(`${SEARCH_URL}/${SEARCH_INDEX}/_search`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ size: 5, query: { match: { title: marker } } }), }); if (!res.ok) { const body = await res.text().catch(() => ""); return { name: "3. OpenSearch doc (seeded content doc)", pass: false, detail: `OS query failed: HTTP ${res.status} ${body.slice(0, 200)}`, measurement: { osDocFound: false, httpStatus: res.status }, }; } const body = await res.json(); const hits = body.hits?.hits ?? []; const hit = hits.find( (h) => typeof h._source?.title === "string" && h._source.title.includes(marker), ); if (!hit) { return { name: "3. OpenSearch doc (seeded content doc)", pass: false, detail: `OS query returned ${hits.length} hit(s), none titled with the marker`, measurement: { osDocFound: false, hitCount: hits.length }, }; } const bodyIntact = typeof hit._source.body === "string" && hit._source.body.includes(marker); return { name: "3. OpenSearch doc (seeded content doc)", pass: bodyIntact, detail: bodyIntact ? `OS doc ${hit._id} returned with marker body intact` : `OS doc ${hit._id} found but body lacks marker`, measurement: { osDocFound: true, osDocId: hit._id, osDocTitle: hit._source.title, osDocBodyIntact: bodyIntact, }, }; } // ── main ──────────────────────────────────────────────────────────────────── async function main() { const { marker, state } = await resolveMarker(); const targetTime = state?.targetTime ?? process.env.PITR_TARGET_TIME ?? null; console.log(`verify-restore: marker ${marker}`); console.log(` DATABASE_URL ${DATABASE_URL.replace(/:[^:@/]+@/, ":***@")}`); console.log(` SEARCH_URL ${SEARCH_URL} (index ${SEARCH_INDEX})`); console.log(` PITR target ${targetTime ?? "(not recorded — measure RPO against the seed log)"}`); console.log(` verify start ${VERIFY_START.toISOString()}`); console.log(""); const results = [ await checkPgPage(marker, state), await checkPgRecord(marker, state), await checkOsDoc(marker), ]; for (const r of results) { console.log(` ${r.pass ? "✓ PASS" : "✗ FAIL"} ${r.name}`); console.log(` ${r.detail}`); } // ── measurements (for the drill log, runbook §8.8) ──────────────────────── const verifyEnd = new Date(); const measurements = { marker, targetTime, verifyStart: VERIFY_START.toISOString(), verifyEnd: verifyEnd.toISOString(), verifyDurationMs: verifyEnd.getTime() - VERIFY_START.getTime(), page: results[0].measurement, record: results[1].measurement, os: results[2].measurement, }; // RPO window actually preserved: rows committed BEFORE the target survived. if (targetTime && measurements.page.pageFound) { const t = new Date(targetTime).getTime(); measurements.rpoWindow = { targetMinusPageCommitMs: t - new Date(measurements.page.pageCreatedAt).getTime(), targetMinusRecordCommitMs: measurements.record.recordFound && measurements.record.recordCreatedAt ? t - new Date(measurements.record.recordCreatedAt).getTime() : null, note: "positive values = committed before the target and must have survived", }; } console.log(""); console.log("measurements (paste into the drill log, runbook §8.8):"); console.log(JSON.stringify(measurements, null, 2)); const failed = results.filter((r) => !r.pass); if (failed.length > 0) { console.error(`\nverify-restore: FAIL — ${failed.length}/3 checks failed`); process.exitCode = 1; } else { console.log(`\nverify-restore: OK — all 3 checks pass`); } } main() .catch(async (err) => { console.error(`verify-restore crashed: ${err.stack ?? err}`); process.exitCode = 1; }) .finally(async () => { await prisma.$disconnect().catch(() => {}); });