1
39
40import { PrismaClient } from "@prisma/client";
41import { existsSync, readFileSync } from "node:fs";
42
43
44
45const DATABASE_URL =
46 process.env.DATABASE_URL ?? "postgresql://factory:factory@localhost:5432/factory";
47const SEARCH_URL = (process.env.SEARCH_URL ?? "http://localhost:9200").replace(/\/+$/, "");
48const SEARCH_INDEX = process.env.SEARCH_INDEX ?? "factory-content";
49const STATE_FILE = process.env.DRILL_STATE_FILE ?? "/tmp/pitr-drill-state.json";
50
51const markerArgIdx = process.argv.indexOf("--marker");
52const markerArg = markerArgIdx >= 0 ? process.argv[markerArgIdx + 1] : null;
53
54const VERIFY_START = new Date();
55const prisma = new PrismaClient({ datasources: { db: { url: DATABASE_URL } } });
56
57
58
59async function resolveMarker() {
60 if (markerArg) return { marker: markerArg, state: null };
61 if (existsSync(STATE_FILE)) {
62 const state = JSON.parse(readFileSync(STATE_FILE, "utf8"));
63 return { marker: state.marker, state };
64 }
65
66 const page = await prisma.page.findFirst({
67 where: { title: { startsWith: "PITR-DRILL-" } },
68 orderBy: { createdAt: "desc" },
69 });
70 if (!page) {
71 throw new Error(
72 "no drill marker found (no --marker, no state file, no PITR-DRILL- page). " +
73 "Run seed-drill.mjs first or pass --marker PITR-DRILL-…",
74 );
75 }
76 const marker = page.title.replace(/ page$/, "");
77 const recordPage = await prisma.page.findFirst({
78 where: { workspaceId: page.workspaceId, title: { startsWith: marker } },
79 orderBy: { createdAt: "desc" },
80 });
81 return {
82 marker,
83 state: {
84 marker,
85 workspaceId: page.workspaceId,
86 pageId: page.id,
87 recordPageId: recordPage?.id,
88 },
89 };
90}
91
92
93
94async function checkPgPage(marker, state) {
95 const page = state?.pageId
96 ? await prisma.page.findUnique({ where: { id: state.pageId } })
97 : await prisma.page.findFirst({ where: { title: `${marker} page` } });
98 if (!page) {
99 return {
100 name: "1. PG page (created ~10 min before target)",
101 pass: false,
102 detail: "page row NOT present after restore",
103 measurement: { pageFound: false },
104 };
105 }
106 const blocks = await prisma.block.findMany({ where: { pageId: page.id } });
107 const contentOk = blocks.some((b) => JSON.stringify(b.content).includes(marker));
108 const pass = blocks.length >= 2 && contentOk;
109 return {
110 name: "1. PG page (created ~10 min before target)",
111 pass,
112 detail: pass
113 ? `page ${page.id} present with ${blocks.length} blocks, marker content intact`
114 : `page ${page.id} present but ${blocks.length} blocks (expected ≥2), marker content intact=${contentOk}`,
115 measurement: {
116 pageFound: true,
117 pageId: page.id,
118 pageCreatedAt: page.createdAt.toISOString(),
119 blockCount: blocks.length,
120 blockContentIntact: contentOk,
121 },
122 };
123}
124
125async function checkPgRecord(marker, state) {
126
127
128 const workspaceId =
129 state?.workspaceId ??
130 (await prisma.page.findFirst({ where: { title: `${marker} page` } }))?.workspaceId;
131 if (!workspaceId) {
132 return {
133 name: "2. PG record (created ~2 min before target)",
134 pass: false,
135 detail: "could not resolve workspace (marker page absent)",
136 measurement: { recordFound: false },
137 };
138 }
139 const recordPage = state?.recordPageId
140 ? await prisma.page.findUnique({ where: { id: state.recordPageId } })
141 : await prisma.page.findFirst({
142 where: { workspaceId, title: { startsWith: marker } },
143 orderBy: { createdAt: "desc" },
144 });
145 if (!recordPage) {
146 return {
147 name: "2. PG record (created ~2 min before target)",
148 pass: false,
149 detail: "record-page NOT present after restore",
150 measurement: { recordFound: false },
151 };
152 }
153 const record = await prisma.databaseRecord.findFirst({
154 where: { workspaceId, pageId: recordPage.id },
155 include: { database: true },
156 });
157 if (!record) {
158 return {
159 name: "2. PG record (created ~2 min before target)",
160 pass: false,
161 detail: `record-page ${recordPage.id} present but no DatabaseRecord row`,
162 measurement: { recordFound: false, recordPageId: recordPage.id },
163 };
164 }
165 return {
166 name: "2. PG record (created ~2 min before target)",
167 pass: true,
168 detail: `record ${record.id} present (database ${record.database.id}, page ${record.pageId})`,
169 measurement: {
170 recordFound: true,
171 recordId: record.id,
172 recordCreatedAt: record.createdAt.toISOString(),
173 databaseId: record.database.id,
174 recordPageId: record.pageId,
175 },
176 };
177}
178
179async function checkOsDoc(marker) {
180
181
182 const res = await fetch(`${SEARCH_URL}/${SEARCH_INDEX}/_search`, {
183 method: "POST",
184 headers: { "content-type": "application/json" },
185 body: JSON.stringify({ size: 5, query: { match: { title: marker } } }),
186 });
187 if (!res.ok) {
188 const body = await res.text().catch(() => "");
189 return {
190 name: "3. OpenSearch doc (seeded content doc)",
191 pass: false,
192 detail: `OS query failed: HTTP ${res.status} ${body.slice(0, 200)}`,
193 measurement: { osDocFound: false, httpStatus: res.status },
194 };
195 }
196 const body = await res.json();
197 const hits = body.hits?.hits ?? [];
198 const hit = hits.find(
199 (h) => typeof h._source?.title === "string" && h._source.title.includes(marker),
200 );
201 if (!hit) {
202 return {
203 name: "3. OpenSearch doc (seeded content doc)",
204 pass: false,
205 detail: `OS query returned ${hits.length} hit(s), none titled with the marker`,
206 measurement: { osDocFound: false, hitCount: hits.length },
207 };
208 }
209 const bodyIntact =
210 typeof hit._source.body === "string" && hit._source.body.includes(marker);
211 return {
212 name: "3. OpenSearch doc (seeded content doc)",
213 pass: bodyIntact,
214 detail: bodyIntact
215 ? `OS doc ${hit._id} returned with marker body intact`
216 : `OS doc ${hit._id} found but body lacks marker`,
217 measurement: {
218 osDocFound: true,
219 osDocId: hit._id,
220 osDocTitle: hit._source.title,
221 osDocBodyIntact: bodyIntact,
222 },
223 };
224}
225
226
227
228async function main() {
229 const { marker, state } = await resolveMarker();
230 const targetTime = state?.targetTime ?? process.env.PITR_TARGET_TIME ?? null;
231
232 console.log(`verify-restore: marker ${marker}`);
233 console.log(` DATABASE_URL ${DATABASE_URL.replace(/:[^:@/]+@/, ":***@")}`);
234 console.log(` SEARCH_URL ${SEARCH_URL} (index ${SEARCH_INDEX})`);
235 console.log(` PITR target ${targetTime ?? "(not recorded — measure RPO against the seed log)"}`);
236 console.log(` verify start ${VERIFY_START.toISOString()}`);
237 console.log("");
238
239 const results = [
240 await checkPgPage(marker, state),
241 await checkPgRecord(marker, state),
242 await checkOsDoc(marker),
243 ];
244
245 for (const r of results) {
246 console.log(` ${r.pass ? "✓ PASS" : "✗ FAIL"} ${r.name}`);
247 console.log(` ${r.detail}`);
248 }
249
250
251 const verifyEnd = new Date();
252 const measurements = {
253 marker,
254 targetTime,
255 verifyStart: VERIFY_START.toISOString(),
256 verifyEnd: verifyEnd.toISOString(),
257 verifyDurationMs: verifyEnd.getTime() - VERIFY_START.getTime(),
258 page: results[0].measurement,
259 record: results[1].measurement,
260 os: results[2].measurement,
261 };
262
263 if (targetTime && measurements.page.pageFound) {
264 const t = new Date(targetTime).getTime();
265 measurements.rpoWindow = {
266 targetMinusPageCommitMs: t - new Date(measurements.page.pageCreatedAt).getTime(),
267 targetMinusRecordCommitMs:
268 measurements.record.recordFound && measurements.record.recordCreatedAt
269 ? t - new Date(measurements.record.recordCreatedAt).getTime()
270 : null,
271 note: "positive values = committed before the target and must have survived",
272 };
273 }
274
275 console.log("");
276 console.log("measurements (paste into the drill log, runbook §8.8):");
277 console.log(JSON.stringify(measurements, null, 2));
278
279 const failed = results.filter((r) => !r.pass);
280 if (failed.length > 0) {
281 console.error(`\nverify-restore: FAIL — ${failed.length}/3 checks failed`);
282 process.exitCode = 1;
283 } else {
284 console.log(`\nverify-restore: OK — all 3 checks pass`);
285 }
286}
287
288main()
289 .catch(async (err) => {
290 console.error(`verify-restore crashed: ${err.stack ?? err}`);
291 process.exitCode = 1;
292 })
293 .finally(async () => {
294 await prisma.$disconnect().catch(() => {});
295 });
296
