storage-orphans.ts

storage-orphans.tsscripts/storage-orphans.tsraw ← scripts4,753 B← Back to the app
1/**
2 * AI Software Factory — S3 orphan-object check (TASK-0104 §1/§3).
3 *
4 * The acceptance-criterion check script: reports how many objects under the
5 * files bucket's `files/` prefix have NO matching `File`/`FileVersion` ledger
6 * row ("orphan keys"). After a correct purge/replace workflow this reports
7 * `0`.
8 *
9 *   pnpm storage:orphans              # DRY-RUN — report orphans (default)
10 *   pnpm storage:orphans --purge      # delete orphans (manual action)
11 *   pnpm storage:orphans --grace-days 0   # no grace window (default for --purge)
12 *
13 * The daily `maintenance:orphans` worker job runs the SAME core
14 * (`apps/worker/src/processors/files/cleanup.ts::runOrphanSweep`) with a 7-day
15 * grace + deletes; this CLI is the on-demand/manual entry point (and the
16 * integration/E2E acceptance check).
17 *
18 * Reads `.env` (repo root) for `DATABASE_URL` + the S3 `files/` bucket config.
19 */
20import { readFileSync, existsSync } from "node:fs";
21import { dirname, join } from "node:path";
22import { fileURLToPath } from "node:url";
23import { PrismaClient } from "@prisma/client";
24import { S3Client, type S3Config } from "@factory/storage";
25// The worker's orphan core (imports only @prisma/client + @factory/storage —
26// resolvable from the root via tsx; see cleanup.ts dependency note).
27import { runOrphanSweep, ORPHAN_GRACE_DAYS_DEFAULT } from "../apps/worker/src/processors/files/cleanup.js";
28 
29const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..");
30 
31/** Minimal `.env` loader (KEY=VALUE lines, `#` comments) — mirrors
32 *  scripts/minio-init.ts (the root has no dotenv dep). */
33function loadDotEnv(): Record<string, string> {
34  const out: Record<string, string> = {};
35  const path = join(REPO_ROOT, ".env");
36  if (!existsSync(path)) return out;
37  for (const line of readFileSync(path, "utf8").split("\n")) {
38    const t = line.trim();
39    if (t === "" || t.startsWith("#")) continue;
40    const eq = t.indexOf("=");
41    if (eq <= 0) continue;
42    out[t.slice(0, eq).trim()] = t.slice(eq + 1).trim();
43  }
44  return out;
45}
46 
47const fileEnv = loadDotEnv();
48const env = (key: string, fallback: string): string =>
49  process.env[key] ?? fileEnv[key] ?? fallback;
50 
51// Prisma reads DATABASE_URL from the environment (not .env automatically when
52// a client is built programmatically) — seed it from the file if absent.
53if (!process.env.DATABASE_URL) process.env.DATABASE_URL = fileEnv.DATABASE_URL;
54 
55interface Args {
56  purge: boolean;
57  graceDays: number;
58  fileId?: string;
59}
60 
61function parseArgs(argv: string[]): Args {
62  const args: Args = { purge: false, graceDays: ORPHAN_GRACE_DAYS_DEFAULT };
63  for (let i = 0; i < argv.length; i++) {
64    const a = argv[i];
65    if (a === "--purge") args.purge = true;
66    else if (a === "--grace-days") args.graceDays = Number(argv[++i] ?? "0");
67    else if (a === "--file") args.fileId = argv[++i];
68  }
69  // Default: a manual --purge acts immediately (grace 0); a dry-run reports
70  // every orphan regardless of age (the "reports 0" acceptance check).
71  if (args.purge && !argv.includes("--grace-days")) args.graceDays = 0;
72  return args;
73}
74 
75async function main(): Promise<void> {
76  const args = parseArgs(process.argv.slice(2));
77  const config: S3Config = {
78    // Mirrors apps/api/src/modules/files/s3.ts makeFilesClient — the same
79    // bucket the pipeline writes to.
80    endpoint: env("S3_ENDPOINT", "http://localhost:9000").replace(/\/+$/, ""),
81    region: env("S3_REGION", "us-east-1"),
82    bucket: env("S3_BUCKET_FILES", env("S3_BUCKET", "factory-files")),
83    accessKeyId: env("S3_ACCESS_KEY", "minioadmin"),
84    secretAccessKey: env("S3_SECRET_KEY", "minioadmin"),
85    forcePathStyle: true,
86  };
87  const s3 = new S3Client({ config });
88  const prisma = new PrismaClient();
89 
90  const consoleLog = {
91    info: (o: Record<string, unknown>) => console.log(JSON.stringify(o)),
92    warn: (o: Record<string, unknown>) => console.warn(JSON.stringify(o)),
93  };
94  const report = await runOrphanSweep({
95    s3,
96    prisma,
97    dryRun: !args.purge,
98    graceDays: args.graceDays,
99    fileId: args.fileId,
100    log: consoleLog,
101  });
102 
103  console.log(
104    `\nstorage:orphans — ${args.purge ? "PURGE" : "dry-run"}${args.fileId ? ` (file ${args.fileId})` : " (whole bucket)"}: ` +
105      `${report.orphansFound} orphan key(s) under files/ ` +
106      `(scanned ${report.scannedKeys}, deleted ${report.deleted}, retained ${report.retained})`
107  );
108  for (const k of report.keys) console.log(`  ${args.purge ? "deleted " : "orphan  "}${k}`);
109  if (report.orphansFound === 0) console.log("  ✓ zero orphan keys");
110 
111  await prisma.$disconnect();
112  process.exit(report.orphansFound > 0 && !args.purge ? 1 : 0);
113}
114 
115main().catch((err) => {
116  console.error("storage:orphans failed:", err);
117  process.exit(2);
118});
119