temp.ts · Source

temp.tsapps/worker/src/processors/files/temp.tsraw ← files5,606 B← Back to the app
1/**
2 * apps/worker — files pipeline temp-dir management (TASK-0102 §3/§7).
3 *
4 * Objects are downloaded to a local temp dir for the ClamAV scan (clamd
5 * `zCONTSCAN` needs a path) and for streaming metadata extraction. Security
6 * invariants:
7 *
8 *   - the temp dir lives on a noexec mount in the worker image (the bytes
9 *     are NEVER executed — metadata parsers + clamd only read them);
10 *   - the object "never stays on disk > scan duration + 5 min": the
11 *     processor deletes its own temp file in a `finally`, and a 5-minute
12 *     cleanup cron deletes anything older than FILES_TEMP_MAX_AGE_MS
13 *     (crash/backstop — a SIGKILL mid-pipeline still gets reaped);
14 *   - a disk guard aborts a run as FATAL when free space <
15 *     FILES_TEMP_FREE_MIN_BYTES (512 MB default) so a full disk can't wedge
16 *     the worker or corrupt other services' storage.
17 */
18import { statfs, readdir, unlink, chmod } from "node:fs/promises";
19import os from "node:os";
20import path from "node:path";
21import { randomUUID } from "node:crypto";
22import { getLogger } from "../../logger.js";
23import { FatalError } from "./errors.js";
24 
25const log = getLogger("files.temp");
26 
27/** Resolve the temp dir (FILES_TEMP_DIR or os.tmpdir()/factory-files). */
28export function tempDir(env: { FILES_TEMP_DIR: string }): string {
29  return env.FILES_TEMP_DIR || path.join(os.tmpdir(), "factory-files");
30}
31 
32/**
33 * Disk guard (TASK-0102 §7): throw FATAL when free space on the temp dir's
34 * filesystem is below the threshold. statfs is async (Node 18.15+).
35 */
36export async function assertDiskSpace(
37  dir: string,
38  minFreeBytes: number
39): Promise<void> {
40  let free: number;
41  try {
42    const stat = await statfs(dir);
43    free = stat.bavail * stat.bsize;
44  } catch (err) {
45    // statfs unsupported → we can't verify; log and allow (dev safety).
46    log.warn({ err: (err as Error).message, dir }, "statfs failed — disk guard skipped");
47    return;
48  }
49  if (free < minFreeBytes) {
50    throw new FatalError(
51      `temp disk guard: ${Math.floor(free / 1024 / 1024)} MB free < ${Math.floor(
52        minFreeBytes / 1024 / 1024
53      )} MB required`,
54      "disk_full"
55    );
56  }
57}
58 
59/**
60 * Create a private temp file in the pipeline temp dir. Returns its path;
61 * the caller owns deletion (finally block + cleanup cron as backstop).
62 */
63export async function createTempFile(
64  env: { FILES_TEMP_DIR: string },
65  minFreeBytes: number
66): Promise<string> {
67  const dir = tempDir(env);
68  // mkdir is best-effort here (the dir is created at boot by the worker);
69  // a failure will surface as a clear download error instead.
70  await import("node:fs/promises").then((m) => m.mkdir(dir, { recursive: true }));
71  // clamd runs as its own user (the `clamav` service account on bare metal;
72  // a separate container in some layouts). A 0700 dir (mkdir's default) is
73  // NOT traversable by it, so `zCONTSCAN <path>` accepts the connection and
74  // then times out — every scan would fail with "clamav scan timeout".
75  // 0755 lets clamd traverse + read; the files are random-UUID named and
76  // live for seconds (deleted in `finally` + reaped by the cleanup cron),
77  // so the transient cross-user readability is the documented trade-off.
78  try {
79    await chmod(dir, 0o755);
80  } catch (err) {
81    log.warn(
82      { err: (err as Error).message, dir },
83      "temp dir chmod 0755 failed — a different-user clamd will time out on zCONTSCAN"
84    );
85  }
86  await assertDiskSpace(dir, minFreeBytes);
87  return path.join(dir, `file-${randomUUID()}`);
88}
89 
90/** Delete a temp file if it exists. Never throws (finally-block safe). */
91export async function removeTempFile(p: string | undefined): Promise<void> {
92  if (!p) return;
93  try {
94    await unlink(p);
95  } catch (err) {
96    if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
97      log.warn({ err: (err as Error).message, p }, "temp file cleanup failed (cron backstops)");
98    }
99  }
100}
101 
102export interface TempCleanupResult {
103  deleted: number;
104  bytesFreed: number;
105}
106 
107/**
108 * The 5-minute backstop (TASK-0102 §7): delete files in the temp dir older
109 * than `maxAgeMs`. Files currently in use are newer than the threshold
110 * (pipeline runs are far shorter), so this only reaps crashes' leftovers.
111 * A directory with a live file still processing (mtime fresh) is untouched.
112 */
113export async function cleanupTempFiles(
114  env: { FILES_TEMP_DIR: string; FILES_TEMP_MAX_AGE_MS: number }
115): Promise<TempCleanupResult> {
116  const dir = tempDir(env);
117  const now = Date.now();
118  let deleted = 0;
119  let bytesFreed = 0;
120  let entries: string[];
121  try {
122    entries = await readdir(dir);
123  } catch {
124    return { deleted: 0, bytesFreed: 0 }; // dir missing → nothing to clean
125  }
126  for (const name of entries) {
127    const p = path.join(dir, name);
128    try {
129      const st = await statfsSafe(p);
130      if (!st) continue;
131      if (now - st.mtimeMs > env.FILES_TEMP_MAX_AGE_MS) {
132        await unlink(p);
133        deleted += 1;
134        bytesFreed += st.size;
135      }
136    } catch (err) {
137      if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
138        log.warn({ err: (err as Error).message, p }, "temp cleanup: stat/unlink failed");
139      }
140    }
141  }
142  if (deleted > 0) {
143    log.info({ deleted, bytesFreed }, "temp cleanup: reaped stale files");
144  }
145  return { deleted, bytesFreed };
146}
147 
148async function statfsSafe(
149  p: string
150): Promise<{ mtimeMs: number; size: number } | null> {
151  const { stat } = await import("node:fs/promises");
152  try {
153    const st = await stat(p);
154    if (!st.isFile()) return null;
155    return { mtimeMs: st.mtimeMs, size: st.size };
156  } catch {
157    return null;
158  }
159}
160