mime.ts · Source

mime.tspackages/storage/src/mime.tsraw ← src17,793 B← Back to the app
1/**
2 * @factory/storage — MIME detection (PURE).
3 *
4 * Magic-byte sniffing first (the content is the truth), then zip-container
5 * refinement for the OOXML family, then the extension fallback, then a text
6 * heuristic. The result always carries a `confidence` so the pipeline can
7 * decide how much to trust it (e.g. a `docx` refined from a bare `zip` magic
8 * is still "magic" confidence on the container, "ext" on the refinement).
9 *
10 * NOTE: the `filenameHint` is advisory. The API layer enforces the policy
11 * (`isAllowedUpload`) on the client-claimed type; the pipeline re-detects
12 * from bytes after the object lands and stores the detected type in
13 * `FileMeta.mimeType` (untrusted client input is never trusted, AGENTS.md §20).
14 */
15 
16/** How much to trust a detection result. */
17export type MimeConfidence = "magic" | "ext" | "text" | "unknown";
18 
19/** OOXML/zip refinements (docx/xlsx/pptx/epub are all zip containers). */
20export type ZipRefinement = "docx" | "xlsx" | "pptx" | "epub";
21 
22export interface MimeDetection {
23  mime: string;
24  confidence: MimeConfidence;
25  /** True when the leading magic says "zip" — the content is a container. */
26  isZipContainer?: boolean;
27  /** Refinement of a zip container (word/excel/powerpoint/epub). */
28  refinement?: ZipRefinement;
29}
30 
31export type Signature = {
32  mime: string;
33  /** Bytes that must match at `at`. */
34  bytes: readonly number[];
35  /** Offset of `bytes` in the buffer (default 0). */
36  at?: number;
37};
38 
39/**
40 * Leading-byte magic table. Order matters: more specific prefixes first.
41 * (webp/avif/heic all start with `RIFF`/`ftyp` boxes — the box payload at a
42 * fixed offset disambiguates them.)
43 */
44export const MIME_SIGNATURES: readonly Signature[] = [
45  { mime: "image/png", bytes: [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a] },
46  { mime: "image/jpeg", bytes: [0xff, 0xd8, 0xff] },
47  { mime: "image/gif", bytes: [0x47, 0x49, 0x46, 0x38] }, // "GIF8"
48  { mime: "image/webp", bytes: [0x57, 0x45, 0x42, 0x50], at: 8 }, // "RIFF....WEBP"
49  // ISO-BMFF family: all of mp4/avif/heic are `ftyp` boxes — the BRAND at
50  // offset 8 disambiguates. Checked BEFORE the generic mp4 signature.
51  { mime: "image/avif", bytes: [0x61, 0x76, 0x69, 0x66], at: 8 }, // ftyp + "avif"
52  { mime: "image/heic", bytes: [0x6d, 0x69, 0x66, 0x31], at: 8 }, // ftyp + "mif1"
53  { mime: "audio/mp4", bytes: [0x4d, 0x34, 0x41, 0x20], at: 8 }, // ftyp + "M4A " (m4a)
54  { mime: "video/quicktime", bytes: [0x71, 0x74, 0x20, 0x20], at: 8 }, // ftyp + "qt  " (mov)
55  { mime: "application/pdf", bytes: [0x25, 0x50, 0x44, 0x46] }, // "%PDF"
56  { mime: "video/mp4", bytes: [0x66, 0x74, 0x79, 0x70], at: 4 }, // "....ftyp"
57  { mime: "video/webm", bytes: [0x1a, 0x45, 0xdf, 0xa3] }, // EBML header
58  { mime: "audio/ogg", bytes: [0x4f, 0x67, 0x67, 0x53] }, // "OggS"
59  { mime: "audio/mpeg", bytes: [0x49, 0x44, 0x33] }, // "ID3" (ID3 tag)
60  { mime: "audio/mpeg", bytes: [0xff, 0xfb] }, // MPEG audio frame sync
61  { mime: "audio/wav", bytes: [0x57, 0x41, 0x56, 0x45], at: 8 }, // "RIFF....WAVE"
62  { mime: "application/zip", bytes: [0x50, 0x4b, 0x03, 0x04] }, // local file header
63];
64 
65/**
66 * ~60 common extensions → MIME (fallback when the magic is inconclusive).
67 * The OOXML family maps to the refined types; bare `.zip` stays generic.
68 */
69export const EXTENSION_MIME: Record<string, string> = {
70  // images
71  png: "image/png",
72  jpg: "image/jpeg",
73  jpeg: "image/jpeg",
74  gif: "image/gif",
75  webp: "image/webp",
76  avif: "image/avif",
77  heic: "image/heic",
78  heif: "image/heic",
79  svg: "image/svg+xml",
80  bmp: "image/bmp",
81  ico: "image/x-icon",
82  tif: "image/tiff",
83  tiff: "image/tiff",
84  tga: "image/targa",
85  // video
86  mp4: "video/mp4",
87  m4v: "video/mp4",
88  webm: "video/webm",
89  mov: "video/quicktime",
90  avi: "video/x-msvideo",
91  mkv: "video/x-matroska",
92  // audio
93  mp3: "audio/mpeg",
94  m4a: "audio/mp4",
95  ogg: "audio/ogg",
96  oga: "audio/ogg",
97  opus: "audio/ogg",
98  wav: "audio/wav",
99  flac: "audio/flac",
100  aac: "audio/aac",
101  // documents
102  pdf: "application/pdf",
103  txt: "text/plain",
104  md: "text/markdown",
105  csv: "text/csv",
106  html: "text/html",
107  htm: "text/html",
108  xml: "application/xml",
109  json: "application/json",
110  // archives
111  zip: "application/zip",
112  gz: "application/gzip",
113  tar: "application/x-tar",
114  "7z": "application/x-7z-compressed",
115  rar: "application/vnd.rar",
116  // office
117  doc: "application/msword",
118  docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
119  xls: "application/vnd.ms-excel",
120  xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
121  ppt: "application/vnd.ms-powerpoint",
122  pptx: "application/vnd.openxmlformats-officedocument.presentationml.presentation",
123  odt: "application/vnd.oasis.opendocument.text",
124  ods: "application/vnd.oasis.opendocument.spreadsheet",
125  epub: "application/epub+zip",
126  rtf: "application/rtf",
127  // web/app
128  js: "text/javascript",
129  mjs: "text/javascript",
130  css: "text/css",
131  wasm: "application/wasm",
132  apk: "application/vnd.android.package-archive",
133  exe: "application/x-msdownload",
134  dll: "application/vnd.microsoft.dll",
135  bat: "application/x-dosexec",
136  msi: "application/x-msi-installer",
137  bin: "application/octet-stream",
138};
139 
140/** Inverse of {@link EXTENSION_MIME} (first extension wins on ties). */
141export const MIME_EXTENSION: Record<string, string> = (() => {
142  const out: Record<string, string> = {};
143  for (const [ext, mime] of Object.entries(EXTENSION_MIME)) {
144    if (!(mime in out)) out[mime] = ext;
145  }
146  return out;
147})();
148 
149/** Max bytes scanned for the text heuristic (null-free ⇒ text/plain). */
150export const TEXT_SCAN_BYTES = 8000;
151 
152function bytesAt(buf: Uint8Array, bytes: readonly number[], at: number): boolean {
153  if (at + bytes.length > buf.length) return false;
154  for (let i = 0; i < bytes.length; i++) {
155    if (buf[at + i] !== bytes[i]) return false;
156  }
157  return true;
158}
159 
160function extensionOf(filename: string): string {
161  const clean = filename.split(/[\\/]/).pop() ?? filename;
162  const dot = clean.lastIndexOf(".");
163  if (dot <= 0 || dot === clean.length - 1) return "";
164  return clean.slice(dot + 1).toLowerCase();
165}
166 
167/**
168 * Look at the start of a zip archive and tell which OOXML flavor it is.
169 * Scans local file headers (`PK\x03\x04`) until a file name is found under
170 * one of the known roots. Pure + bounded (first 1 MB, first 50 headers).
171 */
172function refineZip(buf: Uint8Array): { mime: string; refinement: ZipRefinement } | null {
173  const limit = Math.min(buf.length, 1_000_000);
174  let pos = 0;
175  for (let i = 0; i < 50 && pos + 30 <= limit; i++) {
176    if (
177      buf[pos] !== 0x50 ||
178      buf[pos + 1] !== 0x4b ||
179      buf[pos + 2] !== 0x03 ||
180      buf[pos + 3] !== 0x04
181    ) {
182      break; // no more local headers
183    }
184    const nameLen = buf[pos + 26]! | (buf[pos + 27]! << 8);
185    const extraLen = buf[pos + 28]! | (buf[pos + 29]! << 8);
186    const nameStart = pos + 30;
187    if (nameStart + nameLen > limit) break;
188    const name = new TextDecoder("utf-8", { fatal: false }).decode(
189      buf.subarray(nameStart, nameStart + nameLen),
190    );
191    if (name.startsWith("word/")) {
192      return {
193        mime: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
194        refinement: "docx",
195      };
196    }
197    if (name.startsWith("xl/")) {
198      return {
199        mime: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
200        refinement: "xlsx",
201      };
202    }
203    if (name.startsWith("ppt/")) {
204      return {
205        mime: "application/vnd.openxmlformats-officedocument.presentationml.presentation",
206        refinement: "pptx",
207      };
208    }
209    if (name.startsWith("META-INF/")) {
210      return { mime: "application/epub+zip", refinement: "epub" };
211    }
212    pos = nameStart + nameLen + extraLen;
213    // (compressed size would let us skip fully; name/extra scan is enough
214    //  for the first entries of every real OOXML file, which come first)
215  }
216  return null;
217}
218 
219/**
220 * Detect the MIME type of a buffer.
221 *
222 * Order: magic signatures → zip-container refinement → extension fallback →
223 * text heuristic (no null byte in the first 8000 bytes) → octet-stream.
224 */
225export function detectMimeType(buf: Uint8Array, filenameHint?: string): MimeDetection {
226  for (const sig of MIME_SIGNATURES) {
227    const at = sig.at ?? 0;
228    if (bytesAt(buf, sig.bytes, at)) {
229      if (sig.mime === "application/zip") {
230        const refined = refineZip(buf);
231        return {
232          mime: refined?.mime ?? "application/zip",
233          confidence: "magic",
234          isZipContainer: true,
235          ...(refined ? { refinement: refined.refinement } : {}),
236        };
237      }
238      return { mime: sig.mime, confidence: "magic" };
239    }
240  }
241 
242  if (filenameHint) {
243    const ext = extensionOf(filenameHint);
244    const byExt = ext !== "" ? EXTENSION_MIME[ext] : undefined;
245    if (byExt) return { mime: byExt, confidence: "ext" };
246  }
247 
248  // Text heuristic: any null byte in the first TEXT_SCAN_BYTES bytes ⇒ binary.
249  const scan = buf.length <= TEXT_SCAN_BYTES ? buf : buf.subarray(0, TEXT_SCAN_BYTES);
250  let isText = scan.length > 0;
251  for (let i = 0; i < scan.length; i++) {
252    if (scan[i] === 0) {
253      isText = false;
254      break;
255    }
256  }
257  if (isText) return { mime: "text/plain", confidence: "text" };
258 
259  return { mime: "application/octet-stream", confidence: "unknown" };
260}
261 
262const IMAGE_MIMES = new Set([
263  "image/png",
264  "image/jpeg",
265  "image/gif",
266  "image/webp",
267  "image/avif",
268  "image/heic",
269  "image/svg+xml",
270  "image/bmp",
271  "image/x-icon",
272]);
273 
274const VIDEO_MIMES = new Set([
275  "video/mp4",
276  "video/webm",
277  "video/quicktime",
278  "video/x-msvideo",
279  "video/x-matroska",
280  "video/ogg",
281]);
282 
283const AUDIO_MIMES = new Set([
284  "audio/mpeg",
285  "audio/mp4",
286  "audio/ogg",
287  "audio/wav",
288  "audio/flac",
289  "audio/aac",
290]);
291 
292const DOCUMENT_MIMES = new Set([
293  "application/pdf",
294  "application/msword",
295  "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
296  "application/vnd.ms-excel",
297  "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
298  "application/vnd.ms-powerpoint",
299  "application/vnd.openxmlformats-officedocument.presentationml.presentation",
300  "application/vnd.oasis.opendocument.text",
301  "application/vnd.oasis.opendocument.spreadsheet",
302  "application/epub+zip",
303  "application/rtf",
304  "text/plain",
305  "text/markdown",
306  "text/csv",
307  "text/html",
308  "text/javascript",
309  "text/css",
310  "application/json",
311  "application/xml",
312]);
313 
314export function isImageMime(mime: string): boolean {
315  return IMAGE_MIMES.has(mime);
316}
317 
318export function isVideoMime(mime: string): boolean {
319  return VIDEO_MIMES.has(mime);
320}
321 
322export function isAudioMime(mime: string): boolean {
323  return AUDIO_MIMES.has(mime);
324}
325 
326export function isDocumentMime(mime: string): boolean {
327  return DOCUMENT_MIMES.has(mime);
328}
329 
330/** MIME types that are never allowed to be uploaded (executables/binaries). */
331const BLOCKED_MIMES = new Set([
332  "application/x-msdownload",
333  "application/vnd.microsoft.dll",
334  "application/x-msdos-program",
335  "application/x-executable",
336  "application/x-sharedlib",
337  "application/vnd.microsoft.portable-executable",
338  "application/x-dosexec",
339]);
340 
341/** Extensions that map to blocked content regardless of the claimed MIME. */
342const BLOCKED_EXTENSIONS = new Set(["exe", "dll", "bat", "sh", "app", "msi", "com", "scr"]);
343 
344/** Upload policy input. */
345export interface AllowedUploadOptions {
346  /** Max size in bytes — defaults to LIMITS.MAX_FILE_SIZE (500 MB). */
347  maxBytes?: number;
348  /** Actual (or claimed) size in bytes, when known — enforced against maxBytes. */
349  size?: number;
350  /** File name, when known — extension is checked against the blocklist. */
351  filename?: string;
352}
353 
354export interface UploadPolicyResult {
355  allowed: boolean;
356  /** Machine reason when blocked (safe to log; not for end users verbatim). */
357  reason?: "executable" | "too_large";
358}
359 
360/**
361 * Upload policy (runs on the client-claimed type at request time and again
362 * on the detected type after upload — untrusted input is never trusted,
363 * AGENTS.md §20).
364 *
365 * - Block executables by MIME (`application/x-msdownload`, …) or extension
366 *   (`.exe/.dll/.bat/.sh/.app`, …) — the classic upload RCE vector.
367 * - Enforce `size ≤ maxBytes` when a size is given
368 *   (default: LIMITS.MAX_FILE_SIZE = 500 MB).
369 */
370export function isAllowedUpload(mime: string, opts: AllowedUploadOptions = {}): UploadPolicyResult {
371  const maxBytes = opts.maxBytes ?? 500 * 1024 * 1024;
372  const ext = opts.filename !== undefined ? extensionOf(opts.filename) : "";
373  if (BLOCKED_MIMES.has(mime) || BLOCKED_EXTENSIONS.has(ext)) {
374    return { allowed: false, reason: "executable" };
375  }
376  if (opts.size !== undefined && opts.size > maxBytes) {
377    return { allowed: false, reason: "too_large" };
378  }
379  return { allowed: true };
380}
381 
382// ── TASK-0101 upload allow-list + content-type policy (spec §4) ─────────────
383 
384/**
385 * The CLOSED upload allow-list (TASK-0101 Requirement 4). Anything outside
386 * this set is rejected at session-creation time — including types the
387 * blocklist above would not catch.
388 *
389 *  - images:    png / jpg / webp / gif / svg
390 *  - documents: pdf / docx / xlsx / pptx / md / txt / csv
391 *  - archives:  zip
392 *  - video:     mp4 / webm / mov
393 *  - audio:     mp3 / wav / m4a
394 *  - other:     application/octet-stream (binary blobs)
395 */
396export const UPLOAD_ALLOWED_MIMES: ReadonlySet<string> = new Set([
397  // images
398  "image/png",
399  "image/jpeg",
400  "image/webp",
401  "image/gif",
402  "image/svg+xml",
403  // documents
404  "application/pdf",
405  "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
406  "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
407  "application/vnd.openxmlformats-officedocument.presentationml.presentation",
408  "text/markdown",
409  "text/plain",
410  "text/csv",
411  // archives
412  "application/zip",
413  // video
414  "video/mp4",
415  "video/webm",
416  "video/quicktime",
417  // audio
418  "audio/mpeg",
419  "audio/wav",
420  "audio/mp4",
421  // other
422  "application/octet-stream",
423]);
424 
425/**
426 * `isAllowedMime` — the closed allow-list check for the CLIENT-CLAIMED
427 * `mimeType` at session-creation time (TASK-0101 Requirement 4).
428 */
429export function isAllowedMime(mime: string): boolean {
430  return UPLOAD_ALLOWED_MIMES.has(mime);
431}
432 
433/**
434 * `MIME_SNIFF_TABLE` — the magic-byte signatures used by
435 * {@link sniffMagicBytes} (a subset of {@link MIME_SIGNATURES}; the zip entry
436 * is present, OOXML refinement happens through `detectMimeType` at the
437 * metadata step). Order matters — more specific prefixes first.
438 */
439export const MIME_SNIFF_TABLE: readonly Signature[] = MIME_SIGNATURES;
440 
441/**
442 * `CONTENT_TYPE_OVERRIDES` — per-detected-MIME response-content-type
443 * overrides applied to SIGNED URLs (TASK-0101 Requirement 4, Security):
444 *
445 *  - SVG is served as `text/plain` with `Content-Disposition: attachment`
446 *    + `X-Content-Type-Options: nosniff` so a browser never renders it as
447 *    XML/JS (classic stored-XSS vector).
448 *  - everything else is served with its true detected MIME.
449 */
450export const CONTENT_TYPE_OVERRIDES: Record<string, { contentType: string; disposition: "inline" | "attachment" }> =
451  {
452    "image/svg+xml": { contentType: "text/plain", disposition: "attachment" },
453  };
454 
455/**
456 * `sniffMagicBytes` — run the magic-byte table over the leading bytes of the
457 * stored object. Returns the first matching MIME, or `null` when no
458 * signature matches (text-family content has no magic — the caller decides
459 * text vs octet via `detectMimeType`, which also applies the extension +
460 * text heuristics).
461 *
462 * This is the completion-time verifier (TASK-0101 Requirement 4): the
463 * detected type is compared against the client-claimed type.
464 */
465export function sniffMagicBytes(buffer: Uint8Array): string | null {
466  for (const sig of MIME_SIGNATURES) {
467    if (bytesAt(buffer, sig.bytes, sig.at ?? 0)) return sig.mime;
468  }
469  return null;
470}
471 
472/**
473 * `sniffVerdict` — the AUTHORITY for the completion-time MIME check
474 * (TASK-0101 Requirement 4, Security: "sniffing is authoritative over the
475 * client claim; mismatch = failure, never 'trust and warn'").
476 *
477 * Verdict rules:
478 *  - the CLAIMED type has no magic signature (text family: text/plain,
479 *    markdown, csv, svg — and `application/octet-stream`) → `"no-signal"`:
480 *    no leading bytes can prove a mismatch, so the claim stands (the
481 *    worker re-sniffs at the metadata step and is the final authority on
482 *    the stored mime).
483 *  - the claimed type HAS a signature → the bytes must match it (or a
484 *    same-artifact alias): PNG claiming `image/png` → `"ok"`; TEXT claiming
485 *    `image/png` → `"mismatch"` (signature-bearing claim, bytes match no
486 *    signature); ZIP claiming `image/png` → `"mismatch"`.
487 *
488 * `"mismatch"` ⇒ the caller must mark the file FAILED (`MIME_MISMATCH`) and
489 * delete the object.
490 */
491export type SniffVerdict = "ok" | "no-signal" | "mismatch";
492 
493/** Same-artifact aliases within the ISO-BMFF (ftyp) family: the sniff table
494 *  returns the most specific brand (`qt  ` → video/quicktime, `M4A ` →
495 *  audio/mp4); a client claiming the generic `video/mp4` for the same bytes
496 *  is not a mismatch. */
497const SNIFF_FAMILY_ALIASES: Record<string, string[]> = {
498  "video/quicktime": ["video/mp4"],
499  "audio/mp4": ["video/mp4"],
500};
501 
502export function sniffVerdict(buffer: Uint8Array, claimed: string): SniffVerdict {
503  const claimHasSignature = MIME_SIGNATURES.some((s) => s.mime === claimed);
504  if (!claimHasSignature) return "no-signal";
505  const sniffed = sniffMagicBytes(buffer);
506  if (sniffed === null) return "mismatch"; // signature-bearing claim, no match
507  if (sniffed === claimed) return "ok";
508  if ((SNIFF_FAMILY_ALIASES[sniffed] ?? []).includes(claimed)) return "ok";
509  return "mismatch";
510}
511