s3.ts · Source

s3.tsapps/worker/src/lib/s3.tsraw ← lib16,649 B← Back to the app
1/**
2 * apps/worker — minimal S3-compatible client (NO aws-sdk).
3 *
4 * Implements the standard AWS Signature Version 4 flow directly with
5 * node:crypto + fetch:
6 *
7 *   1. canonical request   (method, URI, query, headers, signed headers,
8 *                           payload hash)
9 *   2. string to sign      (algorithm, amz-date, scope, hash(canonical))
10 *   3. signing key         (HMAC chain: date → region → service → request)
11 *   4. signature           (hex HMAC of string-to-sign with signing key)
12 *
13 * Scope: path-style addressing only (S3_ENDPOINT + /bucket/key), the shapes
14 * needed by the worker pipeline:
15 *   headObject, getObjectStream (small files < 50MB — buffers in memory),
16 *   putObject, deleteObject, presignedGet, presignedPut.
17 *
18 * The canonical/signing functions are exported separately so unit tests can
19 * pin the exact signature output against known inputs (see
20 * src/__tests__/s3-sign.test.ts).
21 */
22import { createHash, createHmac, timingSafeEqual } from "node:crypto";
23 
24export interface S3Config {
25  /** e.g. http://localhost:9000 — path-style, no trailing slash. */
26  endpoint: string;
27  region: string;
28  bucket: string;
29  accessKey: string;
30  secretKey: string;
31  /** Public base for browser-facing URLs (MinIO console / public bucket). */
32  publicBase?: string;
33  /** Injectable for tests (defaults to global fetch). */
34  fetchImpl?: typeof fetch;
35}
36 
37export interface S3HeadResult {
38  size: number;
39  contentType: string;
40  etag: string;
41  lastModified: Date;
42}
43 
44export class S3Error extends Error {
45  constructor(
46    public code: string,
47    message: string,
48    public statusCode: number
49  ) {
50    super(message);
51    this.name = "S3Error";
52  }
53}
54 
55// ---------------------------------------------------------------------------
56// SigV4 primitives (exported for unit testing)
57// ---------------------------------------------------------------------------
58 
59export const AWS_SHA256_HEX =
60  "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
61 
62const UNRESERVED = /^[A-Za-z0-9-_.~]+$/;
63 
64/** RFC 3986 percent-encoding (S3 "strict" encoding: unreserved set kept). */
65export function s3Encode(value: string): string {
66  let out = "";
67  const bytes = new TextEncoder().encode(value);
68  for (const b of bytes) {
69    const c = String.fromCharCode(b);
70    if (UNRESERVED.test(c)) out += c;
71    else out += "%" + b.toString(16).toUpperCase().padStart(2, "0");
72  }
73  return out;
74}
75 
76/** Encode an object key for use in the URI path (keeps '/' separators). */
77export function encodeKey(key: string): string {
78  return key.split("/").map(s3Encode).join("/");
79}
80 
81export interface CanonicalRequestInput {
82  method: string;
83  /** URI-encoded path (key already encoded via encodeKey). */
84  path: string;
85  /** Query params, unsorted (will be canonically sorted). */
86  query?: Record<string, string>;
87  /** Header names — normalized to lowercase inside {@link canonicalRequest}
88   * (AWS SigV4 requires lowercase names in the canonical request; the
89   * function is the single normalization point, callers may pass mixed case).
90   * Values trimmed. */
91  headers: Record<string, string>;
92  /** sha256 hex of the payload (or UNSIGNED-PAYLOAD for presigned). */
93  payloadHash: string;
94}
95 
96/** Build the AWS SigV4 canonical request string. */
97export function canonicalRequest(input: CanonicalRequestInput): string {
98  const queryLines = Object.entries(input.query ?? {})
99    .map(([k, v]) => [s3Encode(k), s3Encode(v)] as const)
100    .sort((a, b) => (a[0] === b[0] ? a[1].localeCompare(b[1]) : a[0].localeCompare(b[0])))
101    .map(([k, v]) => `${k}=${v}`)
102    .join("&");
103  const normalized: Record<string, string> = {};
104  for (const [k, v] of Object.entries(input.headers)) normalized[k.toLowerCase()] = v;
105  const sortedNames = Object.keys(normalized).sort();
106  const headerBlock = sortedNames.map((n) => `${n}:${normalized[n]!}`).join("\n");
107  const signedHeaders = sortedNames.join(";");
108  return [
109    input.method.toUpperCase(),
110    input.path,
111    queryLines,
112    headerBlock + "\n",
113    signedHeaders,
114    input.payloadHash
115  ].join("\n");
116}
117 
118export interface SigningInput {
119  method: string;
120  host: string;
121  path: string;
122  query?: Record<string, string>;
123  headers?: Record<string, string>;
124  payloadHash: string;
125  region: string;
126  service: string; // "s3"
127  accessKey: string;
128  secretKey: string;
129  amzDate?: string; // ISO without ms? We use YYYYMMDDTHHMMSSZ (UTC)
130  accessDate?: string; // YYYYMMDD
131  /**
132   * Whether `x-amz-date` is a SIGNED HEADER (default, header-signed CRUD
133   * requests). Presigned (query-signed) requests must pass `false`: there
134   * `x-amz-date` travels as the `X-Amz-Date` QUERY PARAM, so it must NOT be in
135   * the canonical header block or `X-Amz-SignedHeaders` — signing it anyway
136   * makes the declared set (`host`) disagree with the signed set
137   * (`host;x-amz-date`) and MinIO rejects with `SignatureDoesNotMatch`.
138   */
139  signAmzDateHeader?: boolean;
140}
141 
142function hmac(key: Buffer | string, data: string): Buffer {
143  return createHmac("sha256", key).update(data, "utf8").digest();
144}
145 
146export function deriveSigningKey(
147  secretKey: string,
148  date: string,
149  region: string,
150  service: string
151): Buffer {
152  const kDate = hmac("AWS4" + secretKey, date);
153  const kRegion = hmac(kDate, region);
154  const kService = hmac(kRegion, service);
155  return hmac(kService, "aws4_request");
156}
157 
158/**
159 * Compute the full SigV4 signature for a request. Returns the signature hex
160 * plus the credential scope — the pieces callers need for the Authorization
161 * header or the X-Amz-Signature query parameter.
162 */
163export function computeSigV4Signature(input: SigningInput): {
164  signature: string;
165  credentialScope: string;
166  stringToSign: string;
167} {
168  const amzDate = input.amzDate ?? toAmzDate(new Date());
169  const accessDate = input.accessDate ?? amzDate.slice(0, 8);
170  const headers: Record<string, string> = {
171    host: input.host,
172    ...(input.signAmzDateHeader === false
173      ? {}
174      : { "x-amz-date": amzDate }),
175    ...(input.headers ?? {})
176  };
177  const canonical = canonicalRequest({
178    method: input.method,
179    path: input.path,
180    ...(input.query ? { query: input.query } : {}),
181    headers,
182    payloadHash: input.payloadHash
183  });
184  const credentialScope = `${accessDate}/${input.region}/${input.service}/aws4_request`;
185  const stringToSign = [
186    "AWS4-HMAC-SHA256",
187    amzDate,
188    credentialScope,
189    sha256Hex(canonical)
190  ].join("\n");
191  const signingKey = deriveSigningKey(
192    input.secretKey,
193    accessDate,
194    input.region,
195    input.service
196  );
197  const signature = hmac(signingKey, stringToSign).toString("hex");
198  return { signature, credentialScope, stringToSign };
199}
200 
201export function sha256Hex(data: Buffer | string): string {
202  return createHash("sha256").update(data).digest("hex");
203}
204 
205/** 20250101T123000Z — the SigV4 timestamp format. */
206export function toAmzDate(d: Date): string {
207  return d.toISOString().replace(/[-:]/g, "").replace(/\.\d{3}Z$/, "Z");
208}
209 
210// ---------------------------------------------------------------------------
211// Client
212// ---------------------------------------------------------------------------
213 
214export class S3Client {
215  private cfg: S3Config;
216  private host: string;
217  private origin: string;
218  private doFetch: typeof fetch;
219 
220  constructor(cfg: S3Config) {
221    this.cfg = cfg;
222    const u = new URL(cfg.endpoint);
223    this.host = u.host; // host:port — matches the host header we sign
224    this.origin = u.protocol + "//" + u.host;
225    this.doFetch = cfg.fetchImpl ?? fetch;
226  }
227 
228  /** HEAD /bucket/key → metadata, or null when the key does not exist. */
229  async headObject(key: string): Promise<S3HeadResult | null> {
230    const path = `/${this.cfg.bucket}/${encodeKey(key)}`;
231    const res = await this.signedRequest("HEAD", path, undefined, null);
232    if (res.status === 404) return null;
233    if (!res.ok) {
234      throw this.errorFrom(res, `headObject(${key})`);
235    }
236    const size = Number(res.headers.get("content-length") ?? 0);
237    const last = res.headers.get("last-modified");
238    return {
239      size,
240      contentType: res.headers.get("content-type") ?? "application/octet-stream",
241      etag: (res.headers.get("etag") ?? "").replace(/^"|"$/g, ""),
242      lastModified: last ? new Date(last) : new Date(0)
243    };
244  }
245 
246  /**
247   * GET /bucket/key → full body as a Buffer.
248   * NOTE: buffers the whole object in memory — the files processor caps this
249   * at 100MB. Large-file streaming needs range requests (known gap).
250   */
251  async getObjectStream(key: string): Promise<Buffer> {
252    const path = `/${this.cfg.bucket}/${encodeKey(key)}`;
253    const res = await this.signedRequest("GET", path, undefined, null);
254    if (!res.ok) throw this.errorFrom(res, `getObject(${key})`);
255    return Buffer.from(await res.arrayBuffer());
256  }
257 
258  /**
259   * GET /bucket/key → stream the body to `destPath`, computing the sha256 in
260   * the same pass (no in-memory copy — the files pipeline uses this so a
261   * 500 MB object is never held in RAM; TASK-0102 §4 "streaming where
262   * possible"). Returns the byte count + hex sha256.
263   */
264  async downloadTo(key: string, destPath: string): Promise<{ size: number; sha256: string }> {
265    const path = `/${this.cfg.bucket}/${encodeKey(key)}`;
266    const res = await this.signedRequest("GET", path, undefined, null);
267    if (!res.ok) throw this.errorFrom(res, `downloadTo(${key})`);
268    if (!res.body) throw this.errorFrom(res, `downloadTo(${key}) empty body`);
269    const { createHash } = await import("node:crypto");
270    const { createWriteStream } = await import("node:fs");
271    const { Readable } = await import("node:stream");
272    const hash = createHash("sha256");
273    let size = 0;
274    await new Promise<void>((resolve, reject) => {
275      const ws = createWriteStream(destPath);
276      const body = Readable.fromWeb(res.body as Parameters<typeof Readable.fromWeb>[0]);
277      body.on("data", (c: Buffer) => {
278        hash.update(c);
279        size += c.length;
280      });
281      body.on("error", reject);
282      ws.on("error", reject);
283      ws.on("finish", () => resolve());
284      body.pipe(ws);
285    });
286    return { size, sha256: hash.digest("hex") };
287  }
288 
289  async putObject(
290    key: string,
291    body: Buffer | string,
292    contentType: string
293  ): Promise<void> {
294    const path = `/${this.cfg.bucket}/${encodeKey(key)}`;
295    const buf = Buffer.isBuffer(body) ? body : Buffer.from(body, "utf8");
296    const res = await this.signedRequest("PUT", path, {
297      "content-type": contentType
298    }, buf);
299    if (!res.ok) throw this.errorFrom(res, `putObject(${key})`);
300  }
301 
302  async deleteObject(key: string): Promise<void> {
303    const path = `/${this.cfg.bucket}/${encodeKey(key)}`;
304    const res = await this.signedRequest("DELETE", path, undefined, null);
305    if (!res.ok && res.status !== 404) {
306      throw this.errorFrom(res, `deleteObject(${key})`);
307    }
308  }
309 
310  /** Short-lived GET url (query-string signed; payload = UNSIGNED-PAYLOAD). */
311  async presignedGet(key: string, ttlSec: number): Promise<string> {
312    const path = `/${this.cfg.bucket}/${encodeKey(key)}`;
313    const url = new URL(this.origin + path);
314    const { signature } = await this.presign("GET", path, ttlSec, url);
315    url.searchParams.set("X-Amz-Signature", signature);
316    return url.toString();
317  }
318 
319  /** Short-lived PUT url + the headers the browser must send. */
320  async presignedPut(
321    key: string,
322    contentType: string,
323    ttlSec: number
324  ): Promise<{ url: string; headers: Record<string, string> }> {
325    const path = `/${this.cfg.bucket}/${encodeKey(key)}`;
326    const url = new URL(this.origin + path);
327    const { signature } = await this.presign(
328      "PUT",
329      path,
330      ttlSec,
331      url,
332      { "content-type": contentType }
333    );
334    url.searchParams.set("X-Amz-Signature", signature);
335    return { url: url.toString(), headers: { "Content-Type": contentType } };
336  }
337 
338  // -- internals ------------------------------------------------------------
339 
340  private async presign(
341    method: string,
342    path: string,
343    ttlSec: number,
344    url: URL,
345    extraHeaders?: Record<string, string>
346  ): Promise<{ signature: string; credentialScope: string }> {
347    const now = new Date();
348    const amzDate = toAmzDate(now);
349    const accessDate = amzDate.slice(0, 8);
350    const credentialScope = `${accessDate}/${this.cfg.region}/s3/aws4_request`;
351    const credential = `${this.cfg.accessKey}/${credentialScope}`;
352 
353    url.searchParams.set("X-Amz-Algorithm", "AWS4-HMAC-SHA256");
354    url.searchParams.set("X-Amz-Credential", credential);
355    url.searchParams.set("X-Amz-Date", amzDate);
356    url.searchParams.set("X-Amz-Expires", String(ttlSec));
357 
358    const query: Record<string, string> = {};
359    for (const [k, v] of url.searchParams) query[k] = v;
360 
361    // host always signed; extra headers (e.g. content-type) signed too.
362    const headers: Record<string, string> = { host: this.host };
363    for (const [k, v] of Object.entries(extraHeaders ?? {})) {
364      headers[k.toLowerCase()] = v;
365    }
366    const signedNames = Object.keys(headers).sort();
367    url.searchParams.set("X-Amz-SignedHeaders", signedNames.join(";"));
368    // Re-read query AFTER setting SignedHeaders (it signs the final set).
369    const finalQuery: Record<string, string> = {};
370    for (const [k, v] of url.searchParams) finalQuery[k] = v;
371 
372    const { signature } = computeSigV4Signature({
373      method,
374      host: this.host,
375      path,
376      query: finalQuery,
377      headers,
378      payloadHash: "UNSIGNED-PAYLOAD",
379      region: this.cfg.region,
380      service: "s3",
381      accessKey: this.cfg.accessKey,
382      secretKey: this.cfg.secretKey,
383      amzDate,
384      accessDate,
385      // Presigned: x-amz-date is the X-Amz-Date QUERY PARAM, never a signed
386      // header (it is already in `query`/`finalQuery`). See SigningInput.
387      signAmzDateHeader: false
388    });
389    return { signature, credentialScope };
390  }
391 
392  private async signedRequest(
393    method: string,
394    path: string,
395    extraHeaders: Record<string, string> | undefined,
396    body: Buffer | null
397  ): Promise<Response> {
398    const now = new Date();
399    const amzDate = toAmzDate(now);
400    const accessDate = amzDate.slice(0, 8);
401    const payloadHash = body ? sha256Hex(body) : AWS_SHA256_HEX;
402 
403    const headers: Record<string, string> = { host: this.host };
404    // S3 requires x-amz-content-sha256 on every signed request (hash of the
405    // empty string when there is no payload — see AWS SigV4 docs).
406    headers["x-amz-content-sha256"] = payloadHash;
407    // x-amz-date must be SIGNED (computeSigV4Signature includes it in the
408    // canonical header set). If it is sent but not listed in
409    // X-Amz-SignedHeaders, MinIO rejects with SignatureDoesNotMatch — the
410    // CRUD path (putObject/headObject/downloadTo) 403'd on every request
411    // until this was added (found via TASK-0102 integration, 2026-09-29).
412    headers["x-amz-date"] = amzDate;
413    for (const [k, v] of Object.entries(extraHeaders ?? {})) {
414      headers[k.toLowerCase()] = v;
415    }
416 
417    const { signature, credentialScope } = computeSigV4Signature({
418      method,
419      host: this.host,
420      path,
421      headers,
422      payloadHash,
423      region: this.cfg.region,
424      service: "s3",
425      accessKey: this.cfg.accessKey,
426      secretKey: this.cfg.secretKey,
427      amzDate,
428      accessDate
429    });
430 
431    const signedNames = Object.keys(headers).sort();
432    const finalHeaders: Record<string, string> = {};
433    for (const n of signedNames) finalHeaders[n] = headers[n]!;
434    finalHeaders["authorization"] =
435      `AWS4-HMAC-SHA256 Credential=${this.cfg.accessKey}/${credentialScope}, ` +
436      `SignedHeaders=${signedNames.join(";")}, Signature=${signature}`;
437 
438    const res = await this.doFetch(this.origin + path, {
439      method,
440      headers: finalHeaders,
441      ...(body !== undefined ? { body } : {}),
442      redirect: "follow"
443    });
444    return res;
445  }
446 
447  private errorFrom(res: Response, ctx: string): S3Error {
448    let code = "S3_ERROR";
449    const message = `${ctx} failed: HTTP ${res.status}`;
450    if (res.status === 403) code = "AccessDenied";
451    else if (res.status === 404) code = "NoSuchKey";
452    return new S3Error(code, message, res.status);
453  }
454}
455 
456// ---------------------------------------------------------------------------
457// Test helpers
458// ---------------------------------------------------------------------------
459 
460/** Constant-time comparison (used by tests / future verification paths). */
461export function safeEqual(a: string, b: string): boolean {
462  const ba = Buffer.from(a, "utf8");
463  const bb = Buffer.from(b, "utf8");
464  if (ba.length !== bb.length) return false;
465  return timingSafeEqual(ba, bb);
466}
467