/** * apps/worker — minimal S3-compatible client (NO aws-sdk). * * Implements the standard AWS Signature Version 4 flow directly with * node:crypto + fetch: * * 1. canonical request (method, URI, query, headers, signed headers, * payload hash) * 2. string to sign (algorithm, amz-date, scope, hash(canonical)) * 3. signing key (HMAC chain: date → region → service → request) * 4. signature (hex HMAC of string-to-sign with signing key) * * Scope: path-style addressing only (S3_ENDPOINT + /bucket/key), the shapes * needed by the worker pipeline: * headObject, getObjectStream (small files < 50MB — buffers in memory), * putObject, deleteObject, presignedGet, presignedPut. * * The canonical/signing functions are exported separately so unit tests can * pin the exact signature output against known inputs (see * src/__tests__/s3-sign.test.ts). */ import { createHash, createHmac, timingSafeEqual } from "node:crypto"; export interface S3Config { /** e.g. http://localhost:9000 — path-style, no trailing slash. */ endpoint: string; region: string; bucket: string; accessKey: string; secretKey: string; /** Public base for browser-facing URLs (MinIO console / public bucket). */ publicBase?: string; /** Injectable for tests (defaults to global fetch). */ fetchImpl?: typeof fetch; } export interface S3HeadResult { size: number; contentType: string; etag: string; lastModified: Date; } export class S3Error extends Error { constructor( public code: string, message: string, public statusCode: number ) { super(message); this.name = "S3Error"; } } // --------------------------------------------------------------------------- // SigV4 primitives (exported for unit testing) // --------------------------------------------------------------------------- export const AWS_SHA256_HEX = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; const UNRESERVED = /^[A-Za-z0-9-_.~]+$/; /** RFC 3986 percent-encoding (S3 "strict" encoding: unreserved set kept). */ export function s3Encode(value: string): string { let out = ""; const bytes = new TextEncoder().encode(value); for (const b of bytes) { const c = String.fromCharCode(b); if (UNRESERVED.test(c)) out += c; else out += "%" + b.toString(16).toUpperCase().padStart(2, "0"); } return out; } /** Encode an object key for use in the URI path (keeps '/' separators). */ export function encodeKey(key: string): string { return key.split("/").map(s3Encode).join("/"); } export interface CanonicalRequestInput { method: string; /** URI-encoded path (key already encoded via encodeKey). */ path: string; /** Query params, unsorted (will be canonically sorted). */ query?: Record; /** Header names — normalized to lowercase inside {@link canonicalRequest} * (AWS SigV4 requires lowercase names in the canonical request; the * function is the single normalization point, callers may pass mixed case). * Values trimmed. */ headers: Record; /** sha256 hex of the payload (or UNSIGNED-PAYLOAD for presigned). */ payloadHash: string; } /** Build the AWS SigV4 canonical request string. */ export function canonicalRequest(input: CanonicalRequestInput): string { const queryLines = Object.entries(input.query ?? {}) .map(([k, v]) => [s3Encode(k), s3Encode(v)] as const) .sort((a, b) => (a[0] === b[0] ? a[1].localeCompare(b[1]) : a[0].localeCompare(b[0]))) .map(([k, v]) => `${k}=${v}`) .join("&"); const normalized: Record = {}; for (const [k, v] of Object.entries(input.headers)) normalized[k.toLowerCase()] = v; const sortedNames = Object.keys(normalized).sort(); const headerBlock = sortedNames.map((n) => `${n}:${normalized[n]!}`).join("\n"); const signedHeaders = sortedNames.join(";"); return [ input.method.toUpperCase(), input.path, queryLines, headerBlock + "\n", signedHeaders, input.payloadHash ].join("\n"); } export interface SigningInput { method: string; host: string; path: string; query?: Record; headers?: Record; payloadHash: string; region: string; service: string; // "s3" accessKey: string; secretKey: string; amzDate?: string; // ISO without ms? We use YYYYMMDDTHHMMSSZ (UTC) accessDate?: string; // YYYYMMDD /** * Whether `x-amz-date` is a SIGNED HEADER (default, header-signed CRUD * requests). Presigned (query-signed) requests must pass `false`: there * `x-amz-date` travels as the `X-Amz-Date` QUERY PARAM, so it must NOT be in * the canonical header block or `X-Amz-SignedHeaders` — signing it anyway * makes the declared set (`host`) disagree with the signed set * (`host;x-amz-date`) and MinIO rejects with `SignatureDoesNotMatch`. */ signAmzDateHeader?: boolean; } function hmac(key: Buffer | string, data: string): Buffer { return createHmac("sha256", key).update(data, "utf8").digest(); } export function deriveSigningKey( secretKey: string, date: string, region: string, service: string ): Buffer { const kDate = hmac("AWS4" + secretKey, date); const kRegion = hmac(kDate, region); const kService = hmac(kRegion, service); return hmac(kService, "aws4_request"); } /** * Compute the full SigV4 signature for a request. Returns the signature hex * plus the credential scope — the pieces callers need for the Authorization * header or the X-Amz-Signature query parameter. */ export function computeSigV4Signature(input: SigningInput): { signature: string; credentialScope: string; stringToSign: string; } { const amzDate = input.amzDate ?? toAmzDate(new Date()); const accessDate = input.accessDate ?? amzDate.slice(0, 8); const headers: Record = { host: input.host, ...(input.signAmzDateHeader === false ? {} : { "x-amz-date": amzDate }), ...(input.headers ?? {}) }; const canonical = canonicalRequest({ method: input.method, path: input.path, ...(input.query ? { query: input.query } : {}), headers, payloadHash: input.payloadHash }); const credentialScope = `${accessDate}/${input.region}/${input.service}/aws4_request`; const stringToSign = [ "AWS4-HMAC-SHA256", amzDate, credentialScope, sha256Hex(canonical) ].join("\n"); const signingKey = deriveSigningKey( input.secretKey, accessDate, input.region, input.service ); const signature = hmac(signingKey, stringToSign).toString("hex"); return { signature, credentialScope, stringToSign }; } export function sha256Hex(data: Buffer | string): string { return createHash("sha256").update(data).digest("hex"); } /** 20250101T123000Z — the SigV4 timestamp format. */ export function toAmzDate(d: Date): string { return d.toISOString().replace(/[-:]/g, "").replace(/\.\d{3}Z$/, "Z"); } // --------------------------------------------------------------------------- // Client // --------------------------------------------------------------------------- export class S3Client { private cfg: S3Config; private host: string; private origin: string; private doFetch: typeof fetch; constructor(cfg: S3Config) { this.cfg = cfg; const u = new URL(cfg.endpoint); this.host = u.host; // host:port — matches the host header we sign this.origin = u.protocol + "//" + u.host; this.doFetch = cfg.fetchImpl ?? fetch; } /** HEAD /bucket/key → metadata, or null when the key does not exist. */ async headObject(key: string): Promise { const path = `/${this.cfg.bucket}/${encodeKey(key)}`; const res = await this.signedRequest("HEAD", path, undefined, null); if (res.status === 404) return null; if (!res.ok) { throw this.errorFrom(res, `headObject(${key})`); } const size = Number(res.headers.get("content-length") ?? 0); const last = res.headers.get("last-modified"); return { size, contentType: res.headers.get("content-type") ?? "application/octet-stream", etag: (res.headers.get("etag") ?? "").replace(/^"|"$/g, ""), lastModified: last ? new Date(last) : new Date(0) }; } /** * GET /bucket/key → full body as a Buffer. * NOTE: buffers the whole object in memory — the files processor caps this * at 100MB. Large-file streaming needs range requests (known gap). */ async getObjectStream(key: string): Promise { const path = `/${this.cfg.bucket}/${encodeKey(key)}`; const res = await this.signedRequest("GET", path, undefined, null); if (!res.ok) throw this.errorFrom(res, `getObject(${key})`); return Buffer.from(await res.arrayBuffer()); } /** * GET /bucket/key → stream the body to `destPath`, computing the sha256 in * the same pass (no in-memory copy — the files pipeline uses this so a * 500 MB object is never held in RAM; TASK-0102 §4 "streaming where * possible"). Returns the byte count + hex sha256. */ async downloadTo(key: string, destPath: string): Promise<{ size: number; sha256: string }> { const path = `/${this.cfg.bucket}/${encodeKey(key)}`; const res = await this.signedRequest("GET", path, undefined, null); if (!res.ok) throw this.errorFrom(res, `downloadTo(${key})`); if (!res.body) throw this.errorFrom(res, `downloadTo(${key}) empty body`); const { createHash } = await import("node:crypto"); const { createWriteStream } = await import("node:fs"); const { Readable } = await import("node:stream"); const hash = createHash("sha256"); let size = 0; await new Promise((resolve, reject) => { const ws = createWriteStream(destPath); const body = Readable.fromWeb(res.body as Parameters[0]); body.on("data", (c: Buffer) => { hash.update(c); size += c.length; }); body.on("error", reject); ws.on("error", reject); ws.on("finish", () => resolve()); body.pipe(ws); }); return { size, sha256: hash.digest("hex") }; } async putObject( key: string, body: Buffer | string, contentType: string ): Promise { const path = `/${this.cfg.bucket}/${encodeKey(key)}`; const buf = Buffer.isBuffer(body) ? body : Buffer.from(body, "utf8"); const res = await this.signedRequest("PUT", path, { "content-type": contentType }, buf); if (!res.ok) throw this.errorFrom(res, `putObject(${key})`); } async deleteObject(key: string): Promise { const path = `/${this.cfg.bucket}/${encodeKey(key)}`; const res = await this.signedRequest("DELETE", path, undefined, null); if (!res.ok && res.status !== 404) { throw this.errorFrom(res, `deleteObject(${key})`); } } /** Short-lived GET url (query-string signed; payload = UNSIGNED-PAYLOAD). */ async presignedGet(key: string, ttlSec: number): Promise { const path = `/${this.cfg.bucket}/${encodeKey(key)}`; const url = new URL(this.origin + path); const { signature } = await this.presign("GET", path, ttlSec, url); url.searchParams.set("X-Amz-Signature", signature); return url.toString(); } /** Short-lived PUT url + the headers the browser must send. */ async presignedPut( key: string, contentType: string, ttlSec: number ): Promise<{ url: string; headers: Record }> { const path = `/${this.cfg.bucket}/${encodeKey(key)}`; const url = new URL(this.origin + path); const { signature } = await this.presign( "PUT", path, ttlSec, url, { "content-type": contentType } ); url.searchParams.set("X-Amz-Signature", signature); return { url: url.toString(), headers: { "Content-Type": contentType } }; } // -- internals ------------------------------------------------------------ private async presign( method: string, path: string, ttlSec: number, url: URL, extraHeaders?: Record ): Promise<{ signature: string; credentialScope: string }> { const now = new Date(); const amzDate = toAmzDate(now); const accessDate = amzDate.slice(0, 8); const credentialScope = `${accessDate}/${this.cfg.region}/s3/aws4_request`; const credential = `${this.cfg.accessKey}/${credentialScope}`; url.searchParams.set("X-Amz-Algorithm", "AWS4-HMAC-SHA256"); url.searchParams.set("X-Amz-Credential", credential); url.searchParams.set("X-Amz-Date", amzDate); url.searchParams.set("X-Amz-Expires", String(ttlSec)); const query: Record = {}; for (const [k, v] of url.searchParams) query[k] = v; // host always signed; extra headers (e.g. content-type) signed too. const headers: Record = { host: this.host }; for (const [k, v] of Object.entries(extraHeaders ?? {})) { headers[k.toLowerCase()] = v; } const signedNames = Object.keys(headers).sort(); url.searchParams.set("X-Amz-SignedHeaders", signedNames.join(";")); // Re-read query AFTER setting SignedHeaders (it signs the final set). const finalQuery: Record = {}; for (const [k, v] of url.searchParams) finalQuery[k] = v; const { signature } = computeSigV4Signature({ method, host: this.host, path, query: finalQuery, headers, payloadHash: "UNSIGNED-PAYLOAD", region: this.cfg.region, service: "s3", accessKey: this.cfg.accessKey, secretKey: this.cfg.secretKey, amzDate, accessDate, // Presigned: x-amz-date is the X-Amz-Date QUERY PARAM, never a signed // header (it is already in `query`/`finalQuery`). See SigningInput. signAmzDateHeader: false }); return { signature, credentialScope }; } private async signedRequest( method: string, path: string, extraHeaders: Record | undefined, body: Buffer | null ): Promise { const now = new Date(); const amzDate = toAmzDate(now); const accessDate = amzDate.slice(0, 8); const payloadHash = body ? sha256Hex(body) : AWS_SHA256_HEX; const headers: Record = { host: this.host }; // S3 requires x-amz-content-sha256 on every signed request (hash of the // empty string when there is no payload — see AWS SigV4 docs). headers["x-amz-content-sha256"] = payloadHash; // x-amz-date must be SIGNED (computeSigV4Signature includes it in the // canonical header set). If it is sent but not listed in // X-Amz-SignedHeaders, MinIO rejects with SignatureDoesNotMatch — the // CRUD path (putObject/headObject/downloadTo) 403'd on every request // until this was added (found via TASK-0102 integration, 2026-09-29). headers["x-amz-date"] = amzDate; for (const [k, v] of Object.entries(extraHeaders ?? {})) { headers[k.toLowerCase()] = v; } const { signature, credentialScope } = computeSigV4Signature({ method, host: this.host, path, headers, payloadHash, region: this.cfg.region, service: "s3", accessKey: this.cfg.accessKey, secretKey: this.cfg.secretKey, amzDate, accessDate }); const signedNames = Object.keys(headers).sort(); const finalHeaders: Record = {}; for (const n of signedNames) finalHeaders[n] = headers[n]!; finalHeaders["authorization"] = `AWS4-HMAC-SHA256 Credential=${this.cfg.accessKey}/${credentialScope}, ` + `SignedHeaders=${signedNames.join(";")}, Signature=${signature}`; const res = await this.doFetch(this.origin + path, { method, headers: finalHeaders, ...(body !== undefined ? { body } : {}), redirect: "follow" }); return res; } private errorFrom(res: Response, ctx: string): S3Error { let code = "S3_ERROR"; const message = `${ctx} failed: HTTP ${res.status}`; if (res.status === 403) code = "AccessDenied"; else if (res.status === 404) code = "NoSuchKey"; return new S3Error(code, message, res.status); } } // --------------------------------------------------------------------------- // Test helpers // --------------------------------------------------------------------------- /** Constant-time comparison (used by tests / future verification paths). */ export function safeEqual(a: string, b: string): boolean { const ba = Buffer.from(a, "utf8"); const bb = Buffer.from(b, "utf8"); if (ba.length !== bb.length) return false; return timingSafeEqual(ba, bb); }