/** * TASK-0134 — formula-engine fuzz (ASVS L2, injection / resource-exhaustion). * * 10,000 random expressions over the engine's OWN grammar (nesting depth * 1–40 — deliberately above the parser's MAX_NESTING of 16 to exercise its * depth guard — random property names incl. nonexistent, random function * trees over the Phase-6 closed function set, a node budget of 2000 so no * generated string can exhaust memory before the engine sees it). * * Generator design (recorded): a deterministic, SEEDED grammar generator * (xorshift32) driven by `fc.property` seed tuples — NOT a fast-check * recursive arbitrary tree. A recursive arbitrary at depth 40 is * exponential in structure size (branching ≈ arity per level) and OOMed the * module at collection; the PRNG builder is linear in node count and fully * reproducible given the seed. `fc.assert` still drives 10k runs, shrinking, * and the seed knob (`SEC_FUZZ_SEED=random`, nightly). * * Assertions (spec): * - **no crash**: every parse/eval either succeeds or throws FormulaError * (only FormulaError may escape — an uncaught TypeError/RangeError is a * crash and fails the suite); * - **no unbounded time / memory**: the whole 10k run stays far under the * test timeout (a hang would blow it); each eval is inside the engine's * own 100k-op / 50 ms budgets by construction (node budget 2000); * - **worst-case family**: a deliberately oversized expression (100.5k-node * `and(true, …)`) MUST hit the op limit with `FORMULA_TIMEOUT` — the * guard is asserted by error type, not "didn't crash". */ import { describe, expect, it } from "vitest"; import fc from "fast-check"; import { parseFormula, evaluateFormula, FormulaError, EVAL_OP_LIMIT, EVAL_TIMEOUT_MS } from "@factory/database-engine/formula"; import type { FormulaContext } from "@factory/database-engine/formula"; // --------------------------------------------------------------------------- // Seed: fixed for CI (10k reproducible runs); random in nightly. // --------------------------------------------------------------------------- const BASE_SEED = process.env.SEC_FUZZ_SEED === "random" ? (Date.now() & 0xffff) : 0x134_f00d; // --------------------------------------------------------------------------- // The Phase-6 closed function table — the generator draws from the SAME set // the engine resolves against (packages/database-engine/src/formula/ // functions.ts). Arity is fixed per name. // --------------------------------------------------------------------------- const FUNC_ARITY: Record = { if: 3, and: 3, or: 3, not: 1, sum: 1, average: 1, min: 1, max: 1, dateAdd: 3, dateBetween: 2, now: 0, concat: 3, replace: 3, contains: 2, length: 1, prop: 1 }; const FUNC_NAMES = Object.keys(FUNC_ARITY); const KNOWN_PROPS: Record = { Amount: 42, Price: 19.99, Status: "Done", Title: "Hello", Note: "a string", DueDate: "2025-06-15", Started: "2025-06-01", Active: true, Tags: ["a", "b"] }; const KNOWN_TYPES: Record = { Amount: "number", Price: "number", Status: "text", Title: "title", Note: "text", DueDate: "date", Started: "date", Active: "checkbox", Tags: "multi_select" }; const PROP_NAMES = Object.keys(KNOWN_PROPS); const UNKNOWN_PROPS = ["Statu", "Quantiti", "Duedate", "Isdone", "Xyz", "amount2"]; const OPS = ["+", "-", "*", "/", "==", "!=", "<", "<=", ">", ">=", "and", "or"] as const; const STR_LITS = ['"a"', '"b"', '"Done"', '"x"', '"Open"'] as const; const DATE_LITS = ['"2025-06-01"', '"2025-07-01"', '"2025-06-15"', "now()"] as const; const PROP_ALPHA = "abcdefghijklmnopqrstuvwxyz0123456789"; // --------------------------------------------------------------------------- // Deterministic seeded generator. xorshift32 (seed must be nonzero). // --------------------------------------------------------------------------- function makeRng(seed: number): () => number { let s = (seed >>> 0) || 0x9e3779b9; return () => { s ^= s << 13; s >>>= 0; s ^= s >> 17; s ^= s << 5; s >>>= 0; return s / 0x100000000; }; } function pick(rng: () => number, arr: readonly T[]): T { return arr[Math.floor(rng() * arr.length)]!; } /** A safe property-name string (alphanumeric only — never breaks a literal). */ function propName(rng: () => number): string { const r = rng(); if (r < 0.5) return pick(rng, PROP_NAMES); if (r < 0.8) return pick(rng, UNKNOWN_PROPS); const n = 1 + Math.floor(rng() * 8); let out = ""; for (let i = 0; i < n; i++) out += PROP_ALPHA[Math.floor(rng() * PROP_ALPHA.length)]!; return out; } function leaf(rng: () => number): string { const r = rng(); if (r < 0.3) return String(Math.floor(rng() * 2001) - 1000); if (r < 0.42) return rng() < 0.5 ? "true" : "false"; if (r < 0.5) return "null"; if (r < 0.64) return pick(rng, STR_LITS); if (r < 0.76) return pick(rng, DATE_LITS); return `prop("${propName(rng)}")`; } /** * Recursive grammar build. `depth` is a hard cap (top level draws 1–40); * `budget` is a node-count cap (2000) so no generated string can exhaust * memory before the engine parses it — the budget path is covered by the * dedicated worst-case family tests, not by the fuzzer's own strings. */ function genExpr(rng: () => number, depth: number, budget: { left: number }): string { budget.left--; if (depth <= 1 || budget.left <= 0) return leaf(rng); const r = rng(); if (r < 0.45) return leaf(rng); if (r < 0.8) { const l = genExpr(rng, depth - 1, budget); const op = pick(rng, OPS); const rr = genExpr(rng, depth - 1, budget); return `(${l} ${op} ${rr})`; } const name = pick(rng, FUNC_NAMES); const arity = FUNC_ARITY[name]!; if (arity === 0) return `${name}()`; const args: string[] = []; for (let i = 0; i < arity; i++) { args.push(name === "prop" ? `prop("${propName(rng)}")` : genExpr(rng, depth - 1, budget)); } return `${name}(${args.join(", ")})`; } function ctxFor(): FormulaContext { return { now: Date.parse("2025-06-15T12:00:00Z"), properties: { ...KNOWN_PROPS }, propertyTypes: { ...KNOWN_TYPES } }; } const ALLOWED_CODES = new Set(["PARSE", "TYPE", "RUNTIME", "UNKNOWN_FUNCTION", "FORMULA_TIMEOUT"]); function assertFormulaError(e: unknown, where: string, src: string): void { if (!(e instanceof FormulaError)) { throw new Error( `[formula-fuzz] ${where} threw a non-FormulaError (a crash — the engine must not leak ` + `TypeError/RangeError): ${e instanceof Error ? e.name + ": " + e.message : String(e)} | src=${src}` ); } if (!ALLOWED_CODES.has(e.code)) { throw new Error(`[formula-fuzz] unexpected FormulaError code "${e.code}" | src=${src}`); } } describe("TASK-0134 formula fuzz (10k expressions, no crash / no hang)", () => { it("10,000 random expressions: only FormulaError may escape, always bounded", () => { const t0 = Date.now(); let ok = 0; let typedErrors = 0; let timeouts = 0; let parseErrors = 0; let maxMs = 0; fc.assert( fc.property( fc.tuple(fc.nat(), fc.nat(), fc.nat()), ([a, b, c]) => { const rng = makeRng(((a ^ Math.imul(b, 0x9e3779b9) ^ Math.imul(c, 0x85ebca6b)) >>> 0) | BASE_SEED); const depth = 1 + (a % 40); const src = genExpr(rng, depth, { left: 2000 }); const perEvalT0 = Date.now(); let ast: unknown; try { ast = parseFormula(src); } catch (e) { assertFormulaError(e, "parse", src); if (e instanceof FormulaError && e.code === "PARSE") parseErrors++; typedErrors++; maxMs = Math.max(maxMs, Date.now() - perEvalT0); return; } try { const r = evaluateFormula(ast as never, ctxFor()); expect(typeof r.steps).toBe("number"); ok++; } catch (e) { assertFormulaError(e, "eval", src); typedErrors++; if (e instanceof FormulaError && e.code === "FORMULA_TIMEOUT") timeouts++; } maxMs = Math.max(maxMs, Date.now() - perEvalT0); } ), { numRuns: 10_000, seed: BASE_SEED, timeout: 120_000 } ); const totalMs = Date.now() - t0; // The whole run must stay far under the 2-min test timeout (a hang blows // it). ~1–2 s typical for 10k bounded evals. expect(totalMs).toBeLessThan(120_000); // The generator must actually exercise the error paths — including the // parser's depth guard (depths 17–40 → PARSE), so parseErrors > 0 is a // liveness check that the depth knob is being used. expect(ok + typedErrors).toBe(10_000); expect(typedErrors).toBeGreaterThan(0); expect(parseErrors).toBeGreaterThan(0); console.log( `[formula-fuzz] baseSeed=${BASE_SEED} ok=${ok} typedErrors=${typedErrors} (parse=${parseErrors}, timeouts=${timeouts}) maxEvalMs=${maxMs} totalMs=${totalMs}` ); }); it("worst-case family: oversized expression hits FORMULA_TIMEOUT (op limit), never hangs", () => { // The documented worst case: a wide `and(true, …)` whose node count // exceeds EVAL_OP_LIMIT. The guard must reject it as a typed // FORMULA_TIMEOUT, not hang or throw an uncaught error. A FROZEN clock // isolates the op budget from the wall-clock budget. const N = EVAL_OP_LIMIT + 500; let src = "and("; for (let i = 0; i < N; i++) src += "true, "; src = src.slice(0, -2) + ")"; const t0 = Date.now(); let caught: unknown; try { evaluateFormula(parseFormula(src) as never, ctxFor(), { now: () => 0 }); } catch (e) { caught = e; } const elapsed = Date.now() - t0; expect(caught).toBeInstanceOf(FormulaError); expect((caught as FormulaError).code).toBe("FORMULA_TIMEOUT"); expect((caught as FormulaError).message).toMatch(/op limit/i); expect(elapsed).toBeLessThan(2_000); // never hangs // The wall-clock budget is the OTHER guard (cheap-but-slow): assert the // engine enforces the documented constants. expect(EVAL_TIMEOUT_MS).toBe(50); expect(EVAL_OP_LIMIT).toBe(100_000); }); it("wide concat family (many prop refs) is bounded by the output/op guard", () => { // The spec's named pathological family: a wide `concat(prop(…), …)` chain // that builds a large output. Depth is capped by the parser (16), so the // realistic worst case is a WIDE concat — the 1MB output guard (RUNTIME) // or the op limit (FORMULA_TIMEOUT) must fire, never an unbounded string // build. 200k args → ~200k node visits → op limit. const parts: string[] = []; for (let i = 0; i < 200_000; i++) parts.push(`prop("Note")`); const src = `concat(${parts.join(", ")})`; const t0 = Date.now(); let caught: unknown; let value: unknown; try { const r = evaluateFormula(parseFormula(src) as never, ctxFor(), { now: () => 0 }); value = r.value.v; } catch (e) { caught = e; } const elapsed = Date.now() - t0; if (caught !== undefined) { expect(caught).toBeInstanceOf(FormulaError); expect(ALLOWED_CODES.has((caught as FormulaError).code)).toBe(true); } else { expect(typeof value).toBe("string"); } expect(elapsed).toBeLessThan(2_000); }); });