/** * TASK-0134 — audit-report cross-check (the 10th suite; spec: "a CI check * (a small script in the report's companion test tests/security/audit-report. * test.ts) asserts the report exists, every finding has a status + * regression-test path, and the test path exists and passes"; "the report is * append-only for findings (status changes recorded with dates, deletions * forbidden); the report is signed"). * * This is a static contract check on docs/security/audit-report.md: * 1. the report exists and names the standard (ASVS 4.0.3, Level 2); * 2. all 16 ASVS verification requirement areas (V1–V16) are present; * 3. every L2 control table row carries a valid status * (pass | N-A | fixed | mitigated | accepted); * 4. the findings table is complete: every row has {id, asvs-ref, severity, * status ∈ fixed|mitigated|accepted|N-A, fix-task, regression-test} and * the regression-test path EXISTS (it runs green as part of the same CI * security stage); * 5. append-only: finding ids are unique, F-\d{3}, in ascending order, and * every `remediation.finding` referenced by gitleaks-baseline.json * resolves to a row in the report (a deleted row would break this); * 6. the suite table (§2) references 10 existing test files; * 7. the report is signed (author + date + "verified against CI run"). */ import { describe, expect, it } from "vitest"; import { existsSync, readFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; const REPO_ROOT = fileURLToPath(new URL("../..", import.meta.url)); const REPORT = path.join(REPO_ROOT, "docs/security/audit-report.md"); const BASELINE = path.join(REPO_ROOT, "gitleaks-baseline.json"); const md = readFileSync(REPORT, "utf8"); /** Split a markdown table row into trimmed cells (drops the edge pipes). */ function cells(line: string): string[] { return line .split("|") .slice(1, -1) .map((c) => c.trim()); } const stripBackticks = (s: string) => s.replace(/`/g, ""); /** `noUncheckedIndexedAccess`-safe indexed access (row shape is asserted). */ const get = (row: string[], i: number): string => row[i] ?? ""; // --------------------------------------------------------------------------- // 1. The report exists and names the standard // --------------------------------------------------------------------------- describe("audit-report / existence + standard", () => { it("exists at docs/security/audit-report.md and names ASVS 4.0.3 Level 2", () => { expect(existsSync(REPORT), "docs/security/audit-report.md must exist").toBe(true); expect(md).toMatch(/ASVS[^.]{0,80}Level 2|Level 2[^.]{0,80}ASVS/i); expect(md).toMatch(/4\.0\.3/); }); }); // --------------------------------------------------------------------------- // 2. All 16 ASVS verification requirement areas are present // --------------------------------------------------------------------------- describe("audit-report / ASVS L2 checklist completeness", () => { it("covers all 16 verification requirement areas (V1–V16)", () => { for (let n = 1; n <= 16; n++) { expect(md, `the ASVS area "V${n}" must have a section`).toMatch(new RegExp(`^### V${n} `, "m")); } }); it("every L2 control row carries a valid status", () => { const rows = md .split("\n") .filter((l) => l.startsWith("| L2 ") && !get(cells(l), 1).toLowerCase().startsWith("control")); expect(rows.length, "the checklist must have L2 control rows (one per verified control)").toBeGreaterThanOrEqual(40); const validStatus = /^(pass|N-A|fixed|mitigated|accepted)\b/i; for (const row of rows) { const c = cells(row); expect(c.length, `control row must have 4 cells (L2 | control | status | evidence), got "${row.slice(0, 60)}"`).toBe(4); expect(validStatus.test(get(c, 2)), `control "${get(c, 1).slice(0, 60)}" has an invalid status "${get(c, 2)}"`).toBe(true); } }); }); // --------------------------------------------------------------------------- // 3 + 5. Findings table: complete, append-only, cross-referenced // --------------------------------------------------------------------------- describe("audit-report / findings table", () => { const FINDING = md .split("\n") .filter((l) => /^\| F-\d{3} /.test(l)) .map((l) => cells(l)); it("has findings, each with id/asvs-ref/severity/status/fix-task/regression-test", () => { expect(FINDING.length, "the findings table must not be empty").toBeGreaterThan(0); const validStatus = /^(fixed|mitigated|accepted|N-A)\b/i; for (const c of FINDING) { expect(c.length, `finding row must have 7 cells, got "${get(c, 0)}"`).toBe(7); expect(get(c, 0), "finding id must be F-\\d{3}").toMatch(/^F-\d{3}$/); expect(get(c, 1), `${get(c, 0)}: an ASVS reference is required`).toMatch(/^V\d{1,2}\./); expect(get(c, 2), `${get(c, 0)}: a severity is required`).toMatch(/^(Critical|High|Medium|Low|Info)$/); expect(validStatus.test(get(c, 3)), `${get(c, 0)}: status must be fixed|mitigated|accepted|N-A, got "${get(c, 3)}"`).toBe(true); expect(get(c, 5).length, `${get(c, 0)}: a fix task (or explicit n/a) is required`).toBeGreaterThan(0); // the regression-test path must exist: const tpath = stripBackticks(get(c, 6)); expect(tpath.endsWith(".test.ts"), `${get(c, 0)}: the regression test must be a .test.ts file, got "${tpath}"`).toBe(true); expect(existsSync(path.join(REPO_ROOT, tpath)), `${get(c, 0)}: the regression test path must exist: ${tpath}`).toBe(true); } }); it("is append-only: ids are unique and in ascending order", () => { const ids = FINDING.map((c) => get(c, 0)); expect(new Set(ids).size, "finding ids must be unique").toBe(ids.length); for (let i = 1; i < ids.length; i++) { const a = Number(get(ids, i - 1).slice(2)); const b = Number(get(ids, i).slice(2)); expect(b, `finding ids must be ascending (append-only), but ${get(ids, i)} follows ${get(ids, i - 1)}`).toBeGreaterThan(a); } }); it("every gitleaks-baseline remediation resolves to a finding row", () => { const baseline = JSON.parse(readFileSync(BASELINE, "utf8")) as { remediation: { finding: string } }[]; const ids = new Set(FINDING.map((c) => get(c, 0))); const refs = new Set(baseline.map((e) => e.remediation.finding)); for (const ref of refs) { expect(ids.has(ref), `baseline remediation "${ref}" must resolve to a finding row in the report`).toBe(true); } }); }); // --------------------------------------------------------------------------- // 6. The 10-suite table references existing test files // --------------------------------------------------------------------------- describe("audit-report / suite table", () => { it("references exactly 10 security suites, each an existing file", () => { const rows = md .split("\n") .filter((l) => /^\| \d+ \|/.test(l)) .map((l) => cells(l)); expect(rows.length, "the suite table must list the 10 deliverable suites").toBe(10); const paths = rows.map((c) => stripBackticks(get(c, 2))); for (const p of paths) { expect(p.endsWith(".test.ts"), `suite path must be a .test.ts file: ${p}`).toBe(true); expect(existsSync(path.join(REPO_ROOT, p)), `the suite file must exist: ${p}`).toBe(true); } }); }); // --------------------------------------------------------------------------- // 7. Sign-off // --------------------------------------------------------------------------- describe("audit-report / sign-off", () => { it("is signed (author + date + verified against CI run)", () => { expect(md, "the report must have a sign-off section").toMatch(/^## 7\. Sign-off/m); expect(md, "the sign-off must name an author").toMatch(/\*\*Author:\*\*/); expect(md, "the sign-off must carry a date").toMatch(/\*\*Date:\*\*\s*\d{4}-\d{2}-\d{2}/); expect(md, "the sign-off must reference the CI run").toMatch(/\*\*Verified against CI run:\*\*/); }); });