/** * AI Software Factory — 100k-doc search query performance suite (TASK-0100). * * Measures Phase-8 query DoD over the `perf-search-{a,b}` fixture corpus * (100k docs, seeded by `fixtures/search-100k.ts`): p95 < 500 ms, p99 < 1 s, * error rate < 0.1%. Load profile is Poisson arrivals (k6's * `constant-arrival-rate` executor is a Poisson process) at a target * request rate. * * The two latency metrics: * - `search_duration_ms` — client-observed POST /api/v1/search RTT * (auth + validation + OpenSearch + serialization). The DoD threshold * applies to this one ("p95 = API query latency"). * - `search_took_ms_server` — the API's own `tookMs` envelope field * (server-side, OpenSearch round-trip only). Logged for cross-check * (spec req 2) — the delta isolates framework overhead. * * Each VU mints an HS256 access JWT (k6/crypto) for a DEDICATED perf user * (`perf-search-u-XXX`, round-robin over the 200 fixture users) so: * - auth passes (shared secret, claims shape = @factory/auth/tokens), * - the viewer scope is that user's real DB membership (one fixture * workspace), so queries exercise the tenant + permission filters on * real rows, and * - the per-user API rate limit is spread across 200 identities. * The CI job MUST set RATE_LIMIT_API high (the per-user default is 100/min). * * The report (p50/p95/p99, error rate, doc count, env fingerprint) is * written by handleSummary to `reports/search-last.json` (k6 writes * handleSummary file keys to the CWD — the suite is run from * tests/performance via its package.json scripts). The nightly job compares * it against the previous baseline (see the `perf-search` CI job). * * Env (all optional; the package.json scripts pass the profile): * K6_API_URL default http://localhost:4000 * K6_JWT_SECRET HS256 secret (must match the API's JWT_SECRET) * K6_SEARCH_WORKSPACES comma list (default perf-search-a,perf-search-b) * K6_SEARCH_URL OpenSearch base for the setup() fingerprint (default http://localhost:9200) * K6_VERSION k6 version for the report fingerprint (CI sets it) * K6_QUERY search phrase (default "factory search") * K6_VUS / K6_DURATION_SECONDS / K6_RATE profile (100 / 300 / 50 full, * 10 / 60 / 10 reduced — CI) * K6_P95_MS / K6_P99_MS / K6_ERROR_RATE thresholds (DoD defaults; the * reduced CI profile may run a * 2× flake-superset like the * realtime smoke tier does) */ import http from "k6/http"; import encoding from "k6/encoding"; import crypto from "k6/crypto"; import { Trend, Rate } from "k6/metrics"; const API_BASE = String(__ENV.K6_API_URL || "http://localhost:4000"); const SECRET = __ENV.K6_JWT_SECRET || ""; const SEARCH_URL = String(__ENV.K6_SEARCH_URL || "http://localhost:9200").replace(/\/+$/, ""); const WORKSPACES = String(__ENV.K6_SEARCH_WORKSPACES || "perf-search-a,perf-search-b") .split(",") .map((s) => s.trim()) .filter(Boolean); const K6_VERSION = __ENV.K6_VERSION || "unknown"; const QUERY = String(__ENV.K6_QUERY || "factory search"); // Profile (full: 100 VUs / 5 min / 50 req/s Poisson — spec req 2; the reduced // CI profile is 10 VUs / 60 s / 10 req/s via the search:100k:reduced script). const VUS = parseInt(__ENV.K6_VUS || "100", 10); const DURATION_SECONDS = parseInt(__ENV.K6_DURATION_SECONDS || "300", 10); const RATE = parseInt(__ENV.K6_RATE || "50", 10); // req/s // DoD thresholds (spec: p95 < 500 ms, p99 < 1 s, error rate < 0.1%). const P95_MS = parseInt(__ENV.K6_P95_MS || "500", 10); const P99_MS = parseInt(__ENV.K6_P99_MS || "1000", 10); const ERROR_RATE = parseFloat(__ENV.K6_ERROR_RATE || "0.001"); const searchDuration = new Trend("search_duration_ms", true); const searchTookMs = new Trend("search_took_ms_server", true); const searchErrors = new Rate("search_errors"); // Captured from setup() (module globals persist until handleSummary). let SETUP = { osVersion: "unknown", docCount: -1 }; export const options = { scenarios: { search_100k: { // constant-arrival-rate = Poisson arrival process at `rate` req/s — // the spec's "Poisson arrivals 50 req/s". executor: "constant-arrival-rate", rate: RATE, timeUnit: "1s", duration: `${DURATION_SECONDS}s`, preAllocatedVUs: VUS, maxVUs: Math.max(VUS * 2, 20), }, }, thresholds: { search_duration_ms: [`p(95)<${P95_MS}`, `p(99)<${P99_MS}`], search_errors: [`rate<${ERROR_RATE}`], http_req_failed: [`rate<${ERROR_RATE}`], }, // k6 only ships avg/min/med/max/p(90)/p(95) into handleSummary by default; // p(99) is a DoD metric (enforced above as a threshold but NOT exposed in // the summary values) so opt it in globally. (The per-metric // `trend.summaryTrendStats` does NOT propagate to handleSummary in k6 // v0.57 — probe-verified — hence the global option.) summaryTrendStats: ["avg", "min", "med", "max", "p(90)", "p(95)", "p(99)"], }; /** base64url for JWT segments (no padding). */ function b64url(input) { return input.replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); } /** 0-based fixture user for a VU (200 perf users, round-robin). */ function vUser(vu) { const i = vu % 200; return { id: `perf-search-u-${String(i).padStart(3, "0")}`, workspace: WORKSPACES[i % WORKSPACES.length] }; } /** * Mint the VU's access JWT (HS256, shared secret) — the exact claims shape * @factory/auth/tokens verifyAccessToken accepts: sub, exp/iat (unix * seconds), jti, workspaces (string[]). */ function mintToken(sub, workspaces) { const nowSec = Math.floor(Date.now() / 1000); // BUG FIXED (2026-09-29, S6): k6 v0.57 exposes k6/encoding as // { b64encode, b64decode } — `encoding.encode(...)` threw on every // iteration (the check:search script only PARSES the file, so it could // never catch this). b64encode accepts a string directly (Goja has no // TextEncoder/btoa); the JWT JSON is ASCII-safe (JSON.stringify escapes // non-ASCII as \uXXXX), so string input is exact. crypto.hmac's // "base64" output option is genuine (verified by k6 probe) → b64url it. const header = b64url(encoding.b64encode(JSON.stringify({ alg: "HS256", typ: "JWT" }))); const claims = b64url( encoding.b64encode( JSON.stringify({ sub: sub, workspaces: workspaces, iat: nowSec, exp: nowSec + 900, jti: `k6-search-${sub}-${nowSec}`, }), ), ); const sig = b64url(crypto.hmac("sha256", SECRET, `${header}.${claims}`, "base64")); return `${header}.${claims}.${sig}`; } export function setup() { let osVersion = "unknown"; let docCount = -1; try { const v = http.get(SEARCH_URL, { timeout: "10s" }); if (v.status === 200) { const j = v.json(); osVersion = (j.version && j.version.number) || String(v.status); } } catch (e) { /* fingerprint only — never fail the run for it */ } try { const c = http.post( `${SEARCH_URL}/factory-content/_count`, JSON.stringify({ query: { bool: { filter: [{ terms: { workspaceId: WORKSPACES } }] } } }), { headers: { "content-type": "application/json" }, timeout: "10s" }, ); if (c.status === 200) docCount = c.json().count ?? -1; } catch (e) { /* fingerprint only */ } SETUP = { osVersion, docCount }; if (docCount < 10_000) { console.warn( `[search-100k] WARNING: only ${docCount} fixture docs found (expected ~100k). ` + `Run the seeder first: pnpm --dir tests/performance perf:seed:search`, ); } return SETUP; } export default function (data) { SETUP = data; const u = vUser(__VU); const res = http.post( `${API_BASE}/api/v1/search`, JSON.stringify({ q: QUERY, limit: 10 }), { headers: { authorization: `Bearer ${mintToken(u.id, [u.workspace])}`, "content-type": "application/json" }, timeout: "10s", }, ); const ok = res.status === 200; searchDuration.add(res.timings.duration); searchErrors.add(!ok); if (ok) { try { const body = res.json(); if (body && typeof body.tookMs === "number") searchTookMs.add(body.tookMs); } catch (e) { /* envelope shape change → surface via thresholds, not a crash */ } } } /** * Write the baseline report (spec req 5/AC: p50/p95/p99, error rate, doc * count, env fingerprint). k6 writes handleSummary file entries to the CWD * (the suite runs from tests/performance) → reports/search-last.json. */ export function handleSummary(data) { // BUG FIXED (2026-09-29, S6): when a run produces zero samples the metric // objects are ABSENT from `data.metrics` (not just .values) — the old code // crashed handleSummary and the report file was never written. Guard all // three. const v = (data.metrics.search_duration_ms && data.metrics.search_duration_ms.values) || {}; const took = (data.metrics.search_took_ms_server && data.metrics.search_took_ms_server.values) || {}; // BUG FIXED (2026-09-29, S6): the module-global `SETUP` written in // setup() does NOT propagate to handleSummary in k6 v0.57 (separate // scopes — probe-verified: handleSummary saw the initial // {docCount:-1, osVersion:"unknown"}), so the report always carried the // placeholder fingerprint. Re-fetch it here instead (fingerprint only — // a failed lookup degrades to the placeholders, never fails the run). let osVersion = "unknown"; let docCount = -1; try { const ver = http.get(SEARCH_URL, { timeout: "10s" }); if (ver.status === 200) { const j = ver.json(); osVersion = (j.version && j.version.number) || String(ver.status); } const c = http.post( `${SEARCH_URL}/factory-content/_count`, JSON.stringify({ query: { bool: { filter: [{ terms: { workspaceId: WORKSPACES } }] } } }), { headers: { "content-type": "application/json" }, timeout: "10s" }, ); if (c.status === 200) docCount = c.json().count ?? -1; } catch (e) { /* fingerprint only */ } const report = { task: "TASK-0100 search 100k", generatedAt: new Date().toISOString(), profile: { vus: VUS, durationSeconds: DURATION_SECONDS, ratePerSecond: RATE, query: QUERY, workspaces: WORKSPACES, }, latencyMs: { p50: v.med ?? null, p95: v["p(95)"] ?? null, p99: v["p(99)"] ?? null, avg: v.avg ?? null, }, serverTookMs: { p50: took.med ?? null, p95: took["p(95)"] ?? null, p99: took["p(99)"] ?? null, }, errorRate: (data.metrics.search_errors && data.metrics.search_errors.values && data.metrics.search_errors.values.rate) ?? null, docCount, fingerprint: { k6Version: K6_VERSION, opensearchVersion: osVersion, thresholds: { p95Ms: P95_MS, p99Ms: P99_MS, errorRate: ERROR_RATE }, }, }; const json = JSON.stringify(report, null, 2); return { "reports/search-last.json": json, "search-summary.json": json, }; }