search-search.k6.js

search-search.k6.jstests/performance/search-search.k6.jsraw ← performance11,149 B← Back to the app
1/**
2 * AI Software Factory — 100k-doc search query performance suite (TASK-0100).
3 *
4 * Measures Phase-8 query DoD over the `perf-search-{a,b}` fixture corpus
5 * (100k docs, seeded by `fixtures/search-100k.ts`): p95 < 500 ms, p99 < 1 s,
6 * error rate < 0.1%. Load profile is Poisson arrivals (k6's
7 * `constant-arrival-rate` executor is a Poisson process) at a target
8 * request rate.
9 *
10 * The two latency metrics:
11 *  - `search_duration_ms` — client-observed POST /api/v1/search RTT
12 *    (auth + validation + OpenSearch + serialization). The DoD threshold
13 *    applies to this one ("p95 = API query latency").
14 *  - `search_took_ms_server` — the API's own `tookMs` envelope field
15 *    (server-side, OpenSearch round-trip only). Logged for cross-check
16 *    (spec req 2) — the delta isolates framework overhead.
17 *
18 * Each VU mints an HS256 access JWT (k6/crypto) for a DEDICATED perf user
19 * (`perf-search-u-XXX`, round-robin over the 200 fixture users) so:
20 *   - auth passes (shared secret, claims shape = @factory/auth/tokens),
21 *   - the viewer scope is that user's real DB membership (one fixture
22 *     workspace), so queries exercise the tenant + permission filters on
23 *     real rows, and
24 *   - the per-user API rate limit is spread across 200 identities.
25 * The CI job MUST set RATE_LIMIT_API high (the per-user default is 100/min).
26 *
27 * The report (p50/p95/p99, error rate, doc count, env fingerprint) is
28 * written by handleSummary to `reports/search-last.json` (k6 writes
29 * handleSummary file keys to the CWD — the suite is run from
30 * tests/performance via its package.json scripts). The nightly job compares
31 * it against the previous baseline (see the `perf-search` CI job).
32 *
33 * Env (all optional; the package.json scripts pass the profile):
34 *   K6_API_URL          default http://localhost:4000
35 *   K6_JWT_SECRET       HS256 secret (must match the API's JWT_SECRET)
36 *   K6_SEARCH_WORKSPACES  comma list (default perf-search-a,perf-search-b)
37 *   K6_SEARCH_URL       OpenSearch base for the setup() fingerprint (default http://localhost:9200)
38 *   K6_VERSION          k6 version for the report fingerprint (CI sets it)
39 *   K6_QUERY            search phrase (default "factory search")
40 *   K6_VUS / K6_DURATION_SECONDS / K6_RATE   profile (100 / 300 / 50 full,
41 *                                            10 / 60 / 10 reduced — CI)
42 *   K6_P95_MS / K6_P99_MS / K6_ERROR_RATE    thresholds (DoD defaults; the
43 *                                            reduced CI profile may run a
44 *                                            2× flake-superset like the
45 *                                            realtime smoke tier does)
46 */
47import http from "k6/http";
48import encoding from "k6/encoding";
49import crypto from "k6/crypto";
50import { Trend, Rate } from "k6/metrics";
51 
52const API_BASE = String(__ENV.K6_API_URL || "http://localhost:4000");
53const SECRET = __ENV.K6_JWT_SECRET || "";
54const SEARCH_URL = String(__ENV.K6_SEARCH_URL || "http://localhost:9200").replace(/\/+$/, "");
55const WORKSPACES = String(__ENV.K6_SEARCH_WORKSPACES || "perf-search-a,perf-search-b")
56  .split(",")
57  .map((s) => s.trim())
58  .filter(Boolean);
59const K6_VERSION = __ENV.K6_VERSION || "unknown";
60const QUERY = String(__ENV.K6_QUERY || "factory search");
61 
62// Profile (full: 100 VUs / 5 min / 50 req/s Poisson — spec req 2; the reduced
63// CI profile is 10 VUs / 60 s / 10 req/s via the search:100k:reduced script).
64const VUS = parseInt(__ENV.K6_VUS || "100", 10);
65const DURATION_SECONDS = parseInt(__ENV.K6_DURATION_SECONDS || "300", 10);
66const RATE = parseInt(__ENV.K6_RATE || "50", 10); // req/s
67 
68// DoD thresholds (spec: p95 < 500 ms, p99 < 1 s, error rate < 0.1%).
69const P95_MS = parseInt(__ENV.K6_P95_MS || "500", 10);
70const P99_MS = parseInt(__ENV.K6_P99_MS || "1000", 10);
71const ERROR_RATE = parseFloat(__ENV.K6_ERROR_RATE || "0.001");
72 
73const searchDuration = new Trend("search_duration_ms", true);
74const searchTookMs = new Trend("search_took_ms_server", true);
75const searchErrors = new Rate("search_errors");
76 
77// Captured from setup() (module globals persist until handleSummary).
78let SETUP = { osVersion: "unknown", docCount: -1 };
79 
80export const options = {
81  scenarios: {
82    search_100k: {
83      // constant-arrival-rate = Poisson arrival process at `rate` req/s —
84      // the spec's "Poisson arrivals 50 req/s".
85      executor: "constant-arrival-rate",
86      rate: RATE,
87      timeUnit: "1s",
88      duration: `${DURATION_SECONDS}s`,
89      preAllocatedVUs: VUS,
90      maxVUs: Math.max(VUS * 2, 20),
91    },
92  },
93  thresholds: {
94    search_duration_ms: [`p(95)<${P95_MS}`, `p(99)<${P99_MS}`],
95    search_errors: [`rate<${ERROR_RATE}`],
96    http_req_failed: [`rate<${ERROR_RATE}`],
97  },
98  // k6 only ships avg/min/med/max/p(90)/p(95) into handleSummary by default;
99  // p(99) is a DoD metric (enforced above as a threshold but NOT exposed in
100  // the summary values) so opt it in globally. (The per-metric
101  // `trend.summaryTrendStats` does NOT propagate to handleSummary in k6
102  // v0.57 — probe-verified — hence the global option.)
103  summaryTrendStats: ["avg", "min", "med", "max", "p(90)", "p(95)", "p(99)"],
104};
105 
106/** base64url for JWT segments (no padding). */
107function b64url(input) {
108  return input.replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
109}
110 
111/** 0-based fixture user for a VU (200 perf users, round-robin). */
112function vUser(vu) {
113  const i = vu % 200;
114  return { id: `perf-search-u-${String(i).padStart(3, "0")}`, workspace: WORKSPACES[i % WORKSPACES.length] };
115}
116 
117/**
118 * Mint the VU's access JWT (HS256, shared secret) — the exact claims shape
119 * @factory/auth/tokens verifyAccessToken accepts: sub, exp/iat (unix
120 * seconds), jti, workspaces (string[]).
121 */
122function mintToken(sub, workspaces) {
123  const nowSec = Math.floor(Date.now() / 1000);
124  // BUG FIXED (2026-09-29, S6): k6 v0.57 exposes k6/encoding as
125  // { b64encode, b64decode } — `encoding.encode(...)` threw on every
126  // iteration (the check:search script only PARSES the file, so it could
127  // never catch this). b64encode accepts a string directly (Goja has no
128  // TextEncoder/btoa); the JWT JSON is ASCII-safe (JSON.stringify escapes
129  // non-ASCII as \uXXXX), so string input is exact. crypto.hmac's
130  // "base64" output option is genuine (verified by k6 probe) → b64url it.
131  const header = b64url(encoding.b64encode(JSON.stringify({ alg: "HS256", typ: "JWT" })));
132  const claims = b64url(
133    encoding.b64encode(
134      JSON.stringify({
135        sub: sub,
136        workspaces: workspaces,
137        iat: nowSec,
138        exp: nowSec + 900,
139        jti: `k6-search-${sub}-${nowSec}`,
140      }),
141    ),
142  );
143  const sig = b64url(crypto.hmac("sha256", SECRET, `${header}.${claims}`, "base64"));
144  return `${header}.${claims}.${sig}`;
145}
146 
147export function setup() {
148  let osVersion = "unknown";
149  let docCount = -1;
150  try {
151    const v = http.get(SEARCH_URL, { timeout: "10s" });
152    if (v.status === 200) {
153      const j = v.json();
154      osVersion = (j.version && j.version.number) || String(v.status);
155    }
156  } catch (e) {
157    /* fingerprint only — never fail the run for it */
158  }
159  try {
160    const c = http.post(
161      `${SEARCH_URL}/factory-content/_count`,
162      JSON.stringify({ query: { bool: { filter: [{ terms: { workspaceId: WORKSPACES } }] } } }),
163      { headers: { "content-type": "application/json" }, timeout: "10s" },
164    );
165    if (c.status === 200) docCount = c.json().count ?? -1;
166  } catch (e) {
167    /* fingerprint only */
168  }
169  SETUP = { osVersion, docCount };
170  if (docCount < 10_000) {
171    console.warn(
172      `[search-100k] WARNING: only ${docCount} fixture docs found (expected ~100k). ` +
173        `Run the seeder first: pnpm --dir tests/performance perf:seed:search`,
174    );
175  }
176  return SETUP;
177}
178 
179export default function (data) {
180  SETUP = data;
181  const u = vUser(__VU);
182  const res = http.post(
183    `${API_BASE}/api/v1/search`,
184    JSON.stringify({ q: QUERY, limit: 10 }),
185    {
186      headers: { authorization: `Bearer ${mintToken(u.id, [u.workspace])}`, "content-type": "application/json" },
187      timeout: "10s",
188    },
189  );
190  const ok = res.status === 200;
191  searchDuration.add(res.timings.duration);
192  searchErrors.add(!ok);
193  if (ok) {
194    try {
195      const body = res.json();
196      if (body && typeof body.tookMs === "number") searchTookMs.add(body.tookMs);
197    } catch (e) {
198      /* envelope shape change → surface via thresholds, not a crash */
199    }
200  }
201}
202 
203/**
204 * Write the baseline report (spec req 5/AC: p50/p95/p99, error rate, doc
205 * count, env fingerprint). k6 writes handleSummary file entries to the CWD
206 * (the suite runs from tests/performance) → reports/search-last.json.
207 */
208export function handleSummary(data) {
209  // BUG FIXED (2026-09-29, S6): when a run produces zero samples the metric
210  // objects are ABSENT from `data.metrics` (not just .values) — the old code
211  // crashed handleSummary and the report file was never written. Guard all
212  // three.
213  const v = (data.metrics.search_duration_ms && data.metrics.search_duration_ms.values) || {};
214  const took = (data.metrics.search_took_ms_server && data.metrics.search_took_ms_server.values) || {};
215  // BUG FIXED (2026-09-29, S6): the module-global `SETUP` written in
216  // setup() does NOT propagate to handleSummary in k6 v0.57 (separate
217  // scopes — probe-verified: handleSummary saw the initial
218  // {docCount:-1, osVersion:"unknown"}), so the report always carried the
219  // placeholder fingerprint. Re-fetch it here instead (fingerprint only —
220  // a failed lookup degrades to the placeholders, never fails the run).
221  let osVersion = "unknown";
222  let docCount = -1;
223  try {
224    const ver = http.get(SEARCH_URL, { timeout: "10s" });
225    if (ver.status === 200) {
226      const j = ver.json();
227      osVersion = (j.version && j.version.number) || String(ver.status);
228    }
229    const c = http.post(
230      `${SEARCH_URL}/factory-content/_count`,
231      JSON.stringify({ query: { bool: { filter: [{ terms: { workspaceId: WORKSPACES } }] } } }),
232      { headers: { "content-type": "application/json" }, timeout: "10s" },
233    );
234    if (c.status === 200) docCount = c.json().count ?? -1;
235  } catch (e) {
236    /* fingerprint only */
237  }
238  const report = {
239    task: "TASK-0100 search 100k",
240    generatedAt: new Date().toISOString(),
241    profile: {
242      vus: VUS,
243      durationSeconds: DURATION_SECONDS,
244      ratePerSecond: RATE,
245      query: QUERY,
246      workspaces: WORKSPACES,
247    },
248    latencyMs: {
249      p50: v.med ?? null,
250      p95: v["p(95)"] ?? null,
251      p99: v["p(99)"] ?? null,
252      avg: v.avg ?? null,
253    },
254    serverTookMs: {
255      p50: took.med ?? null,
256      p95: took["p(95)"] ?? null,
257      p99: took["p(99)"] ?? null,
258    },
259    errorRate: (data.metrics.search_errors && data.metrics.search_errors.values && data.metrics.search_errors.values.rate) ?? null,
260    docCount,
261    fingerprint: {
262      k6Version: K6_VERSION,
263      opensearchVersion: osVersion,
264      thresholds: { p95Ms: P95_MS, p99Ms: P99_MS, errorRate: ERROR_RATE },
265    },
266  };
267  const json = JSON.stringify(report, null, 2);
268  return {
269    "reports/search-last.json": json,
270    "search-summary.json": json,
271  };
272}
273