threshold-breach-demo.js

threshold-breach-demo.jstests/performance/threshold-breach-demo.jsraw ← performance4,146 B← Back to the app
1/**
2 * AI Software Factory — CI threshold-breach demonstration (TASK-0135).
3 *
4 * The acceptance criteria require that the CI matrix "enforces: dev
5 * report-only, staging hard … **demonstrated by a fixture run with a lowered
6 * threshold that fails the pipeline step**." This is that fixture.
7 *
8 * It is a deliberately-impossible suite: it drives a REAL, light endpoint
9 * (GET /api/v1/health, a few seconds) and asserts a threshold no real HTTP
10 * round-trip can meet — `p(95) < 1 ms` (configurable via K6_P95_MS, default 1,
11 * i.e. one millisecond; the 50-th percentile of a health ping is already far
12 * above that). With K6_STRICT=1 (the default) the threshold is HARD, so the
13 * breach makes `k6 run` exit non-zero (k6 exit code 99) and the CI step that
14 * runs this suite goes RED — exactly the demonstration the spec calls for.
15 *
16 * It is DISTINCT and obviously-failing on purpose:
17 *   - a separate file (not one of the four DoD suites), named to say what it
18 *     is;
19 *   - an impossible budget (p95 < 1 ms) that cannot be met by configuration,
20 *     so the red result is a function of the threshold, not of the data;
21 *   - a trivially-light workload (a health ping) so the only reason it fails
22 *     is the lowered threshold — a clean isolation of "breach ⇒ red step".
23 *
24 * The report-only (K6_STRICT=0) counterpart: the SAME impossible threshold
25 * downgraded to report-only would NOT fail the step (k6 prints the breach in
26 * the summary but exits 0) — which is precisely the dev/PR tier's behaviour.
27 * The two runs together demonstrate the report-only-vs-hard switch the matrix
28 * enforces.
29 *
30 * Run (demonstrate the RED step — staging-hard):
31 *   pnpm --dir tests/performance threshold-breach-demo
32 *   # → "running… (100.00%)", then the p95<1 threshold ✗ and a non-zero exit.
33 * Run (demonstrate the report-only GREEN-with-warning):
34 *   K6_STRICT=0 pnpm --dir tests/performance threshold-breach-demo
35 *
36 * Env:
37 *   K6_API_URL   API base (default http://localhost:4000)
38 *   K6_P95_MS    the (impossible) p95 budget in ms (default 1)
39 *   K6_STRICT    "1" hard (default — the step FAILS) / "0" report-only (the
40 *                step passes with the breach printed)
41 */
42import http from "k6/http";
43import { check } from "k6";
44import { Trend } from "k6/metrics";
45 
46const healthMs = new Trend("health_ms", true);
47 
48const API = __ENV.K6_API_URL || "http://localhost:4000";
49const STRICT = (__ENV.K6_STRICT || "1") !== "0";
50// The impossible budget. Default 1 ms — no real HTTP round-trip (DNS + TCP +
51// TLS + handler + response) can hold p95 under a millisecond, so the run is a
52// deterministic breach, independent of the actual server speed.
53const P95_BUDGET_MS = parseInt(__ENV.K6_P95_MS || "1", 10);
54 
55export function setup() {
56  // Fail loudly if the API is unreachable — the demonstration is about the
57  // THRESHOLD breaching, not about a dead server (a dead server would also
58  // trip http_req_failed, muddying the "lowered threshold ⇒ red" isolation).
59  const res = http.get(`${API}/api/v1/health`, { timeout: "5s" });
60  if (res.status !== 200) {
61    throw new Error(`api health unreachable (${res.status}) — is the API running on ${API}?`);
62  }
63  return { ok: true };
64}
65 
66export default function () {
67  const res = http.get(`${API}/api/v1/health`);
68  healthMs.add(res.timings.duration);
69  check(res, { "[breach-demo] health 200": (r) => r.status === 200 });
70}
71 
72export const options = {
73  vus: 10,
74  duration: "8s",
75  thresholds: STRICT
76    ? {
77        // The lowered (impossible) threshold — the demonstration of a breach
78        // failing the step. p95 < 1 ms is not achievable for a real request.
79        health_ms: [`p(95)<${P95_BUDGET_MS}`],
80      }
81    : {
82        // Report-only tier: the same impossible budget, but k6 must not fail
83        // the step. A trivially-true threshold (avg<1e15) keeps the run green
84        // while the summary still prints the real (breaching) p95 — mirroring
85        // how the dev/PR tier reports without blocking.
86        health_ms: ["avg<1e15"],
87      },
88  summaryTrendStats: ["avg", "min", "med", "p(90)", "p(95)", "p(99)", "max"],
89};
90