ci.yml · Source

ci.yml.github/workflows/ci.ymlraw ← workflows71,772 B← Back to the app
1# =============================================================================
2# AI Software Factory — CI
3#
4# Pipeline (mirrors AGENTS.md "CI/CD Contract"):
5#   install → lint → typecheck → unit → build → integration → security → e2e
6#
7# Job graph:
8#   lint ─┐
9#   typecheck ─┼→ build → integration ─┐
10#   unit ──┘        security  ─────────┼→ e2e
11#
12# Concurrency: a new push to the same PR ref cancels the in-progress run.
13# =============================================================================
14name: ci
15 
16on:
17  push:
18    branches: [main, develop]
19  pull_request:
20  # TASK-0100 S7: nightly full-profile search perf run (03:17 UTC — an odd
21  # minute, spread off the top-of-hour GitHub schedule pile-up). The
22  # perf-search job runs ONLY on this event or on push to main.
23  schedule:
24    - cron: "17 3 * * *"
25  # Manual trigger so the perf lane can be exercised before/without a merge.
26  # perf_mode: "report" = the report-only (dev) tier — the default, safe on any
27  #   runner. "staging-hard" = the HARD tier (K6_STRICT=1, full profiles + the
28  #   zero-op-loss verify) — intended for the self-hosted staging runner; it
29  #   BLOCKS on a threshold breach (exit 99) or a dropped Yjs op.
30  workflow_dispatch:
31    inputs:
32      perf_mode:
33        description: "Perf tier: report (report-only, default) or staging-hard (K6_STRICT=1, blocks on breach)"
34        required: false
35        default: "report"
36        type: choice
37        options: ["report", "staging-hard"]
38 
39concurrency:
40  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
41  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
42 
43# Shared setup for every job: checkout + pnpm (version from the root
44# package.json "packageManager" field: pnpm@9.15.0) + Node 20 + frozen install.
45# YAML anchor — expanded inline in each job.
46x-setup: &setup
47  - name: Checkout
48    uses: actions/checkout@v4
49 
50  - name: Setup pnpm
51    uses: pnpm/action-setup@v4
52    with:
53      version: 9.15.0 # keep in sync with "packageManager" in package.json
54 
55  - name: Setup Node.js
56    uses: actions/setup-node@v4
57    with:
58      node-version: 20
59      cache: pnpm
60 
61  - name: Install dependencies
62    run: pnpm install --frozen-lockfile
63 
64env:
65  CI: true
66  NODE_ENV: test
67 
68jobs:
69  # ---------------------------------------------------------------------------
70  lint:
71    name: Lint
72    runs-on: ubuntu-24.04
73    timeout-minutes: 10
74    steps:
75      - <<: *setup
76      - name: Lint (turbo)
77        run: pnpm lint
78 
79  # ---------------------------------------------------------------------------
80  typecheck:
81    name: Typecheck
82    runs-on: ubuntu-24.04
83    timeout-minutes: 15
84    steps:
85      - <<: *setup
86      - name: Prisma generate
87        run: pnpm db:generate
88      - name: Typecheck (turbo)
89        run: pnpm typecheck
90 
91  # ---------------------------------------------------------------------------
92  unit:
93    name: Unit tests
94    runs-on: ubuntu-24.04
95    timeout-minutes: 15
96    steps:
97      - <<: *setup
98      - name: Prisma generate
99        run: pnpm db:generate
100      - name: Unit tests (turbo)
101        run: pnpm test:unit
102      - name: Upload unit coverage
103        if: always()
104        uses: actions/upload-artifact@v4
105        with:
106          name: unit-coverage
107          path: |
108            **/coverage/**
109            !node_modules
110          if-no-files-found: ignore
111          retention-days: 14
112 
113  # ---------------------------------------------------------------------------
114  build:
115    name: Build
116    runs-on: ubuntu-24.04
117    timeout-minutes: 20
118    needs: [lint, typecheck, unit]
119    steps:
120      - <<: *setup
121      - name: Prisma generate
122        run: pnpm db:generate
123      - name: Build all workspaces (turbo)
124        run: pnpm build
125      - name: Upload build artifacts
126        uses: actions/upload-artifact@v4
127        with:
128          name: build-outputs
129          path: |
130            apps/**/dist/**
131            apps/web/.next/**
132          if-no-files-found: ignore
133          retention-days: 7
134 
135  # ---------------------------------------------------------------------------
136  integration:
137    name: Integration tests
138    runs-on: ubuntu-24.04
139    timeout-minutes: 20
140    needs: [build]
141    services:
142      postgres:
143        image: postgres:16-alpine
144        env:
145          POSTGRES_USER: postgres
146          POSTGRES_PASSWORD: postgres
147          POSTGRES_DB: factory
148        ports:
149          - 5432:5432
150        options: >-
151          --health-cmd "pg_isready -U postgres -d factory"
152          --health-interval 5s
153          --health-timeout 5s
154          --health-retries 12
155      redis:
156        image: redis:7-alpine
157        ports:
158          - 6379:6379
159        options: >-
160          --health-cmd "redis-cli ping"
161          --health-interval 5s
162          --health-timeout 3s
163          --health-retries 12
164      minio:
165        image: minio/minio
166        command: server /data --console-address ":9001"
167        env:
168          MINIO_ROOT_USER: minioadmin
169          MINIO_ROOT_PASSWORD: minioadmin
170        ports:
171          - 9000:9000
172        options: >-
173          --health-cmd "curl -fsS http://localhost:9000/minio/health/live"
174          --health-interval 5s
175          --health-timeout 5s
176          --health-retries 12
177      opensearch:
178        image: opensearchproject/opensearch:2
179        env:
180          discovery.type: single-node
181          plugins.security.disabled: "true"
182          OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m"
183        ports:
184          - 9200:9200
185        # OpenSearch is SLOW to start on GitHub runners; it has no health
186        # option we can rely on via `options:` (curl exists in the image, but
187        # security-disable startup takes 30-90s), so we wait explicitly below.
188    env:
189      # Variable names match .env.example exactly; only the connection
190      # targets differ (GitHub service containers).
191      DATABASE_URL: postgresql://postgres:postgres@localhost:5432/factory?schema=public
192      REDIS_URL: redis://localhost:6379
193      SEARCH_URL: http://localhost:9200
194      SEARCH_USERNAME: ""
195      SEARCH_PASSWORD: ""
196      SEARCH_INDEX_PREFIX: factory
197      S3_ENDPOINT: http://localhost:9000
198      S3_ACCESS_KEY: minioadmin
199      S3_SECRET_KEY: minioadmin
200      S3_BUCKET: factory-uploads
201      S3_REGION: us-east-1
202      S3_FORCE_PATH_STYLE: "true"
203      # Test-only secrets (never real values).
204      JWT_SECRET: ci-test-jwt-secret
205      REFRESH_TOKEN_SECRET: ci-test-refresh-secret
206      SESSION_SECRET: ci-test-session-secret
207      MASTER_ENCRYPTION_KEY: "Y2ktdGVzdC1tYXN0ZXIta2V5LTMyc2J5dGVzISE="
208    steps:
209      - <<: *setup
210      - name: Wait for OpenSearch
211        # Retry loop: the image needs well over a minute on a busy runner.
212        run: |
213          for i in $(seq 1 60); do
214            if curl -sf http://localhost:9200/_cluster/health >/dev/null 2>&1; then
215              echo "opensearch is up after ${i} attempts"
216              exit 0
217            fi
218            sleep 5
219          done
220          echo "opensearch did not start in 300s" >&2
221          exit 1
222      - name: Create MinIO bucket
223        run: |
224          # mc is not in the runner; use the S3 API via the AWS CLI instead.
225          aws s3api create-bucket \
226            --endpoint-url http://localhost:9000 \
227            --bucket factory-uploads \
228            --region us-east-1 || true
229      - name: Create OpenSearch index
230        run: |
231          # Minimal index so search-dependent integration tests have a target.
232          curl -sf -X PUT "http://localhost:9200/factory-docs" \
233            -H 'Content-Type: application/json' -d '{}' || true
234          curl -sf -X POST "http://localhost:9200/factory-docs/_refresh" >/dev/null || true
235      - name: Prisma generate
236        run: pnpm db:generate
237      - name: Apply migrations
238        run: pnpm db:deploy
239      - name: Seed
240        run: pnpm db:seed
241      - name: Integration tests (turbo)
242        run: pnpm test:integration
243      - name: Upload integration output
244        if: always()
245        uses: actions/upload-artifact@v4
246        with:
247          name: integration-reports
248          path: tests/integration/*.report.*
249          if-no-files-found: ignore
250          retention-days: 14
251 
252  # ---------------------------------------------------------------------------
253  # TASK-0134 — the security stage is the OWASP ASVS L2 gate. It runs:
254  #   1. pnpm audit (high+ fails the build)
255  #   2. the 10 security suites (tests/security) against a RUNNING dev stack
256  #      (auth-paths / headers / upload-abuse / rate-limits are live;
257  #      SEC_REQUIRE_LIVE=1 makes a down stack fail, not skip)
258  #   3. gitleaks over the FULL git history with the pinned config + the
259  #      tracked-remediation baseline (a new secret → red build)
260  #   4. the four app images built + rootfs-scanned for secrets (the spec's
261  #      5th "test image" has no Dockerfile in the repo — recorded N/A in
262  #      docs/security/audit-report.md)
263  # ---------------------------------------------------------------------------
264  security:
265    name: Security scan
266    runs-on: ubuntu-24.04
267    timeout-minutes: 40
268    needs: [build]
269    services:
270      postgres:
271        image: postgres:16-alpine
272        env:
273          POSTGRES_USER: postgres
274          POSTGRES_PASSWORD: postgres
275          POSTGRES_DB: factory
276        ports:
277          - 5432:5432
278        options: >-
279          --health-cmd "pg_isready -U postgres -d factory"
280          --health-interval 5s
281          --health-timeout 5s
282          --health-retries 12
283      redis:
284        image: redis:7-alpine
285        ports:
286          - 6379:6379
287        options: >-
288          --health-cmd "redis-cli ping"
289          --health-interval 5s
290          --health-timeout 3s
291          --health-retries 12
292      minio:
293        image: minio/minio
294        command: server /data --console-address ":9001"
295        env:
296          MINIO_ROOT_USER: minioadmin
297          MINIO_ROOT_PASSWORD: minioadmin
298        ports:
299          - 9000:9000
300        options: >-
301          --health-cmd "curl -fsS http://localhost:9000/minio/health/live"
302          --health-interval 5s
303          --health-timeout 5s
304          --health-retries 12
305      opensearch:
306        image: opensearchproject/opensearch:2
307        env:
308          discovery.type: single-node
309          plugins.security.disabled: "true"
310          OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m"
311        ports:
312          - 9200:9200
313    env:
314      # Dev-mode stack (the live security suites assert dev behavior: no
315      # HSTS, no Secure cookie on plain-HTTP :4000) — NODE_ENV must NOT be
316      # production here.
317      DATABASE_URL: postgresql://postgres:postgres@localhost:5432/factory?schema=public
318      REDIS_URL: redis://localhost:6379
319      SEARCH_URL: http://localhost:9200
320      SEARCH_USERNAME: ""
321      SEARCH_PASSWORD: ""
322      SEARCH_INDEX_PREFIX: factory
323      S3_ENDPOINT: http://localhost:9000
324      S3_ACCESS_KEY: minioadmin
325      S3_SECRET_KEY: minioadmin
326      S3_BUCKET: factory-uploads
327      S3_REGION: us-east-1
328      S3_FORCE_PATH_STYLE: "true"
329      # Test-only secrets (never real values).
330      JWT_SECRET: ci-sec-jwt-secret
331      REFRESH_TOKEN_SECRET: ci-sec-refresh-secret
332      SESSION_SECRET: ci-sec-session-secret
333      MASTER_ENCRYPTION_KEY: "Y2ktc2VjLW1hc3Rlci1rZXktMzJieXRlcyEh"
334      INTERNAL_SERVICE_TOKEN: ci-sec-internal-service-token
335      INTERNAL_TEST_SEAM: "1"
336      AI_PROVIDER: mock
337      SEARCH_VECTOR_DIMENSION: "8"
338      AI_MOCK_DELTA_DELAY_MS: "60"
339      # Live-app security suites: a down stack FAILS the build (not skips).
340      SEC_REQUIRE_LIVE: "1"
341      # The two NEW rate-limit buckets are env-independent (hard-coded
342      # 60/min webhook, 5/min form) so they stay live-testable here; the
343      # global buckets are raised (as in the e2e job) so the other suites
344      # don't trip them.
345      RATE_LIMIT_API: "100000"
346      RATE_LIMIT_AUTH: "10000"
347    steps:
348      - <<: *setup
349      - name: Wait for OpenSearch
350        run: |
351          for i in $(seq 1 60); do
352            if curl -sf http://localhost:9200/_cluster/health >/dev/null 2>&1; then
353              echo "opensearch is up after ${i} attempts"
354              exit 0
355            fi
356            sleep 5
357          done
358          echo "opensearch did not start in 300s" >&2
359          exit 1
360      - name: Create MinIO bucket + OpenSearch index
361        run: |
362          aws s3api create-bucket \
363            --endpoint-url http://localhost:9000 \
364            --bucket factory-uploads \
365            --region us-east-1 || true
366          curl -sf -X PUT "http://localhost:9200/factory-docs" \
367            -H 'Content-Type: application/json' -d '{}' || true
368      - name: Download build outputs
369        uses: actions/download-artifact@v4
370        with:
371          name: build-outputs
372      - name: Prisma generate + migrate + seed
373        run: |
374          pnpm db:generate
375          pnpm db:deploy
376          pnpm db:seed
377      - name: Start dev stack (background)
378        run: |
379          set -o pipefail
380          nohup pnpm dev > /tmp/factory-dev.log 2>&1 &
381          echo $! > /tmp/factory-dev.pid
382          wait_for_port() {
383            local port="$1" name="$2"
384            for i in $(seq 1 120); do
385              if (exec 3<>"/dev/tcp/127.0.0.1/${port}") 2>/dev/null; then
386                exec 3>&- 3<&- || true
387                echo "${name} is up on :${port} after ${i}s"
388                return 0
389              fi
390              sleep 1
391            done
392            echo "${name} did not come up on :${port} in 120s" >&2
393            tail -n 100 /tmp/factory-dev.log >&2 || true
394            return 1
395          }
396          wait_for_port 4000 api
397          wait_for_port 3000 web
398          wait_for_port 4001 realtime
399      - name: pnpm audit (high+ vulnerabilities fail the build)
400        run: pnpm audit --audit-level=high
401      - name: Security suites (10 suites, live stack required)
402        run: pnpm test:security
403      - name: Stop dev stack
404        if: always()
405        run: |
406          if [ -f /tmp/factory-dev.pid ]; then
407            kill "$(cat /tmp/factory-dev.pid)" 2>/dev/null || true
408            pkill -f "turbo" 2>/dev/null || true
409            pkill -f "next dev" 2>/dev/null || true
410          fi
411          tail -n 50 /tmp/factory-dev.log || true
412      - name: Install gitleaks (pinned 8.18.4)
413        run: |
414          curl -sSfL "https://github.com/gitleaks/gitleaks/releases/download/v8.18.4/gitleaks_8.18.4_linux_x64.tar.gz" | tar -xz -C /usr/local/bin gitleaks
415          gitleaks version
416      - name: Gitleaks — full git history (new secret fails the build)
417        # protect = full history. --baseline-path suppresses exactly the
418        # pre-existing findings documented in gitleaks-baseline.json (each
419        # entry a tracked remediation); ANY other finding exits non-zero.
420        run: |
421          gitleaks protect \
422            --source=. \
423            --config gitleaks.toml \
424            --baseline-path gitleaks-baseline.json \
425            --report-format json \
426            --report-path gitleaks-report.json \
427            --exit-code 1
428          echo "full-history scan clean (baseline applied)"
429      - name: Gitleaks — build artifacts (dist/ + rootfs of every image)
430        run: |
431          scan_dir() {
432            local dir="$1" label="$2"
433            echo "=== scanning $label ($dir) ==="
434            if [ -d "$dir" ]; then
435              (cd "$dir" && gitleaks detect --source=. --config=../../gitleaks.toml --no-banner)
436            else
437              echo "(dir absent — skipped)"
438            fi
439          }
440          # Built JS/Next output (downloaded from the build job):
441          for d in apps/api/dist apps/web/.next apps/worker/dist apps/realtime/dist; do
442            scan_dir "$d" "$d"
443          done
444      - name: Build the four app images
445        run: |
446          for app in api web worker realtime; do
447            echo "=== building factory-$app ==="
448            docker build -f "apps/$app/Dockerfile" -t "factory-$app:scan" .
449          done
450      - name: Scan image root filesystems for secrets
451        # Spec: "the image-scan step runs on all 5 images ... and is clean".
452        # The 5th "test image" has no Dockerfile in the repo (only 4 apps
453        # exist) — recorded N/A in docs/security/audit-report.md.
454        run: |
455          for app in api web worker realtime; do
456            echo "=== scanning factory-$app rootfs ==="
457            CONTAINER=$(docker create "factory-$app:scan")
458            # No live .env may be baked into the image (.dockerignore):
459            docker export "$CONTAINER" | tar -tf - 2>/dev/null | grep -E '(^|/)\.env$|(^|/)\.env\.(local|production|ci)$' \
460              && { echo "FAIL: factory-$app ships a .env file"; docker rm "$CONTAINER" >/dev/null; exit 1; } \
461              || echo "no .env in factory-$app rootfs"
462            # gitleaks over the extracted rootfs (config + baseline applied):
463            mkdir -p /tmp/img-$app
464            docker export "$CONTAINER" | tar -xf - -C /tmp/img-$app
465            (cd /tmp/img-$app && gitleaks detect --source=. --config=../../gitleaks.toml --no-banner)
466            docker rm "$CONTAINER" >/dev/null
467            docker rmi "factory-$app:scan" >/dev/null || true
468          done
469          echo "all 4 images secret-free"
470      - name: Generate npm audit summary
471        if: always()
472        run: |
473          npm audit --json > npm-audit.json 2>&1 || true
474          node -e "
475            const a = require('./npm-audit.json');
476            const m = a.metadata && a.metadata.vulnerabilities || {};
477            const lines = [
478              '# npm audit summary', '',
479              '| Severity | Count |', '| --- | --- |',
480              '| critical | ' + (m.critical ?? 0) + ' |',
481              '| high | ' + (m.high ?? 0) + ' |',
482              '| moderate | ' + (m.moderate ?? 0) + ' |',
483              '| low | ' + (m.low ?? 0) + ' |', '',
484              'Fails CI only via the pnpm audit gate above (high+).'
485            ];
486            require('fs').writeFileSync('npm-audit-summary.md', lines.join('\n'));
487          "
488      - name: Upload audit summary
489        if: always()
490        uses: actions/upload-artifact@v4
491        with:
492          name: security-audit
493          path: |
494            npm-audit.json
495            npm-audit-summary.md
496            gitleaks-report.json
497          if-no-files-found: ignore
498          retention-days: 14
499 
500  # ---------------------------------------------------------------------------
501  e2e:
502    name: E2E (Playwright)
503    runs-on: ubuntu-24.04
504    timeout-minutes: 20
505    needs: [integration, security]
506    services:
507      postgres:
508        image: postgres:16-alpine
509        env:
510          POSTGRES_USER: postgres
511          POSTGRES_PASSWORD: postgres
512          POSTGRES_DB: factory
513        ports:
514          - 5432:5432
515        options: >-
516          --health-cmd "pg_isready -U postgres -d factory"
517          --health-interval 5s
518          --health-timeout 5s
519          --health-retries 12
520      redis:
521        image: redis:7-alpine
522        ports:
523          - 6379:6379
524        options: >-
525          --health-cmd "redis-cli ping"
526          --health-interval 5s
527          --health-timeout 3s
528          --health-retries 12
529      minio:
530        image: minio/minio
531        command: server /data --console-address ":9001"
532        env:
533          MINIO_ROOT_USER: minioadmin
534          MINIO_ROOT_PASSWORD: minioadmin
535        ports:
536          - 9000:9000
537        options: >-
538          --health-cmd "curl -fsS http://localhost:9000/minio/health/live"
539          --health-interval 5s
540          --health-timeout 5s
541          --health-retries 12
542      opensearch:
543        image: opensearchproject/opensearch:2
544        env:
545          discovery.type: single-node
546          plugins.security.disabled: "true"
547          OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m"
548        ports:
549          - 9200:9200
550    env:
551      DATABASE_URL: postgresql://postgres:postgres@localhost:5432/factory?schema=public
552      REDIS_URL: redis://localhost:6379
553      SEARCH_URL: http://localhost:9200
554      SEARCH_USERNAME: ""
555      SEARCH_PASSWORD: ""
556      SEARCH_INDEX_PREFIX: factory
557      S3_ENDPOINT: http://localhost:9000
558      S3_ACCESS_KEY: minioadmin
559      S3_SECRET_KEY: minioadmin
560      S3_BUCKET: factory-uploads
561      S3_REGION: us-east-1
562      S3_FORCE_PATH_STYLE: "true"
563      JWT_SECRET: ci-e2e-jwt-secret
564      REFRESH_TOKEN_SECRET: ci-e2e-refresh-secret
565      SESSION_SECRET: ci-e2e-session-secret
566      MASTER_ENCRYPTION_KEY: "Y2ktZTJlLW1hc3Rlci1rZXktMzJieXRlcyEh"
567      # The e2e specs' seam token — the API's INTERNAL_SERVICE_TOKEN (job env
568      # below; the .env copy is inert, see the note on AI_PROVIDER) — TASK-0095 S9.
569      E2E_INTERNAL_SERVICE_TOKEN: ci-e2e-internal-service-token
570      # The API reads process.env ONLY (config/env.ts zod parse — no dotenv
571      # anywhere in apps/api), so the "Prepare .env" step's INTERNAL_SERVICE_TOKEN /
572      # INTERNAL_TEST_SEAM / RATE_LIMIT_AUTH never reached it: the internal
573      # seam was fail-closed in CI (401 on every /internal/* call, breaking the
574      # two-user specs) and the auth limiter stayed at 10/min/IP. Mirroring
575      # them here is what actually opens the seam (TASK-0095 S9 repair, found
576      # while building the TASK-0122 AI seam).
577      INTERNAL_SERVICE_TOKEN: ci-e2e-internal-service-token
578      INTERNAL_TEST_SEAM: "1"
579      RATE_LIMIT_AUTH: "100000"
580      # TASK-0122 (Ask AI E2E): run the deterministic mock provider (no real
581      # provider keys in CI) + the 8-dim mock embedding space. The API reads
582      # process.env directly (config/env.ts) — the job env is what reaches it.
583      AI_PROVIDER: mock
584      SEARCH_VECTOR_DIMENSION: "8"
585      # 60 ms between mock deltas so the panel's streaming state (the live
586      # message grows word by word) is observable in the E2E spec — without
587      # it a 7-word answer arrives in one network round-trip and the
588      # "≥3 incremental updates" assertion can't distinguish it from a
589      # single-shot render.
590      AI_MOCK_DELTA_DELAY_MS: "60"
591    steps:
592      - <<: *setup
593      - name: Install Playwright browser
594        # The e2e package's OWN pinned 1.49.1 binary (the workspace member's
595        # lockfile entry — TASK-0095 S13). `npx playwright …` was pulling an
596        # unrelated latest Playwright whose expected browser revision did not
597        # match the 1.49.1 runtime the specs are written against (the e2e
598        # job had no working install path at all until the workspace fix —
599        # recorded in the S13 record).
600        run: pnpm --dir tests/e2e exec playwright install --with-deps chromium
601      - name: Wait for OpenSearch
602        run: |
603          for i in $(seq 1 60); do
604            if curl -sf http://localhost:9200/_cluster/health >/dev/null 2>&1; then
605              echo "opensearch is up after ${i} attempts"
606              exit 0
607            fi
608            sleep 5
609          done
610          echo "opensearch did not start in 300s" >&2
611          exit 1
612      - name: Create MinIO bucket + OpenSearch index
613        run: |
614          aws s3api create-bucket \
615            --endpoint-url http://localhost:9000 \
616            --bucket factory-uploads \
617            --region us-east-1 || true
618          curl -sf -X PUT "http://localhost:9200/factory-docs" \
619            -H 'Content-Type: application/json' -d '{}' || true
620      - name: Prepare .env for the app stack
621        run: |
622          # Start from .env.example; only connection targets change for CI.
623          cp .env.example .env
624          grep -v -e '^DATABASE_URL=' -e '^REDIS_URL=' -e '^SEARCH_URL=' \
625            -e '^S3_ENDPOINT=' -e '^S3_ACCESS_KEY=' -e '^S3_SECRET_KEY=' \
626            -e '^JWT_SECRET=' -e '^REFRESH_TOKEN_SECRET=' \
627            -e '^SESSION_SECRET=' -e '^MASTER_ENCRYPTION_KEY=' \
628            .env.example > /tmp/env.partial || true
629          {
630            cat /tmp/env.partial
631            echo "DATABASE_URL=postgresql://postgres:postgres@localhost:5432/factory?schema=public"
632            echo "REDIS_URL=redis://localhost:6379"
633            echo "SEARCH_URL=http://localhost:9200"
634            echo "S3_ENDPOINT=http://localhost:9000"
635            echo "S3_ACCESS_KEY=minioadmin"
636            echo "S3_SECRET_KEY=minioadmin"
637            echo "JWT_SECRET=ci-e2e-jwt-secret"
638            echo "REFRESH_TOKEN_SECRET=ci-e2e-refresh-secret"
639            echo "SESSION_SECRET=ci-e2e-session-secret"
640            echo "MASTER_ENCRYPTION_KEY=Y2ktZTJlLW1hc3Rlci1rZXktMzJieXRlcyEh"
641            # Test-only two-user seam (TASK-0095 S9): opens POST
642            # /internal/workspaces/:id/members (service-token guarded). The
643            # e2e specs read the same token from E2E_INTERNAL_SERVICE_TOKEN
644            # (job env below) to add members without an invite mail (D3).
645            echo "INTERNAL_SERVICE_TOKEN=ci-e2e-internal-service-token"
646            echo "INTERNAL_TEST_SEAM=1"
647            # The e2e specs perform ~11 fresh logins/registrations from one
648            # test IP in <60 s (per-test api + browser-context logins). The
649            # product default RATE_LIMIT_AUTH is 10/min per IP, so the final
650            # test's auth 429s (RATE_LIMITED). A throwaway test IP is not a
651            # brute-forcer — raise the auth limit for the test stack only,
652            # mirroring the perf job's RATE_LIMIT_API override.
653            echo "RATE_LIMIT_AUTH=100000"
654          } > .env
655      - name: Prisma generate + migrate + seed
656        run: |
657          pnpm db:generate
658          pnpm db:deploy
659          pnpm db:seed
660      - name: Start full stack (turbo dev, background)
661        run: |
662          set -o pipefail
663          # Run the whole dev stack (web 3000, api 4000, worker, realtime 4001)
664          # in the background, then wait for every port to accept connections.
665          nohup pnpm dev > /tmp/factory-dev.log 2>&1 &
666          echo $! > /tmp/factory-dev.pid
667          wait_for_port() {
668            local port="$1" name="$2"
669            for i in $(seq 1 90); do
670              if (exec 3<>"/dev/tcp/127.0.0.1/${port}") 2>/dev/null; then
671                exec 3>&- 3<&- || true
672                echo "${name} is up on :${port} after ${i}s"
673                return 0
674              fi
675              sleep 1
676            done
677            echo "${name} did not come up on :${port} in 90s" >&2
678            tail -n 100 /tmp/factory-dev.log >&2 || true
679            return 1
680          }
681          wait_for_port 4000 api
682          wait_for_port 3000 web
683          wait_for_port 4001 realtime
684      - name: Run E2E suite
685        run: pnpm test:e2e
686      - name: Upload Playwright report
687        if: failure()
688        uses: actions/upload-artifact@v4
689        with:
690          name: playwright-report
691          path: |
692            tests/e2e/playwright-report/**
693            tests/e2e/test-results/**
694          if-no-files-found: ignore
695          retention-days: 14
696      - name: Stop dev stack
697        if: always()
698        run: |
699          if [ -f /tmp/factory-dev.pid ]; then
700            kill "$(cat /tmp/factory-dev.pid)" 2>/dev/null || true
701            pkill -f "turbo" 2>/dev/null || true
702            pkill -f "next dev" 2>/dev/null || true
703          fi
704          tail -n 50 /tmp/factory-dev.log || true
705 
706  # ============================================================================
707  # perf — TASK-0095 S14: the realtime 1k load baseline (R4) + the per-PR smoke.
708  #
709  # Self-contained (does NOT need [integration, security]): the k6 load test
710  # targets ONLY the realtime gateway (:4001) + Postgres + Redis — the
711  # realtime service boots with the Prisma doc store directly (no
712  # INTERNAL_SERVICE_TOKEN → no HttpDocStore → the API is not required).
713  # k6 is a Go binary (not an npm dep): pinned to v0.57.0, the first release
714  # whose goja engine parses this suite's `1_000` numeric separators (verified
715  # `k6 archive`-clean in S14 — the earlier `ramping-vusers` typo was caught
716  # the same way). The DoD thresholds (p95<200 ms, 0 protocol errors) are
717  # enforced INSIDE the script's `options.thresholds` — a threshold breach
718  # makes `k6 run` exit non-zero, so the lane FAILS.
719  #
720  # Tiers: SMOKE (100 conns / 30 s) on every PR; the FULL 1k baseline (1,000
721  # conns, 50/30/20 mix, 3 min) on push to develop (the merge gate) — the
722  # expensive lane does not run on every PR push.
723  # ============================================================================
724  perf:
725    name: Perf (k6 realtime)
726    runs-on: ubuntu-24.04
727    timeout-minutes: 25
728    services:
729      postgres:
730        image: postgres:16-alpine
731        env:
732          POSTGRES_USER: postgres
733          POSTGRES_PASSWORD: postgres
734          POSTGRES_DB: factory
735        ports:
736          - 5432:5432
737        options: >-
738          --health-cmd "pg_isready -U postgres -d factory"
739          --health-interval 5s
740          --health-timeout 5s
741          --health-retries 12
742      redis:
743        image: redis:7-alpine
744        ports:
745          - 6379:6379
746        options: >-
747          --health-cmd "redis-cli ping"
748          --health-interval 5s
749          --health-timeout 3s
750          --health-retries 12
751    env:
752      DATABASE_URL: postgresql://postgres:postgres@localhost:5432/factory?schema=public
753      REDIS_URL: redis://localhost:6379
754      JWT_SECRET: ci-perf-jwt-secret
755      REALTIME_TRUST_PROXY: "1" # per-VU X-Forwarded-For connect-limiter identity
756      K6_REALTIME_URL: ws://localhost:4001
757      K6_JWT_SECRET: ci-perf-jwt-secret
758      # The seeder's deterministic load workspace + its 10 load pages (100
759      # peers/room). The k6 script's setup() THROWS when K6_PAGE_IDS is
760      # empty, so these are job env (not per-step).
761      K6_WORKSPACE_ID: load-ws
762      K6_PAGE_IDS: load-p-000,load-p-001,load-p-002,load-p-003,load-p-004,load-p-005,load-p-006,load-p-007,load-p-008,load-p-009
763    steps:
764      - <<: *setup
765      - name: Install k6 v0.57.0 (pinned — parses the suite's numeric separators)
766        run: |
767          K6_VER=v0.57.0
768          curl -sfL "https://github.com/grafana/k6/releases/download/${K6_VER}/k6-${K6_VER}-linux-amd64.tar.gz" -o /tmp/k6.tgz
769          tar xzf /tmp/k6.tgz -C /tmp
770          mv "/tmp/k6-${K6_VER}-linux-amd64/k6" /usr/local/bin/k6
771          chmod +x /usr/local/bin/k6
772          k6 version
773      - name: Prisma generate + migrate
774        run: pnpm db:generate && pnpm db:deploy
775      - name: Seed the 1,000 load identities (workspace + pages + EDIT grants)
776        run: |
777          K6_WORKSPACE_ID=load-ws K6_USERS=1000 node tests/performance/helpers/seed-1k-load.mjs
778      - name: Archive-check the suite (k6 parse gate)
779        run: pnpm --dir tests/performance check:1k
780      - name: Build the realtime service (node dist/index.js start)
781        run: pnpm --filter @factory/realtime build
782      - name: Boot realtime (Prisma doc store, no API required)
783        run: |
784          nohup pnpm --dir apps/realtime start > /tmp/realtime.log 2>&1 &
785          echo $! > /tmp/realtime.pid
786          for i in $(seq 1 90); do
787            if curl -sf http://localhost:4002/health >/dev/null 2>&1; then
788              echo "realtime health up on :4002 after ${i}s"
789              exit 0
790            fi
791            sleep 1
792          done
793          echo "realtime did not come up in 90s" >&2
794          tail -n 100 /tmp/realtime.log >&2 || true
795          exit 1
796      - name: Smoke (100 conns / 30 s — every PR)
797        run: pnpm --dir tests/performance realtime:smoke
798      - name: Full 1k baseline (push to develop/main — the merge gate)
799        if: github.event_name == 'push' && (github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/main')
800        run: pnpm --dir tests/performance realtime:1k
801      - name: TASK-0135 realtime-load smoke (new 10k suite — report-only, every PR)
802        # The new TASK-0135 suite (realtime-load.js via run-realtime-load.sh) in
803        # SMOKE mode: 100 conns / 10 s ramp / 20 s hold, report-only
804        # (K6_STRICT=0) and K6_VERIFY=0 (skip the host-side zero-op-loss check —
805        # that is the hard gate, which bites in staging). It seeds its OWN
806        # 100-user fixture (load-ws-smoke + smoke-p-* + smoke-verify) — distinct
807        # from the load-ws/1k seed above — so it is self-contained. The hard
808        # 10k run (K6_PROFILE=full, K6_STRICT=1, K6_VERIFY=1) runs in
809        # perf-staging-hard on the self-hosted runner.
810        # NOTE: K6_WORKSPACE_ID is overridden to load-ws-smoke — the job env
811        # carries load-ws (for the old realtime-1k.js suite), and the smoke
812        # runner would otherwise reuse it and not find its smoke-p-* pages.
813        run: |
814          K6_WORKSPACE_ID=load-ws-smoke K6_PROFILE=smoke K6_STRICT=0 K6_VERIFY=0 \
815          K6_JWT_SECRET=ci-perf-jwt-secret bash tests/performance/run-realtime-load.sh
816      - name: Upload the perf summary artifact
817        if: always()
818        uses: actions/upload-artifact@v4
819        with:
820          name: perf-realtime-1k
821          path: |
822            tests/performance/realtime-1k-final.json
823            tests/performance/realtime-smoke.json
824            tests/performance/reports/realtime-load.json
825          if-no-files-found: ignore
826          retention-days: 30
827      - name: Stop realtime
828        if: always()
829        run: |
830          if [ -f /tmp/realtime.pid ]; then kill "$(cat /tmp/realtime.pid)" 2>/dev/null || true; fi
831          pkill -f "apps/realtime" 2>/dev/null || true
832          tail -n 30 /tmp/realtime.log || true
833 
834  # ============================================================================
835  # perf-search — TASK-0100 S7: the 100k-doc search query performance lane.
836  #
837  # Self-contained (does NOT need [integration, security]): the k6 suite
838  # targets ONLY the API's POST /api/v1/search + Postgres + Redis +
839  # OpenSearch. The API boots with tsx (dev runtime — this is a perf lane,
840  # not the production build) against the fixture corpus seeded directly
841  # into OpenSearch by tests/performance/fixtures/search-100k.ts (100,000
842  # docs + a minimal 2-ws/200-user DB footprint for real HS256 JWTs).
843  #
844  # Tiers (per the Phase-8 task doc):
845  #   - REDUCED profile (10 VUs / 60 s / 10 req/s, 2x-flake thresholds) on
846  #     push to main — the merge gate inside the CI contract.
847  #   - FULL profile (100 VUs / 5 min / 50 req/s, DoD thresholds
848  #     p95<500 ms, p99<1 s, err<0.1%) on the nightly schedule (and manual
849  #     workflow_dispatch). The DoD thresholds are enforced INSIDE the
850  #     script's options.thresholds — a breach makes `k6 run` exit
851  #     non-zero, so the lane FAILS.
852  #
853  # The suite writes tests/performance/reports/search-last.json (p50/p95/p99,
854  # error rate, doc count, env fingerprint) — uploaded as an artifact, not
855  # committed (the task doc: "committed on demand by a maintainer, not
856  # per-PR").
857  # ============================================================================
858  perf-search:
859    name: Perf (k6 search 100k)
860    runs-on: ubuntu-24.04
861    timeout-minutes: 25
862    if: >-
863      github.event_name == 'schedule' ||
864      github.event_name == 'workflow_dispatch' ||
865      (github.event_name == 'push' && github.ref == 'refs/heads/main')
866    services:
867      postgres:
868        image: postgres:16-alpine
869        env:
870          POSTGRES_USER: postgres
871          POSTGRES_PASSWORD: postgres
872          POSTGRES_DB: factory
873        ports:
874          - 5432:5432
875        options: >-
876          --health-cmd "pg_isready -U postgres -d factory"
877          --health-interval 5s
878          --health-timeout 5s
879          --health-retries 12
880      redis:
881        image: redis:7-alpine
882        ports:
883          - 6379:6379
884        options: >-
885          --health-cmd "redis-cli ping"
886          --health-interval 5s
887          --health-timeout 3s
888          --health-retries 12
889      opensearch:
890        image: opensearchproject/opensearch:2
891        env:
892          discovery.type: single-node
893          plugins.security.disabled: "true"
894          OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m"
895        ports:
896          - 9200:9200
897    env:
898      # Same variable names as the integration job; targets = GitHub
899      # service containers.
900      DATABASE_URL: postgresql://postgres:postgres@localhost:5432/factory?schema=public
901      REDIS_URL: redis://localhost:6379
902      SEARCH_URL: http://localhost:9200
903      SEARCH_USERNAME: ""
904      SEARCH_PASSWORD: ""
905      SEARCH_INDEX_PREFIX: factory
906      S3_ENDPOINT: http://localhost:9000
907      S3_ACCESS_KEY: minioadmin
908      S3_SECRET_KEY: minioadmin
909      S3_BUCKET: factory-uploads
910      S3_REGION: us-east-1
911      S3_FORCE_PATH_STYLE: "true"
912      # Test-only secrets (never real values). K6_JWT_SECRET MUST equal the
913      # API's JWT_SECRET: the k6 suite mints HS256 access tokens for the
914      # fixture perf users under this shared secret.
915      JWT_SECRET: ci-perf-search-jwt-secret
916      REFRESH_TOKEN_SECRET: ci-perf-search-refresh-secret
917      SESSION_SECRET: ci-perf-search-session-secret
918      MASTER_ENCRYPTION_KEY: "ZmFjdG9yeS1zZWFyY2gtY2ktbWFzdGVyLWtleS0zMmJ5dGVzISE="
919      # The k6 script's header mandates a high per-user API rate limit for
920      # load runs (default is 100/min per user).
921      RATE_LIMIT_API: "100000"
922      K6_API_URL: http://localhost:4000
923      K6_JWT_SECRET: ci-perf-search-jwt-secret
924      K6_SEARCH_WORKSPACES: perf-search-a,perf-search-b
925      K6_VERSION: ci
926    steps:
927      - <<: *setup
928      - name: Install k6 v0.57.0 (pinned — parses the suite's numeric separators)
929        run: |
930          K6_VER=v0.57.0
931          curl -sfL "https://github.com/grafana/k6/releases/download/${K6_VER}/k6-${K6_VER}-linux-amd64.tar.gz" -o /tmp/k6.tgz
932          tar xzf /tmp/k6.tgz -C /tmp
933          mv "/tmp/k6-${K6_VER}-linux-amd64/k6" /usr/local/bin/k6
934          chmod +x /usr/local/bin/k6
935          k6 version
936      - name: Wait for OpenSearch
937        # Same retry pattern as the integration job: the image needs well
938        # over a minute on a busy runner and has no reliable health option.
939        run: |
940          for i in $(seq 1 60); do
941            if curl -sf http://localhost:9200/_cluster/health >/dev/null 2>&1; then
942              echo "opensearch is up after ${i} attempts"
943              exit 0
944            fi
945            sleep 5
946          done
947          echo "opensearch did not start in 300s" >&2
948          exit 1
949      - name: Prisma generate + migrate
950        run: pnpm db:generate && pnpm db:deploy
951      - name: Archive-check the suite (k6 parse gate)
952        run: pnpm --dir tests/performance check:search
953      - name: Seed the 100,000-doc search fixture (2 ws, 200 users, 2 teams)
954        # Idempotent (reset first, deterministic _id upserts); the seeder
955        # verifies the landed count is exactly 100,000 before finishing.
956        run: pnpm --dir tests/performance perf:seed:search
957      - name: Boot the API (the search endpoint under test)
958        # tsx dev runtime from apps/api (perf lane, not the prod build).
959        # Job env (DATABASE_URL/REDIS_URL/SEARCH_URL/…/RATE_LIMIT_API) is
960        # inherited by the background process.
961        run: |
962          cd apps/api
963          nohup node_modules/.bin/tsx src/main.ts > /tmp/perf-search-api.log 2>&1 &
964          echo $! > /tmp/perf-search-api.pid
965          for i in $(seq 1 90); do
966            if curl -sf http://localhost:4000/api/v1/health >/dev/null 2>&1; then
967              echo "api is up on :4000 after ${i}s"
968              exit 0
969            fi
970            sleep 1
971          done
972          echo "api did not come up in 90s" >&2
973          tail -n 100 /tmp/perf-search-api.log >&2 || true
974          exit 1
975      - name: Search perf — FULL profile (100 VUs / 5 min / 50 req/s, DoD thresholds; nightly + manual)
976        if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
977        run: pnpm --dir tests/performance search:100k
978      - name: Search perf — REDUCED profile (10 VUs / 60 s / 10 req/s; merge-to-main gate)
979        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
980        run: pnpm --dir tests/performance search:100k:reduced
981      - name: Upload the perf report artifact
982        if: always()
983        uses: actions/upload-artifact@v4
984        with:
985          name: perf-search-100k
986          path: |
987            tests/performance/reports/search-last.json
988            tests/performance/reports/search-full.json
989            tests/performance/reports/search-reduced.json
990            tests/performance/search-summary.json
991          if-no-files-found: ignore
992          retention-days: 30
993      - name: Stop the API
994        if: always()
995        run: |
996          if [ -f /tmp/perf-search-api.pid ]; then
997            kill "$(cat /tmp/perf-search-api.pid)" 2>/dev/null || true
998          fi
999          pkill -f "apps/api" 2>/dev/null || true
1000          tail -n 30 /tmp/perf-search-api.log || true
1001 
1002  # ============================================================================
1003  # perf-load — TASK-0135: the API + database 100k-record load lane.
1004  #
1005  # Self-contained (does NOT need [integration, security]): the k6 suites
1006  # (api-load.js, database-load.js) target ONLY the API's page-tree / record-
1007  # list / record-create endpoints + Postgres + Redis. The API boots with tsx
1008  # (dev runtime — this is a perf lane, not the production build, the same
1009  # pattern as perf-search) against the 100k-record fixture from
1010  # seed-100k.mjs (one database / 100,000 records + 200 hub pages + one
1011  # 10k-block page). The seeder writes a `seed-100k-v1` marker row into the
1012  # `test_seeds` table; extract-100k-env.mjs reads it back into K6_WORKSPACE_ID
1013  # / K6_DB_ID / K6_PAGE_IDS / K6_BIG_PAGE_ID — the ids the suites' setup()
1014  # requires (they throw when empty).
1015  #
1016  # Tiers (spec §26: "report-only in dev, hard in staging" — API 1k rps
1017  # p95<300 ms; database 100k table p95<500 ms + 100 rps create):
1018  #   - THIS job (dev, ubuntu-24.04): SMOKE profiles, K6_STRICT=0 — the
1019  #     report-only tier. It proves the suites + endpoints work end-to-end and
1020  #     prints the real p95/p99/error values into the artifact; it NEVER
1021  #     blocks on a threshold. The dev runner is not sized for the 1k-rps /
1022  #     9-min full profile, so dev numbers are mechanics, not budgets.
1023  #   - perf-staging-hard (self-hosted, below): FULL profiles, K6_STRICT=1 —
1024  #     the hard tier; a breach makes the step FAIL (k6 exit 99).
1025  #
1026  # No OpenSearch service: the API only talks to OpenSearch on the search
1027  # endpoint, and this lane skips that scenario (K6_SKIP_SEARCH=1) — the 100k
1028  # fixture is inserted via Prisma directly (no indexing events) and the
1029  # content index is only ever created by the worker, which this lane does not
1030  # run. Search-query perf at the 100k-doc scale is the perf-search job's job.
1031  # No MinIO either: the file endpoints are outside this lane's scenarios
1032  # (S3_* set to the tolerated dead-endpoint pattern, like perf-search).
1033  # ============================================================================
1034  perf-load:
1035    name: Perf (k6 API+DB 100k — report-only)
1036    runs-on: ubuntu-24.04
1037    timeout-minutes: 40
1038    services:
1039      postgres:
1040        image: postgres:16-alpine
1041        env:
1042          POSTGRES_USER: postgres
1043          POSTGRES_PASSWORD: postgres
1044          POSTGRES_DB: factory
1045        ports:
1046          - 5432:5432
1047        options: >-
1048          --health-cmd "pg_isready -U postgres -d factory"
1049          --health-interval 5s
1050          --health-timeout 5s
1051          --health-retries 12
1052      redis:
1053        image: redis:7-alpine
1054        ports:
1055          - 6379:6379
1056        options: >-
1057          --health-cmd "redis-cli ping"
1058          --health-interval 5s
1059          --health-timeout 3s
1060          --health-retries 12
1061    env:
1062      DATABASE_URL: postgresql://postgres:postgres@localhost:5432/factory?schema=public
1063      REDIS_URL: redis://localhost:6379
1064      # Test-only secrets (never real values). K6_JWT_SECRET MUST equal the
1065      # API's JWT_SECRET: the k6 suites mint HS256 access tokens for the
1066      # fixture perf user under this shared secret.
1067      JWT_SECRET: ci-perf-load-jwt-secret
1068      REFRESH_TOKEN_SECRET: ci-perf-load-refresh-secret
1069      SESSION_SECRET: ci-perf-load-session-secret
1070      MASTER_ENCRYPTION_KEY: "ZmFjdG9yeS1zZWFyY2gtY2ktbWFzdGVyLWtleS0zMmJ5dGVzISE="
1071      # High per-user API + auth limits for the load run: each VU authenticates
1072      # once (cached), but the full profile's 2000 VUs would trip the default
1073      # 10/min per IP auth limiter (same override as perf-search / e2e).
1074      RATE_LIMIT_API: "100000"
1075      RATE_LIMIT_AUTH: "100000"
1076      S3_ENDPOINT: http://localhost:9000
1077      S3_ACCESS_KEY: minioadmin
1078      S3_SECRET_KEY: minioadmin
1079      S3_BUCKET: factory-uploads
1080      S3_REGION: us-east-1
1081      S3_FORCE_PATH_STYLE: "true"
1082      K6_API_URL: http://localhost:4000
1083      K6_JWT_SECRET: ci-perf-load-jwt-secret
1084      # The CI lane has no indexed corpus (see the header) → skip the search
1085      # scenario; the other three (tree-load / record-list / record-create)
1086      # carry the spec's API-load criteria.
1087      K6_SKIP_SEARCH: "1"
1088    steps:
1089      - <<: *setup
1090      - name: Install k6 v0.57.0 (pinned — parses the suite's numeric separators)
1091        run: |
1092          K6_VER=v0.57.0
1093          curl -sfL "https://github.com/grafana/k6/releases/download/${K6_VER}/k6-${K6_VER}-linux-amd64.tar.gz" -o /tmp/k6.tgz
1094          tar xzf /tmp/k6.tgz -C /tmp
1095          mv "/tmp/k6-${K6_VER}-linux-amd64/k6" /usr/local/bin/k6
1096          chmod +x /usr/local/bin/k6
1097          k6 version
1098      - name: Prisma generate + migrate
1099        run: pnpm db:generate && pnpm db:deploy
1100      - name: Archive-check the suites (k6 parse gate)
1101        run: pnpm --dir tests/performance check:load
1102      - name: Boot the API (the endpoints under test)
1103        # tsx dev runtime from apps/api (perf lane, not the prod build).
1104        # Job env (DATABASE_URL/REDIS_URL/RATE_LIMIT_*/…) is inherited by the
1105        # background process.
1106        run: |
1107          cd apps/api
1108          nohup node_modules/.bin/tsx src/main.ts > /tmp/perf-load-api.log 2>&1 &
1109          echo $! > /tmp/perf-load-api.pid
1110          for i in $(seq 1 90); do
1111            if curl -sf http://localhost:4000/api/v1/health >/dev/null 2>&1; then
1112              echo "api is up on :4000 after ${i}s"
1113              exit 0
1114            fi
1115            sleep 1
1116          done
1117          echo "api did not come up in 90s" >&2
1118          tail -n 100 /tmp/perf-load-api.log >&2 || true
1119          exit 1
1120      - name: Seed the 100k-record fixture (1 DB / 100k records / 200 hubs / 10k-block page)
1121        # Idempotent (marker row seed-100k-v1 in test_seeds; a no-op when the
1122        # marker matches). ~4-5 min. The API must be up: the seeder
1123        # registers/logs in the perf user via the API, then inserts the 100k
1124        # records directly via Prisma.
1125        run: pnpm --dir tests/performance seed:100k
1126      - name: Extract the seeded ids → K6_* env (from the marker row)
1127        run: |
1128          source <(node tests/performance/extract-100k-env.mjs)
1129          echo "K6_WORKSPACE_ID=$K6_WORKSPACE_ID" >> "$GITHUB_ENV"
1130          echo "K6_DB_ID=$K6_DB_ID" >> "$GITHUB_ENV"
1131          echo "K6_PAGE_IDS=$K6_PAGE_IDS" >> "$GITHUB_ENV"
1132          echo "K6_BIG_PAGE_ID=$K6_BIG_PAGE_ID" >> "$GITHUB_ENV"
1133      - name: "API load — SMOKE (report-only: proves the endpoints + suite mechanics)"
1134        # K6_STRICT=0 (the :smoke script) — the summary prints the real p95/p99
1135        # /error values but never fails the step.
1136        run: pnpm --dir tests/performance api-load:smoke
1137      - name: "Database load — SMOKE (report-only: 100k-table scroll/open + create)"
1138        run: pnpm --dir tests/performance database-load:smoke
1139      - name: Threshold-breach red-proof (a lowered threshold MUST turn the step red)
1140        # The spec requires the hard tier to actually BLOCK on a breach — so
1141        # the lane proves it: the demo suite runs against the live API with an
1142        # impossible p95 budget (K6_P95_MS=1) under K6_STRICT=1, so k6 exits
1143        # 99. This step inverts that: an exit OTHER than 99 (the demo
1144        # unexpectedly passing) fails the lane; the expected 99 keeps it green
1145        # and asserts the red tier is real.
1146        run: |
1147          set +e
1148          k6 run tests/performance/threshold-breach-demo.js \
1149            --env K6_API_URL=http://localhost:4000 \
1150            --env K6_STRICT=1 --env K6_P95_MS=1
1151          rc=$?
1152          set -e
1153          if [ "$rc" -ne 99 ]; then
1154            echo "breach demo exited ${rc} (expected 99) — the hard tier is NOT blocking on a threshold breach!" >&2
1155            exit 1
1156          fi
1157          echo "breach demo exited 99 as expected — the hard tier blocks on a threshold breach."
1158      - name: Upload the perf report artifact
1159        if: always()
1160        uses: actions/upload-artifact@v4
1161        with:
1162          name: perf-load-100k
1163          path: tests/performance/reports/
1164          if-no-files-found: ignore
1165          retention-days: 30
1166      - name: Stop the API
1167        if: always()
1168        run: |
1169          if [ -f /tmp/perf-load-api.pid ]; then
1170            kill "$(cat /tmp/perf-load-api.pid)" 2>/dev/null || true
1171          fi
1172          pkill -f "apps/api" 2>/dev/null || true
1173          tail -n 30 /tmp/perf-load-api.log || true
1174 
1175  # ============================================================================
1176  # perf-editor — TASK-0135: the editor E2E interaction-latency lane.
1177  #
1178  # editor-local.mjs drives a real headless Chromium against the FULL dev
1179  # stack (turbo dev → web :3000 + api :4000 + realtime :4001, the e2e job's
1180  # boot pattern) and measures the LOCAL (client-side, no network) interaction
1181  # latency — spec §26 "editor E2E interaction p95 < 50 ms + block move <
1182  # 16 ms (local)": type/create p95<50 ms and block-move p95<16 ms (plus a
1183  # frame budget). Report-only here (K6_STRICT=0 — the suite prints the real
1184  # p95/p99 and never fails the step); the hard tier is
1185  # perf-staging-hard's full-profile editor run on the self-hosted runner.
1186  #
1187  # The full 100k fixture is seeded first so the page tree / permission
1188  # lookups run against production-scale data (marker → K6_WORKSPACE_ID).
1189  #
1190  # NOTE (committed defect, parent task): the security headers (TASK-0134,
1191  # packages/shared/src/security-headers.ts) send
1192  # script-src 'self' 'unsafe-inline' with NO 'unsafe-eval', and Next 14 DEV
1193  # builds emit their chunks via eval() (devtool eval-source-map) — so the dev
1194  # web is unbootable under its own headers ("Loading workspace…" forever,
1195  # zero API calls). editor-local bypasses CSP on the test context
1196  # (bypassCSP: true) so it can still MEASURE; the proper fix (a dev-only
1197  # devtool override / 'unsafe-eval' carve-out) is an app change, out of scope
1198  # for TASK-0135. The e2e job's fixture does NOT bypass CSP — it hits the
1199  # same wall until the parent fixes the headers.
1200  # ============================================================================
1201  perf-editor:
1202    name: Perf (editor E2E — report-only)
1203    runs-on: ubuntu-24.04
1204    timeout-minutes: 35
1205    services:
1206      postgres:
1207        image: postgres:16-alpine
1208        env:
1209          POSTGRES_USER: postgres
1210          POSTGRES_PASSWORD: postgres
1211          POSTGRES_DB: factory
1212        ports:
1213          - 5432:5432
1214        options: >-
1215          --health-cmd "pg_isready -U postgres -d factory"
1216          --health-interval 5s
1217          --health-timeout 5s
1218          --health-retries 12
1219      redis:
1220        image: redis:7-alpine
1221        ports:
1222          - 6379:6379
1223        options: >-
1224          --health-cmd "redis-cli ping"
1225          --health-interval 5s
1226          --health-timeout 3s
1227          --health-retries 12
1228      minio:
1229        image: minio/minio
1230        command: server /data --console-address ":9001"
1231        env:
1232          MINIO_ROOT_USER: minioadmin
1233          MINIO_ROOT_PASSWORD: minioadmin
1234        ports:
1235          - 9000:9000
1236        options: >-
1237          --health-cmd "curl -fsS http://localhost:9000/minio/health/live"
1238          --health-interval 5s
1239          --health-timeout 5s
1240          --health-retries 12
1241      opensearch:
1242        image: opensearchproject/opensearch:2
1243        env:
1244          discovery.type: single-node
1245          plugins.security.disabled: "true"
1246          OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m"
1247        ports:
1248          - 9200:9200
1249    env:
1250      DATABASE_URL: postgresql://postgres:postgres@localhost:5432/factory?schema=public
1251      REDIS_URL: redis://localhost:6379
1252      SEARCH_URL: http://localhost:9200
1253      SEARCH_USERNAME: ""
1254      SEARCH_PASSWORD: ""
1255      SEARCH_INDEX_PREFIX: factory
1256      S3_ENDPOINT: http://localhost:9000
1257      S3_ACCESS_KEY: minioadmin
1258      S3_SECRET_KEY: minioadmin
1259      S3_BUCKET: factory-uploads
1260      S3_REGION: us-east-1
1261      S3_FORCE_PATH_STYLE: "true"
1262      JWT_SECRET: ci-perf-editor-jwt-secret
1263      REFRESH_TOKEN_SECRET: ci-perf-editor-refresh-secret
1264      SESSION_SECRET: ci-perf-editor-session-secret
1265      MASTER_ENCRYPTION_KEY: "Y2ktZTJlLW1hc3Rlci1rZXktMzJieXRlcyEh"
1266      INTERNAL_SERVICE_TOKEN: ci-perf-editor-internal-service-token
1267      INTERNAL_TEST_SEAM: "1"
1268      # The editor lane logs in (API + web cookie) and the dev stack's worker
1269      # is up; raise the auth limiter as in the e2e job.
1270      RATE_LIMIT_AUTH: "100000"
1271      # Deterministic mock AI provider (the full dev stack boots the AI
1272      # module; no real provider keys in CI) + 8-dim mock embedding space.
1273      AI_PROVIDER: mock
1274      SEARCH_VECTOR_DIMENSION: "8"
1275    steps:
1276      - <<: *setup
1277      - name: Install Playwright browser (tests/performance's pinned 1.49.1)
1278        run: pnpm --dir tests/performance exec playwright install --with-deps chromium
1279      - name: Wait for OpenSearch
1280        run: |
1281          for i in $(seq 1 60); do
1282            if curl -sf http://localhost:9200/_cluster/health >/dev/null 2>&1; then
1283              echo "opensearch is up after ${i} attempts"
1284              exit 0
1285            fi
1286            sleep 5
1287          done
1288          echo "opensearch did not start in 300s" >&2
1289          exit 1
1290      - name: Create MinIO bucket + OpenSearch index
1291        run: |
1292          aws s3api create-bucket \
1293            --endpoint-url http://localhost:9000 \
1294            --bucket factory-uploads \
1295            --region us-east-1 || true
1296          curl -sf -X PUT "http://localhost:9200/factory-docs" \
1297            -H 'Content-Type: application/json' -d '{}' || true
1298      - name: Prepare .env for the app stack
1299        run: |
1300          cp .env.example .env
1301          grep -v -e '^DATABASE_URL=' -e '^REDIS_URL=' -e '^SEARCH_URL=' \
1302            -e '^S3_ENDPOINT=' -e '^S3_ACCESS_KEY=' -e '^S3_SECRET_KEY=' \
1303            -e '^JWT_SECRET=' -e '^REFRESH_TOKEN_SECRET=' \
1304            -e '^SESSION_SECRET=' -e '^MASTER_ENCRYPTION_KEY=' \
1305            .env.example > /tmp/env.partial || true
1306          {
1307            cat /tmp/env.partial
1308            echo "DATABASE_URL=postgresql://postgres:postgres@localhost:5432/factory?schema=public"
1309            echo "REDIS_URL=redis://localhost:6379"
1310            echo "SEARCH_URL=http://localhost:9200"
1311            echo "S3_ENDPOINT=http://localhost:9000"
1312            echo "S3_ACCESS_KEY=minioadmin"
1313            echo "S3_SECRET_KEY=minioadmin"
1314            echo "JWT_SECRET=ci-perf-editor-jwt-secret"
1315            echo "REFRESH_TOKEN_SECRET=ci-perf-editor-refresh-secret"
1316            echo "SESSION_SECRET=ci-perf-editor-session-secret"
1317            echo "MASTER_ENCRYPTION_KEY=Y2ktZTJlLW1hc3Rlci1rZXktMzJieXRlcyEh"
1318            echo "INTERNAL_SERVICE_TOKEN=ci-perf-editor-internal-service-token"
1319            echo "INTERNAL_TEST_SEAM=1"
1320            echo "RATE_LIMIT_AUTH=100000"
1321            echo "AI_PROVIDER=mock"
1322            echo "SEARCH_VECTOR_DIMENSION=8"
1323          } > .env
1324      - name: Prisma generate + migrate + seed
1325        run: |
1326          pnpm db:generate
1327          pnpm db:deploy
1328          pnpm db:seed
1329      - name: Start full stack (turbo dev, background)
1330        run: |
1331          set -o pipefail
1332          nohup pnpm dev > /tmp/factory-dev-editor.log 2>&1 &
1333          echo $! > /tmp/factory-dev-editor.pid
1334          wait_for_port() {
1335            local port="$1" name="$2"
1336            for i in $(seq 1 120); do
1337              if (exec 3<>"/dev/tcp/127.0.0.1/${port}") 2>/dev/null; then
1338                exec 3>&- 3<&- || true
1339                echo "${name} is up on :${port} after ${i}s"
1340                return 0
1341              fi
1342              sleep 1
1343            done
1344            echo "${name} did not come up on :${port} in 120s" >&2
1345            tail -n 100 /tmp/factory-dev-editor.log >&2 || true
1346            return 1
1347          }
1348          wait_for_port 4000 api
1349          wait_for_port 3000 web
1350          wait_for_port 4001 realtime
1351      - name: Seed the 100k-record fixture (production-scale data for the editor)
1352        run: pnpm --dir tests/performance seed:100k
1353      - name: Extract the seeded workspace → K6_WORKSPACE_ID (from the marker row)
1354        run: |
1355          source <(node tests/performance/extract-100k-env.mjs)
1356          echo "K6_WORKSPACE_ID=$K6_WORKSPACE_ID" >> "$GITHUB_ENV"
1357      - name: "Editor-local — SMOKE (report-only: real p95/p99, never blocks)"
1358        # K6_STRICT=0 — the suite measures type/create/move against the dev
1359        # web and prints the report; it only fails on infra errors (no stack).
1360        # See the CSP note in the job header (bypassCSP inside the suite).
1361        run: K6_PROFILE=smoke K6_STRICT=0 pnpm --dir tests/performance editor-local
1362      - name: Upload the editor perf report artifact
1363        if: always()
1364        uses: actions/upload-artifact@v4
1365        with:
1366          name: perf-editor-local
1367          path: |
1368            tests/performance/reports/editor-local.json
1369          if-no-files-found: ignore
1370          retention-days: 30
1371      - name: Stop dev stack
1372        if: always()
1373        run: |
1374          if [ -f /tmp/factory-dev-editor.pid ]; then
1375            kill "$(cat /tmp/factory-dev-editor.pid)" 2>/dev/null || true
1376            pkill -f "turbo" 2>/dev/null || true
1377            pkill -f "next dev" 2>/dev/null || true
1378          fi
1379          tail -n 50 /tmp/factory-dev-editor.log || true
1380 
1381  # ============================================================================
1382  # perf-staging-hard — TASK-0135: the HARD perf tier (spec: "hard in staging").
1383  #
1384  # Runs ONLY via `workflow_dispatch` with perf_mode=staging-hard, on the
1385  # self-hosted staging runner (phase-15 §CI L47: "self-hosted runner (4 vCPU)
1386  # for the local editor perf test"). FULL profiles, K6_STRICT=1: a threshold
1387  # breach makes the step FAIL (k6 exit 99), and the realtime run keeps
1388  # K6_VERIFY=1 — the host-side zero-dropped-Yjs-ops check immediately after
1389  # the 10k run (the real DoD gate; it passes only once the apps/realtime
1390  # permission-resolver bug is fixed, so it is a meaningful blocker here).
1391  #
1392  # What this lane boots (self-contained on the 4-vCPU runner):
1393  #   - infra via the repo's dev compose (infrastructure/docker/): postgres,
1394  #     redis, minio, opensearch (+ their one-shot init). GitHub `services:`
1395  #     are unavailable on self-hosted runners, so the canonical dev/staging-
1396  #     like infra is started with `docker compose` (only the infra + one-shot
1397  #     init services — the "apps"-profile services are NOT started; the apps
1398  #     boot from source below, the e2e job's pattern).
1399  #   - the full app stack from source (`pnpm dev`): api :4000 + web :3000 +
1400  #     realtime :4001 + worker — so the editor measurement is CLIENT-SIDE
1401  #     against the locally-served web (the spec's "local" / "self-hosted …
1402  #     local editor perf test"), and the api/realtime/database suites load the
1403  #     same production-shaped stack at full profile.
1404  #   - the 100k-record fixture (seed-100k.mjs, K6_DBS=1 → ONE 100k table, the
1405  #     spec's "database 100k records table" criterion) + the 100k-doc search
1406  #     corpus (perf:seed:search → the factory-content index), so api-load runs
1407  #     its TRUE 1k-rps matrix INCLUDING the 200-rps search scenario (no
1408  #     K6_SKIP_SEARCH here — that is the report-only-lane corpus workaround).
1409  #
1410  # The four suites, all FULL + K6_STRICT=1 (a breach blocks the lane):
1411  #   api-load       1k rps (400 tree / 300 list / 200 search / 100 create),
1412  #                  overall + per-scenario p95 < 300 ms, err < 0.1%
1413  #   database-load  100k-table scroll/open + 100 rps create, p95 < 500 ms
1414  #   realtime-load  10k conns / 200 msg/s, p95 < 200 ms + zero-op-loss verify
1415  #   editor-local   interaction p95 < 50 ms + block move < 16 ms (local)
1416  #
1417  # Provisioning (a self-hosted runner must exist before this job can run):
1418  #   - a Linux runner labeled `factory-staging` (4 vCPU, per spec L47) with
1419  #     Docker available (for the infra `docker compose` step).
1420  #
1421  # NOTE (Phase-15 release-pipeline target): the release pipeline's stage 8
1422  # `performance` (infra/pipeline/release.yml — ABSENT in this repo) runs these
1423  # same four suites reading the DEPLOYED staging after the stage-6 rollout is
1424  # Ready (release-pipeline-only cluster; the dev/PR pipeline never touches
1425  # staging). This ci.yml job is the in-repo, self-contained equivalent: the
1426  # runnable hard tier on the self-hosted runner, and what release-pipeline
1427  # stage 8 invokes (or a target-staging variant of it). Until a runner with
1428  # the label above is provisioned, this job is wired-and-correct-in-shape but
1429  # fails fast at the infra step (no matching runner / no docker).
1430  # ============================================================================
1431  perf-staging-hard:
1432    name: Perf (staging — HARD tier, self-hosted)
1433    runs-on: [self-hosted, factory-staging] # label per spec L47 — provision a 4-vCPU runner with this label + Docker
1434    timeout-minutes: 120
1435    if: github.event_name == 'workflow_dispatch' && inputs.perf_mode == 'staging-hard'
1436    env:
1437      # Compose infra defaults (factory:factory/factory — see
1438      # infrastructure/docker/docker-compose.yml). The app stack reads the
1439      # prepared .env + these job env values.
1440      DATABASE_URL: postgresql://factory:factory@localhost:5432/factory?schema=public
1441      REDIS_URL: redis://localhost:6379
1442      SEARCH_URL: http://localhost:9200
1443      SEARCH_USERNAME: ""
1444      SEARCH_PASSWORD: ""
1445      SEARCH_INDEX_PREFIX: factory
1446      S3_ENDPOINT: http://localhost:9000
1447      S3_ACCESS_KEY: minioadmin
1448      S3_SECRET_KEY: minioadmin
1449      S3_BUCKET: factory-uploads
1450      S3_REGION: us-east-1
1451      S3_FORCE_PATH_STYLE: "true"
1452      # Test-only secrets (never real values). K6_JWT_SECRET MUST equal the app
1453      # JWT_SECRET: the k6 suites mint HS256 tokens for the perf users.
1454      JWT_SECRET: ci-perf-staging-hard-jwt-secret
1455      REFRESH_TOKEN_SECRET: ci-perf-staging-hard-refresh-secret
1456      SESSION_SECRET: ci-perf-staging-hard-session-secret
1457      MASTER_ENCRYPTION_KEY: "ZmFjdG9yeS1zZWFyY2gtY2ktbWFzdGVyLWtleS0zMmJ5dGVzISE="
1458      INTERNAL_SERVICE_TOKEN: ci-perf-staging-hard-internal-service-token
1459      INTERNAL_TEST_SEAM: "1"
1460      REALTIME_TRUST_PROXY: "1" # per-VU X-Forwarded-For connect-limiter identity
1461      RATE_LIMIT_API: "100000"
1462      RATE_LIMIT_AUTH: "100000"
1463      AI_PROVIDER: mock
1464      SEARCH_VECTOR_DIMENSION: "8"
1465      K6_API_URL: http://localhost:4000
1466      K6_REALTIME_URL: ws://localhost:4001
1467      K6_JWT_SECRET: ci-perf-staging-hard-jwt-secret
1468    steps:
1469      - <<: *setup
1470      - name: Install k6 v0.57.0 (pinned — parses the suite's numeric separators)
1471        run: |
1472          K6_VER=v0.57.0
1473          curl -sfL "https://github.com/grafana/k6/releases/download/${K6_VER}/k6-${K6_VER}-linux-amd64.tar.gz" -o /tmp/k6.tgz
1474          tar xzf /tmp/k6.tgz -C /tmp
1475          mv "/tmp/k6-${K6_VER}-linux-amd64/k6" /usr/local/bin/k6
1476          chmod +x /usr/local/bin/k6
1477          k6 version
1478      - name: "Start infra (dev compose: postgres, redis, minio, opensearch)"
1479        # GitHub `services:` are unavailable on self-hosted runners → start the
1480        # repo's canonical dev/staging-like infra with docker compose. Only the
1481        # infra + one-shot init services (minio-init / opensearch-init,
1482        # restart:"no") — the "apps"-profile services are NOT started here.
1483        run: |
1484          COMPOSE="docker compose -f infrastructure/docker/docker-compose.yml"
1485          $COMPOSE up -d postgres redis minio opensearch minio-init opensearch-init
1486          wait_http() { # url name tries
1487            for i in $(seq 1 "$3"); do
1488              if curl -sf "$1" >/dev/null 2>&1; then echo "$2 is up after ${i} checks"; return 0; fi
1489              sleep 2
1490            done
1491            echo "$2 did not come up in $(( $3 * 2 ))s" >&2; return 1
1492          }
1493          wait_http "http://localhost:9200/_cluster/health" "opensearch" 150 # ~5 min (slow JVM start)
1494          wait_http "http://localhost:9000/minio/health/live" "minio" 60
1495          for i in $(seq 1 90); do
1496            if $COMPOSE exec -T postgres pg_isready -U factory -d factory >/dev/null 2>&1; then
1497              echo "postgres is ready after ${i}s"; break
1498            fi; sleep 1
1499          done
1500          for i in $(seq 1 90); do
1501            if $COMPOSE exec -T redis redis-cli ping 2>/dev/null | grep -q PONG; then
1502              echo "redis is ready after ${i}s"; break
1503            fi; sleep 1
1504          done
1505          $COMPOSE ps
1506      - name: Prepare .env for the app stack
1507        run: |
1508          cp .env.example .env
1509          grep -v -e '^DATABASE_URL=' -e '^REDIS_URL=' -e '^SEARCH_URL=' \
1510            -e '^S3_ENDPOINT=' -e '^S3_ACCESS_KEY=' -e '^S3_SECRET_KEY=' \
1511            -e '^JWT_SECRET=' -e '^REFRESH_TOKEN_SECRET=' \
1512            -e '^SESSION_SECRET=' -e '^MASTER_ENCRYPTION_KEY=' \
1513            .env.example > /tmp/env.partial || true
1514          {
1515            cat /tmp/env.partial
1516            echo "DATABASE_URL=postgresql://factory:factory@localhost:5432/factory?schema=public"
1517            echo "REDIS_URL=redis://localhost:6379"
1518            echo "SEARCH_URL=http://localhost:9200"
1519            echo "SEARCH_USERNAME="
1520            echo "SEARCH_PASSWORD="
1521            echo "SEARCH_INDEX_PREFIX=factory"
1522            echo "S3_ENDPOINT=http://localhost:9000"
1523            echo "S3_ACCESS_KEY=minioadmin"
1524            echo "S3_SECRET_KEY=minioadmin"
1525            echo "S3_BUCKET=factory-uploads"
1526            echo "S3_REGION=us-east-1"
1527            echo "S3_FORCE_PATH_STYLE=true"
1528            echo "JWT_SECRET=ci-perf-staging-hard-jwt-secret"
1529            echo "REFRESH_TOKEN_SECRET=ci-perf-staging-hard-refresh-secret"
1530            echo "SESSION_SECRET=ci-perf-staging-hard-session-secret"
1531            echo "MASTER_ENCRYPTION_KEY=ZmFjdG9yeS1zZWFyY2gtY2ktbWFzdGVyLWtleS0zMmJ5dGVzISE="
1532            echo "INTERNAL_SERVICE_TOKEN=ci-perf-staging-hard-internal-service-token"
1533            echo "INTERNAL_TEST_SEAM=1"
1534            echo "REALTIME_TRUST_PROXY=1"
1535            echo "RATE_LIMIT_API=100000"
1536            echo "RATE_LIMIT_AUTH=100000"
1537            echo "AI_PROVIDER=mock"
1538            echo "SEARCH_VECTOR_DIMENSION=8"
1539          } > .env
1540      - name: Prisma generate + migrate + seed
1541        run: |
1542          pnpm db:generate
1543          pnpm db:deploy
1544          pnpm db:seed
1545      - name: Start full app stack from source (turbo dev, background)
1546        run: |
1547          set -o pipefail
1548          nohup pnpm dev > /tmp/factory-dev-staging.log 2>&1 &
1549          echo $! > /tmp/factory-dev-staging.pid
1550          wait_for_port() {
1551            local port="$1" name="$2"
1552            for i in $(seq 1 120); do
1553              if (exec 3<>"/dev/tcp/127.0.0.1/${port}") 2>/dev/null; then
1554                exec 3>&- 3<&- || true
1555                echo "${name} is up on :${port} after ${i}s"
1556                return 0
1557              fi
1558              sleep 1
1559            done
1560            echo "${name} did not come up on :${port} in 120s" >&2
1561            tail -n 100 /tmp/factory-dev-staging.log >&2 || true
1562            return 1
1563          }
1564          wait_for_port 4000 api
1565          wait_for_port 3000 web
1566          wait_for_port 4001 realtime
1567      - name: Seed the 100k-record fixture (K6_DBS=1 → one 100k table)
1568        # K6_DBS=1 keeps a single 100k-record table (the spec's "database 100k
1569        # records table" criterion) — NOT 5×20k (that would shrink every table
1570        # to 20k and break the 100k-table p95 budget). api-load's record-list
1571        # round-robins this single 100k DB. Idempotent (seed-100k-v1 marker).
1572        # Needs the API up (register/login the perf user) — it is. ~4-5 min.
1573        run: pnpm --dir tests/performance seed:100k
1574      - name: Seed the 100k-doc search corpus (factory-content index)
1575        # Populates the OpenSearch content index so api-load's 200-rps SEARCH
1576        # scenario queries a REAL corpus (the hard tier — no K6_SKIP_SEARCH).
1577        # Idempotent (deterministic _id upserts; verifies 100,000 landed).
1578        run: pnpm --dir tests/performance perf:seed:search
1579      - name: Extract the seeded ids → K6_* env (from the marker row)
1580        run: |
1581          source <(node tests/performance/extract-100k-env.mjs)
1582          echo "K6_WORKSPACE_ID=$K6_WORKSPACE_ID" >> "$GITHUB_ENV"
1583          echo "K6_DB_ID=$K6_DB_ID" >> "$GITHUB_ENV"
1584          echo "K6_PAGE_IDS=$K6_PAGE_IDS" >> "$GITHUB_ENV"
1585          echo "K6_BIG_PAGE_ID=$K6_BIG_PAGE_ID" >> "$GITHUB_ENV"
1586      - name: Archive-check the suites (k6 parse gate)
1587        run: pnpm --dir tests/performance check:load
1588      - name: API load — FULL (1k rps / 9 min, p95 < 300 ms; HARD — blocks on breach)
1589        # K6_STRICT=1 (the api-load script default). Search INCLUDED (the
1590        # factory-content corpus was seeded above) → the TRUE 1k-rps matrix.
1591        run: pnpm --dir tests/performance api-load
1592      - name: Database load — FULL (100k-table p95 < 500 ms + 100 rps create; HARD)
1593        run: pnpm --dir tests/performance database-load
1594      - name: Realtime load — FULL (10k conns / 200 msg/s, p95 < 200 ms; HARD) + zero-op-loss verify
1595        # K6_PROFILE=full (the runner default): 10k conns / 5-min ramp / 15-min
1596        # hold, K6_STRICT=1, K6_VERIFY=1 — the host-side doc-hash check runs
1597        # IMMEDIATELY after k6 (inside the room's ~5-min GC window). A breach
1598        # (k6 exit 99) or a dropped op (checker exit 1) fails the step.
1599        run: bash tests/performance/run-realtime-load.sh
1600      - name: Editor-local — FULL (interaction p95 < 50 ms + move < 16 ms; HARD)
1601        # The spec's "local editor perf test": the headless browser on the 4-vCPU
1602        # runner measures CLIENT-SIDE latency against the locally-served web
1603        # (:3000) — not network-bound. K6_STRICT=1 → a budget breach fails the
1604        # step. The suite bypasses CSP internally (the committed dev-web CSP
1605        # defect — see the perf-editor header note); until the parent fixes the
1606        # headers the non-bypassing e2e job still hits the "Loading workspace…"
1607        # wall.
1608        run: |
1609          WEB_BASE_URL=http://localhost:3000 K6_PROFILE=full K6_STRICT=1 \
1610            pnpm --dir tests/performance editor-local
1611      - name: Upload the perf report artifact
1612        if: always()
1613        uses: actions/upload-artifact@v4
1614        with:
1615          name: perf-staging-hard
1616          path: tests/performance/reports/
1617          if-no-files-found: ignore
1618          retention-days: 30
1619      - name: Stop the app stack + infra
1620        if: always()
1621        run: |
1622          if [ -f /tmp/factory-dev-staging.pid ]; then
1623            kill "$(cat /tmp/factory-dev-staging.pid)" 2>/dev/null || true
1624            pkill -f "turbo" 2>/dev/null || true
1625            pkill -f "next dev" 2>/dev/null || true
1626          fi
1627          docker compose -f infrastructure/docker/docker-compose.yml down -v --remove-orphans 2>/dev/null || true
1628          tail -n 50 /tmp/factory-dev-staging.log || true
1629