# ============================================================================= # AI Software Factory — CI # # Pipeline (mirrors AGENTS.md "CI/CD Contract"): # install → lint → typecheck → unit → build → integration → security → e2e # # Job graph: # lint ─┐ # typecheck ─┼→ build → integration ─┐ # unit ──┘ security ─────────┼→ e2e # # Concurrency: a new push to the same PR ref cancels the in-progress run. # ============================================================================= name: ci on: push: branches: [main, develop] pull_request: # TASK-0100 S7: nightly full-profile search perf run (03:17 UTC — an odd # minute, spread off the top-of-hour GitHub schedule pile-up). The # perf-search job runs ONLY on this event or on push to main. schedule: - cron: "17 3 * * *" # Manual trigger so the perf lane can be exercised before/without a merge. # perf_mode: "report" = the report-only (dev) tier — the default, safe on any # runner. "staging-hard" = the HARD tier (K6_STRICT=1, full profiles + the # zero-op-loss verify) — intended for the self-hosted staging runner; it # BLOCKS on a threshold breach (exit 99) or a dropped Yjs op. workflow_dispatch: inputs: perf_mode: description: "Perf tier: report (report-only, default) or staging-hard (K6_STRICT=1, blocks on breach)" required: false default: "report" type: choice options: ["report", "staging-hard"] concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} # Shared setup for every job: checkout + pnpm (version from the root # package.json "packageManager" field: pnpm@9.15.0) + Node 20 + frozen install. # YAML anchor — expanded inline in each job. x-setup: &setup - name: Checkout uses: actions/checkout@v4 - name: Setup pnpm uses: pnpm/action-setup@v4 with: version: 9.15.0 # keep in sync with "packageManager" in package.json - name: Setup Node.js uses: actions/setup-node@v4 with: node-version: 20 cache: pnpm - name: Install dependencies run: pnpm install --frozen-lockfile env: CI: true NODE_ENV: test jobs: # --------------------------------------------------------------------------- lint: name: Lint runs-on: ubuntu-24.04 timeout-minutes: 10 steps: - <<: *setup - name: Lint (turbo) run: pnpm lint # --------------------------------------------------------------------------- typecheck: name: Typecheck runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - <<: *setup - name: Prisma generate run: pnpm db:generate - name: Typecheck (turbo) run: pnpm typecheck # --------------------------------------------------------------------------- unit: name: Unit tests runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - <<: *setup - name: Prisma generate run: pnpm db:generate - name: Unit tests (turbo) run: pnpm test:unit - name: Upload unit coverage if: always() uses: actions/upload-artifact@v4 with: name: unit-coverage path: | **/coverage/** !node_modules if-no-files-found: ignore retention-days: 14 # --------------------------------------------------------------------------- build: name: Build runs-on: ubuntu-24.04 timeout-minutes: 20 needs: [lint, typecheck, unit] steps: - <<: *setup - name: Prisma generate run: pnpm db:generate - name: Build all workspaces (turbo) run: pnpm build - name: Upload build artifacts uses: actions/upload-artifact@v4 with: name: build-outputs path: | apps/**/dist/** apps/web/.next/** if-no-files-found: ignore retention-days: 7 # --------------------------------------------------------------------------- integration: name: Integration tests runs-on: ubuntu-24.04 timeout-minutes: 20 needs: [build] services: postgres: image: postgres:16-alpine env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: factory ports: - 5432:5432 options: >- --health-cmd "pg_isready -U postgres -d factory" --health-interval 5s --health-timeout 5s --health-retries 12 redis: image: redis:7-alpine ports: - 6379:6379 options: >- --health-cmd "redis-cli ping" --health-interval 5s --health-timeout 3s --health-retries 12 minio: image: minio/minio command: server /data --console-address ":9001" env: MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin ports: - 9000:9000 options: >- --health-cmd "curl -fsS http://localhost:9000/minio/health/live" --health-interval 5s --health-timeout 5s --health-retries 12 opensearch: image: opensearchproject/opensearch:2 env: discovery.type: single-node plugins.security.disabled: "true" OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m" ports: - 9200:9200 # OpenSearch is SLOW to start on GitHub runners; it has no health # option we can rely on via `options:` (curl exists in the image, but # security-disable startup takes 30-90s), so we wait explicitly below. env: # Variable names match .env.example exactly; only the connection # targets differ (GitHub service containers). DATABASE_URL: postgresql://postgres:postgres@localhost:5432/factory?schema=public REDIS_URL: redis://localhost:6379 SEARCH_URL: http://localhost:9200 SEARCH_USERNAME: "" SEARCH_PASSWORD: "" SEARCH_INDEX_PREFIX: factory S3_ENDPOINT: http://localhost:9000 S3_ACCESS_KEY: minioadmin S3_SECRET_KEY: minioadmin S3_BUCKET: factory-uploads S3_REGION: us-east-1 S3_FORCE_PATH_STYLE: "true" # Test-only secrets (never real values). JWT_SECRET: ci-test-jwt-secret REFRESH_TOKEN_SECRET: ci-test-refresh-secret SESSION_SECRET: ci-test-session-secret MASTER_ENCRYPTION_KEY: "Y2ktdGVzdC1tYXN0ZXIta2V5LTMyc2J5dGVzISE=" steps: - <<: *setup - name: Wait for OpenSearch # Retry loop: the image needs well over a minute on a busy runner. run: | for i in $(seq 1 60); do if curl -sf http://localhost:9200/_cluster/health >/dev/null 2>&1; then echo "opensearch is up after ${i} attempts" exit 0 fi sleep 5 done echo "opensearch did not start in 300s" >&2 exit 1 - name: Create MinIO bucket run: | # mc is not in the runner; use the S3 API via the AWS CLI instead. aws s3api create-bucket \ --endpoint-url http://localhost:9000 \ --bucket factory-uploads \ --region us-east-1 || true - name: Create OpenSearch index run: | # Minimal index so search-dependent integration tests have a target. curl -sf -X PUT "http://localhost:9200/factory-docs" \ -H 'Content-Type: application/json' -d '{}' || true curl -sf -X POST "http://localhost:9200/factory-docs/_refresh" >/dev/null || true - name: Prisma generate run: pnpm db:generate - name: Apply migrations run: pnpm db:deploy - name: Seed run: pnpm db:seed - name: Integration tests (turbo) run: pnpm test:integration - name: Upload integration output if: always() uses: actions/upload-artifact@v4 with: name: integration-reports path: tests/integration/*.report.* if-no-files-found: ignore retention-days: 14 # --------------------------------------------------------------------------- # TASK-0134 — the security stage is the OWASP ASVS L2 gate. It runs: # 1. pnpm audit (high+ fails the build) # 2. the 10 security suites (tests/security) against a RUNNING dev stack # (auth-paths / headers / upload-abuse / rate-limits are live; # SEC_REQUIRE_LIVE=1 makes a down stack fail, not skip) # 3. gitleaks over the FULL git history with the pinned config + the # tracked-remediation baseline (a new secret → red build) # 4. the four app images built + rootfs-scanned for secrets (the spec's # 5th "test image" has no Dockerfile in the repo — recorded N/A in # docs/security/audit-report.md) # --------------------------------------------------------------------------- security: name: Security scan runs-on: ubuntu-24.04 timeout-minutes: 40 needs: [build] services: postgres: image: postgres:16-alpine env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: factory ports: - 5432:5432 options: >- --health-cmd "pg_isready -U postgres -d factory" --health-interval 5s --health-timeout 5s --health-retries 12 redis: image: redis:7-alpine ports: - 6379:6379 options: >- --health-cmd "redis-cli ping" --health-interval 5s --health-timeout 3s --health-retries 12 minio: image: minio/minio command: server /data --console-address ":9001" env: MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin ports: - 9000:9000 options: >- --health-cmd "curl -fsS http://localhost:9000/minio/health/live" --health-interval 5s --health-timeout 5s --health-retries 12 opensearch: image: opensearchproject/opensearch:2 env: discovery.type: single-node plugins.security.disabled: "true" OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m" ports: - 9200:9200 env: # Dev-mode stack (the live security suites assert dev behavior: no # HSTS, no Secure cookie on plain-HTTP :4000) — NODE_ENV must NOT be # production here. DATABASE_URL: postgresql://postgres:postgres@localhost:5432/factory?schema=public REDIS_URL: redis://localhost:6379 SEARCH_URL: http://localhost:9200 SEARCH_USERNAME: "" SEARCH_PASSWORD: "" SEARCH_INDEX_PREFIX: factory S3_ENDPOINT: http://localhost:9000 S3_ACCESS_KEY: minioadmin S3_SECRET_KEY: minioadmin S3_BUCKET: factory-uploads S3_REGION: us-east-1 S3_FORCE_PATH_STYLE: "true" # Test-only secrets (never real values). JWT_SECRET: ci-sec-jwt-secret REFRESH_TOKEN_SECRET: ci-sec-refresh-secret SESSION_SECRET: ci-sec-session-secret MASTER_ENCRYPTION_KEY: "Y2ktc2VjLW1hc3Rlci1rZXktMzJieXRlcyEh" INTERNAL_SERVICE_TOKEN: ci-sec-internal-service-token INTERNAL_TEST_SEAM: "1" AI_PROVIDER: mock SEARCH_VECTOR_DIMENSION: "8" AI_MOCK_DELTA_DELAY_MS: "60" # Live-app security suites: a down stack FAILS the build (not skips). SEC_REQUIRE_LIVE: "1" # The two NEW rate-limit buckets are env-independent (hard-coded # 60/min webhook, 5/min form) so they stay live-testable here; the # global buckets are raised (as in the e2e job) so the other suites # don't trip them. RATE_LIMIT_API: "100000" RATE_LIMIT_AUTH: "10000" steps: - <<: *setup - name: Wait for OpenSearch run: | for i in $(seq 1 60); do if curl -sf http://localhost:9200/_cluster/health >/dev/null 2>&1; then echo "opensearch is up after ${i} attempts" exit 0 fi sleep 5 done echo "opensearch did not start in 300s" >&2 exit 1 - name: Create MinIO bucket + OpenSearch index run: | aws s3api create-bucket \ --endpoint-url http://localhost:9000 \ --bucket factory-uploads \ --region us-east-1 || true curl -sf -X PUT "http://localhost:9200/factory-docs" \ -H 'Content-Type: application/json' -d '{}' || true - name: Download build outputs uses: actions/download-artifact@v4 with: name: build-outputs - name: Prisma generate + migrate + seed run: | pnpm db:generate pnpm db:deploy pnpm db:seed - name: Start dev stack (background) run: | set -o pipefail nohup pnpm dev > /tmp/factory-dev.log 2>&1 & echo $! > /tmp/factory-dev.pid wait_for_port() { local port="$1" name="$2" for i in $(seq 1 120); do if (exec 3<>"/dev/tcp/127.0.0.1/${port}") 2>/dev/null; then exec 3>&- 3<&- || true echo "${name} is up on :${port} after ${i}s" return 0 fi sleep 1 done echo "${name} did not come up on :${port} in 120s" >&2 tail -n 100 /tmp/factory-dev.log >&2 || true return 1 } wait_for_port 4000 api wait_for_port 3000 web wait_for_port 4001 realtime - name: pnpm audit (high+ vulnerabilities fail the build) run: pnpm audit --audit-level=high - name: Security suites (10 suites, live stack required) run: pnpm test:security - name: Stop dev stack if: always() run: | if [ -f /tmp/factory-dev.pid ]; then kill "$(cat /tmp/factory-dev.pid)" 2>/dev/null || true pkill -f "turbo" 2>/dev/null || true pkill -f "next dev" 2>/dev/null || true fi tail -n 50 /tmp/factory-dev.log || true - name: Install gitleaks (pinned 8.18.4) run: | curl -sSfL "https://github.com/gitleaks/gitleaks/releases/download/v8.18.4/gitleaks_8.18.4_linux_x64.tar.gz" | tar -xz -C /usr/local/bin gitleaks gitleaks version - name: Gitleaks — full git history (new secret fails the build) # protect = full history. --baseline-path suppresses exactly the # pre-existing findings documented in gitleaks-baseline.json (each # entry a tracked remediation); ANY other finding exits non-zero. run: | gitleaks protect \ --source=. \ --config gitleaks.toml \ --baseline-path gitleaks-baseline.json \ --report-format json \ --report-path gitleaks-report.json \ --exit-code 1 echo "full-history scan clean (baseline applied)" - name: Gitleaks — build artifacts (dist/ + rootfs of every image) run: | scan_dir() { local dir="$1" label="$2" echo "=== scanning $label ($dir) ===" if [ -d "$dir" ]; then (cd "$dir" && gitleaks detect --source=. --config=../../gitleaks.toml --no-banner) else echo "(dir absent — skipped)" fi } # Built JS/Next output (downloaded from the build job): for d in apps/api/dist apps/web/.next apps/worker/dist apps/realtime/dist; do scan_dir "$d" "$d" done - name: Build the four app images run: | for app in api web worker realtime; do echo "=== building factory-$app ===" docker build -f "apps/$app/Dockerfile" -t "factory-$app:scan" . done - name: Scan image root filesystems for secrets # Spec: "the image-scan step runs on all 5 images ... and is clean". # The 5th "test image" has no Dockerfile in the repo (only 4 apps # exist) — recorded N/A in docs/security/audit-report.md. run: | for app in api web worker realtime; do echo "=== scanning factory-$app rootfs ===" CONTAINER=$(docker create "factory-$app:scan") # No live .env may be baked into the image (.dockerignore): docker export "$CONTAINER" | tar -tf - 2>/dev/null | grep -E '(^|/)\.env$|(^|/)\.env\.(local|production|ci)$' \ && { echo "FAIL: factory-$app ships a .env file"; docker rm "$CONTAINER" >/dev/null; exit 1; } \ || echo "no .env in factory-$app rootfs" # gitleaks over the extracted rootfs (config + baseline applied): mkdir -p /tmp/img-$app docker export "$CONTAINER" | tar -xf - -C /tmp/img-$app (cd /tmp/img-$app && gitleaks detect --source=. --config=../../gitleaks.toml --no-banner) docker rm "$CONTAINER" >/dev/null docker rmi "factory-$app:scan" >/dev/null || true done echo "all 4 images secret-free" - name: Generate npm audit summary if: always() run: | npm audit --json > npm-audit.json 2>&1 || true node -e " const a = require('./npm-audit.json'); const m = a.metadata && a.metadata.vulnerabilities || {}; const lines = [ '# npm audit summary', '', '| Severity | Count |', '| --- | --- |', '| critical | ' + (m.critical ?? 0) + ' |', '| high | ' + (m.high ?? 0) + ' |', '| moderate | ' + (m.moderate ?? 0) + ' |', '| low | ' + (m.low ?? 0) + ' |', '', 'Fails CI only via the pnpm audit gate above (high+).' ]; require('fs').writeFileSync('npm-audit-summary.md', lines.join('\n')); " - name: Upload audit summary if: always() uses: actions/upload-artifact@v4 with: name: security-audit path: | npm-audit.json npm-audit-summary.md gitleaks-report.json if-no-files-found: ignore retention-days: 14 # --------------------------------------------------------------------------- e2e: name: E2E (Playwright) runs-on: ubuntu-24.04 timeout-minutes: 20 needs: [integration, security] services: postgres: image: postgres:16-alpine env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: factory ports: - 5432:5432 options: >- --health-cmd "pg_isready -U postgres -d factory" --health-interval 5s --health-timeout 5s --health-retries 12 redis: image: redis:7-alpine ports: - 6379:6379 options: >- --health-cmd "redis-cli ping" --health-interval 5s --health-timeout 3s --health-retries 12 minio: image: minio/minio command: server /data --console-address ":9001" env: MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin ports: - 9000:9000 options: >- --health-cmd "curl -fsS http://localhost:9000/minio/health/live" --health-interval 5s --health-timeout 5s --health-retries 12 opensearch: image: opensearchproject/opensearch:2 env: discovery.type: single-node plugins.security.disabled: "true" OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m" ports: - 9200:9200 env: DATABASE_URL: postgresql://postgres:postgres@localhost:5432/factory?schema=public REDIS_URL: redis://localhost:6379 SEARCH_URL: http://localhost:9200 SEARCH_USERNAME: "" SEARCH_PASSWORD: "" SEARCH_INDEX_PREFIX: factory S3_ENDPOINT: http://localhost:9000 S3_ACCESS_KEY: minioadmin S3_SECRET_KEY: minioadmin S3_BUCKET: factory-uploads S3_REGION: us-east-1 S3_FORCE_PATH_STYLE: "true" JWT_SECRET: ci-e2e-jwt-secret REFRESH_TOKEN_SECRET: ci-e2e-refresh-secret SESSION_SECRET: ci-e2e-session-secret MASTER_ENCRYPTION_KEY: "Y2ktZTJlLW1hc3Rlci1rZXktMzJieXRlcyEh" # The e2e specs' seam token — the API's INTERNAL_SERVICE_TOKEN (job env # below; the .env copy is inert, see the note on AI_PROVIDER) — TASK-0095 S9. E2E_INTERNAL_SERVICE_TOKEN: ci-e2e-internal-service-token # The API reads process.env ONLY (config/env.ts zod parse — no dotenv # anywhere in apps/api), so the "Prepare .env" step's INTERNAL_SERVICE_TOKEN / # INTERNAL_TEST_SEAM / RATE_LIMIT_AUTH never reached it: the internal # seam was fail-closed in CI (401 on every /internal/* call, breaking the # two-user specs) and the auth limiter stayed at 10/min/IP. Mirroring # them here is what actually opens the seam (TASK-0095 S9 repair, found # while building the TASK-0122 AI seam). INTERNAL_SERVICE_TOKEN: ci-e2e-internal-service-token INTERNAL_TEST_SEAM: "1" RATE_LIMIT_AUTH: "100000" # TASK-0122 (Ask AI E2E): run the deterministic mock provider (no real # provider keys in CI) + the 8-dim mock embedding space. The API reads # process.env directly (config/env.ts) — the job env is what reaches it. AI_PROVIDER: mock SEARCH_VECTOR_DIMENSION: "8" # 60 ms between mock deltas so the panel's streaming state (the live # message grows word by word) is observable in the E2E spec — without # it a 7-word answer arrives in one network round-trip and the # "≥3 incremental updates" assertion can't distinguish it from a # single-shot render. AI_MOCK_DELTA_DELAY_MS: "60" steps: - <<: *setup - name: Install Playwright browser # The e2e package's OWN pinned 1.49.1 binary (the workspace member's # lockfile entry — TASK-0095 S13). `npx playwright …` was pulling an # unrelated latest Playwright whose expected browser revision did not # match the 1.49.1 runtime the specs are written against (the e2e # job had no working install path at all until the workspace fix — # recorded in the S13 record). run: pnpm --dir tests/e2e exec playwright install --with-deps chromium - name: Wait for OpenSearch run: | for i in $(seq 1 60); do if curl -sf http://localhost:9200/_cluster/health >/dev/null 2>&1; then echo "opensearch is up after ${i} attempts" exit 0 fi sleep 5 done echo "opensearch did not start in 300s" >&2 exit 1 - name: Create MinIO bucket + OpenSearch index run: | aws s3api create-bucket \ --endpoint-url http://localhost:9000 \ --bucket factory-uploads \ --region us-east-1 || true curl -sf -X PUT "http://localhost:9200/factory-docs" \ -H 'Content-Type: application/json' -d '{}' || true - name: Prepare .env for the app stack run: | # Start from .env.example; only connection targets change for CI. cp .env.example .env grep -v -e '^DATABASE_URL=' -e '^REDIS_URL=' -e '^SEARCH_URL=' \ -e '^S3_ENDPOINT=' -e '^S3_ACCESS_KEY=' -e '^S3_SECRET_KEY=' \ -e '^JWT_SECRET=' -e '^REFRESH_TOKEN_SECRET=' \ -e '^SESSION_SECRET=' -e '^MASTER_ENCRYPTION_KEY=' \ .env.example > /tmp/env.partial || true { cat /tmp/env.partial echo "DATABASE_URL=postgresql://postgres:postgres@localhost:5432/factory?schema=public" echo "REDIS_URL=redis://localhost:6379" echo "SEARCH_URL=http://localhost:9200" echo "S3_ENDPOINT=http://localhost:9000" echo "S3_ACCESS_KEY=minioadmin" echo "S3_SECRET_KEY=minioadmin" echo "JWT_SECRET=ci-e2e-jwt-secret" echo "REFRESH_TOKEN_SECRET=ci-e2e-refresh-secret" echo "SESSION_SECRET=ci-e2e-session-secret" echo "MASTER_ENCRYPTION_KEY=Y2ktZTJlLW1hc3Rlci1rZXktMzJieXRlcyEh" # Test-only two-user seam (TASK-0095 S9): opens POST # /internal/workspaces/:id/members (service-token guarded). The # e2e specs read the same token from E2E_INTERNAL_SERVICE_TOKEN # (job env below) to add members without an invite mail (D3). echo "INTERNAL_SERVICE_TOKEN=ci-e2e-internal-service-token" echo "INTERNAL_TEST_SEAM=1" # The e2e specs perform ~11 fresh logins/registrations from one # test IP in <60 s (per-test api + browser-context logins). The # product default RATE_LIMIT_AUTH is 10/min per IP, so the final # test's auth 429s (RATE_LIMITED). A throwaway test IP is not a # brute-forcer — raise the auth limit for the test stack only, # mirroring the perf job's RATE_LIMIT_API override. echo "RATE_LIMIT_AUTH=100000" } > .env - name: Prisma generate + migrate + seed run: | pnpm db:generate pnpm db:deploy pnpm db:seed - name: Start full stack (turbo dev, background) run: | set -o pipefail # Run the whole dev stack (web 3000, api 4000, worker, realtime 4001) # in the background, then wait for every port to accept connections. nohup pnpm dev > /tmp/factory-dev.log 2>&1 & echo $! > /tmp/factory-dev.pid wait_for_port() { local port="$1" name="$2" for i in $(seq 1 90); do if (exec 3<>"/dev/tcp/127.0.0.1/${port}") 2>/dev/null; then exec 3>&- 3<&- || true echo "${name} is up on :${port} after ${i}s" return 0 fi sleep 1 done echo "${name} did not come up on :${port} in 90s" >&2 tail -n 100 /tmp/factory-dev.log >&2 || true return 1 } wait_for_port 4000 api wait_for_port 3000 web wait_for_port 4001 realtime - name: Run E2E suite run: pnpm test:e2e - name: Upload Playwright report if: failure() uses: actions/upload-artifact@v4 with: name: playwright-report path: | tests/e2e/playwright-report/** tests/e2e/test-results/** if-no-files-found: ignore retention-days: 14 - name: Stop dev stack if: always() run: | if [ -f /tmp/factory-dev.pid ]; then kill "$(cat /tmp/factory-dev.pid)" 2>/dev/null || true pkill -f "turbo" 2>/dev/null || true pkill -f "next dev" 2>/dev/null || true fi tail -n 50 /tmp/factory-dev.log || true # ============================================================================ # perf — TASK-0095 S14: the realtime 1k load baseline (R4) + the per-PR smoke. # # Self-contained (does NOT need [integration, security]): the k6 load test # targets ONLY the realtime gateway (:4001) + Postgres + Redis — the # realtime service boots with the Prisma doc store directly (no # INTERNAL_SERVICE_TOKEN → no HttpDocStore → the API is not required). # k6 is a Go binary (not an npm dep): pinned to v0.57.0, the first release # whose goja engine parses this suite's `1_000` numeric separators (verified # `k6 archive`-clean in S14 — the earlier `ramping-vusers` typo was caught # the same way). The DoD thresholds (p95<200 ms, 0 protocol errors) are # enforced INSIDE the script's `options.thresholds` — a threshold breach # makes `k6 run` exit non-zero, so the lane FAILS. # # Tiers: SMOKE (100 conns / 30 s) on every PR; the FULL 1k baseline (1,000 # conns, 50/30/20 mix, 3 min) on push to develop (the merge gate) — the # expensive lane does not run on every PR push. # ============================================================================ perf: name: Perf (k6 realtime) runs-on: ubuntu-24.04 timeout-minutes: 25 services: postgres: image: postgres:16-alpine env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: factory ports: - 5432:5432 options: >- --health-cmd "pg_isready -U postgres -d factory" --health-interval 5s --health-timeout 5s --health-retries 12 redis: image: redis:7-alpine ports: - 6379:6379 options: >- --health-cmd "redis-cli ping" --health-interval 5s --health-timeout 3s --health-retries 12 env: DATABASE_URL: postgresql://postgres:postgres@localhost:5432/factory?schema=public REDIS_URL: redis://localhost:6379 JWT_SECRET: ci-perf-jwt-secret REALTIME_TRUST_PROXY: "1" # per-VU X-Forwarded-For connect-limiter identity K6_REALTIME_URL: ws://localhost:4001 K6_JWT_SECRET: ci-perf-jwt-secret # The seeder's deterministic load workspace + its 10 load pages (100 # peers/room). The k6 script's setup() THROWS when K6_PAGE_IDS is # empty, so these are job env (not per-step). K6_WORKSPACE_ID: load-ws K6_PAGE_IDS: load-p-000,load-p-001,load-p-002,load-p-003,load-p-004,load-p-005,load-p-006,load-p-007,load-p-008,load-p-009 steps: - <<: *setup - name: Install k6 v0.57.0 (pinned — parses the suite's numeric separators) run: | K6_VER=v0.57.0 curl -sfL "https://github.com/grafana/k6/releases/download/${K6_VER}/k6-${K6_VER}-linux-amd64.tar.gz" -o /tmp/k6.tgz tar xzf /tmp/k6.tgz -C /tmp mv "/tmp/k6-${K6_VER}-linux-amd64/k6" /usr/local/bin/k6 chmod +x /usr/local/bin/k6 k6 version - name: Prisma generate + migrate run: pnpm db:generate && pnpm db:deploy - name: Seed the 1,000 load identities (workspace + pages + EDIT grants) run: | K6_WORKSPACE_ID=load-ws K6_USERS=1000 node tests/performance/helpers/seed-1k-load.mjs - name: Archive-check the suite (k6 parse gate) run: pnpm --dir tests/performance check:1k - name: Build the realtime service (node dist/index.js start) run: pnpm --filter @factory/realtime build - name: Boot realtime (Prisma doc store, no API required) run: | nohup pnpm --dir apps/realtime start > /tmp/realtime.log 2>&1 & echo $! > /tmp/realtime.pid for i in $(seq 1 90); do if curl -sf http://localhost:4002/health >/dev/null 2>&1; then echo "realtime health up on :4002 after ${i}s" exit 0 fi sleep 1 done echo "realtime did not come up in 90s" >&2 tail -n 100 /tmp/realtime.log >&2 || true exit 1 - name: Smoke (100 conns / 30 s — every PR) run: pnpm --dir tests/performance realtime:smoke - name: Full 1k baseline (push to develop/main — the merge gate) if: github.event_name == 'push' && (github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/main') run: pnpm --dir tests/performance realtime:1k - name: TASK-0135 realtime-load smoke (new 10k suite — report-only, every PR) # The new TASK-0135 suite (realtime-load.js via run-realtime-load.sh) in # SMOKE mode: 100 conns / 10 s ramp / 20 s hold, report-only # (K6_STRICT=0) and K6_VERIFY=0 (skip the host-side zero-op-loss check — # that is the hard gate, which bites in staging). It seeds its OWN # 100-user fixture (load-ws-smoke + smoke-p-* + smoke-verify) — distinct # from the load-ws/1k seed above — so it is self-contained. The hard # 10k run (K6_PROFILE=full, K6_STRICT=1, K6_VERIFY=1) runs in # perf-staging-hard on the self-hosted runner. # NOTE: K6_WORKSPACE_ID is overridden to load-ws-smoke — the job env # carries load-ws (for the old realtime-1k.js suite), and the smoke # runner would otherwise reuse it and not find its smoke-p-* pages. run: | K6_WORKSPACE_ID=load-ws-smoke K6_PROFILE=smoke K6_STRICT=0 K6_VERIFY=0 \ K6_JWT_SECRET=ci-perf-jwt-secret bash tests/performance/run-realtime-load.sh - name: Upload the perf summary artifact if: always() uses: actions/upload-artifact@v4 with: name: perf-realtime-1k path: | tests/performance/realtime-1k-final.json tests/performance/realtime-smoke.json tests/performance/reports/realtime-load.json if-no-files-found: ignore retention-days: 30 - name: Stop realtime if: always() run: | if [ -f /tmp/realtime.pid ]; then kill "$(cat /tmp/realtime.pid)" 2>/dev/null || true; fi pkill -f "apps/realtime" 2>/dev/null || true tail -n 30 /tmp/realtime.log || true # ============================================================================ # perf-search — TASK-0100 S7: the 100k-doc search query performance lane. # # Self-contained (does NOT need [integration, security]): the k6 suite # targets ONLY the API's POST /api/v1/search + Postgres + Redis + # OpenSearch. The API boots with tsx (dev runtime — this is a perf lane, # not the production build) against the fixture corpus seeded directly # into OpenSearch by tests/performance/fixtures/search-100k.ts (100,000 # docs + a minimal 2-ws/200-user DB footprint for real HS256 JWTs). # # Tiers (per the Phase-8 task doc): # - REDUCED profile (10 VUs / 60 s / 10 req/s, 2x-flake thresholds) on # push to main — the merge gate inside the CI contract. # - FULL profile (100 VUs / 5 min / 50 req/s, DoD thresholds # p95<500 ms, p99<1 s, err<0.1%) on the nightly schedule (and manual # workflow_dispatch). The DoD thresholds are enforced INSIDE the # script's options.thresholds — a breach makes `k6 run` exit # non-zero, so the lane FAILS. # # The suite writes tests/performance/reports/search-last.json (p50/p95/p99, # error rate, doc count, env fingerprint) — uploaded as an artifact, not # committed (the task doc: "committed on demand by a maintainer, not # per-PR"). # ============================================================================ perf-search: name: Perf (k6 search 100k) runs-on: ubuntu-24.04 timeout-minutes: 25 if: >- github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main') services: postgres: image: postgres:16-alpine env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: factory ports: - 5432:5432 options: >- --health-cmd "pg_isready -U postgres -d factory" --health-interval 5s --health-timeout 5s --health-retries 12 redis: image: redis:7-alpine ports: - 6379:6379 options: >- --health-cmd "redis-cli ping" --health-interval 5s --health-timeout 3s --health-retries 12 opensearch: image: opensearchproject/opensearch:2 env: discovery.type: single-node plugins.security.disabled: "true" OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m" ports: - 9200:9200 env: # Same variable names as the integration job; targets = GitHub # service containers. DATABASE_URL: postgresql://postgres:postgres@localhost:5432/factory?schema=public REDIS_URL: redis://localhost:6379 SEARCH_URL: http://localhost:9200 SEARCH_USERNAME: "" SEARCH_PASSWORD: "" SEARCH_INDEX_PREFIX: factory S3_ENDPOINT: http://localhost:9000 S3_ACCESS_KEY: minioadmin S3_SECRET_KEY: minioadmin S3_BUCKET: factory-uploads S3_REGION: us-east-1 S3_FORCE_PATH_STYLE: "true" # Test-only secrets (never real values). K6_JWT_SECRET MUST equal the # API's JWT_SECRET: the k6 suite mints HS256 access tokens for the # fixture perf users under this shared secret. JWT_SECRET: ci-perf-search-jwt-secret REFRESH_TOKEN_SECRET: ci-perf-search-refresh-secret SESSION_SECRET: ci-perf-search-session-secret MASTER_ENCRYPTION_KEY: "ZmFjdG9yeS1zZWFyY2gtY2ktbWFzdGVyLWtleS0zMmJ5dGVzISE=" # The k6 script's header mandates a high per-user API rate limit for # load runs (default is 100/min per user). RATE_LIMIT_API: "100000" K6_API_URL: http://localhost:4000 K6_JWT_SECRET: ci-perf-search-jwt-secret K6_SEARCH_WORKSPACES: perf-search-a,perf-search-b K6_VERSION: ci steps: - <<: *setup - name: Install k6 v0.57.0 (pinned — parses the suite's numeric separators) run: | K6_VER=v0.57.0 curl -sfL "https://github.com/grafana/k6/releases/download/${K6_VER}/k6-${K6_VER}-linux-amd64.tar.gz" -o /tmp/k6.tgz tar xzf /tmp/k6.tgz -C /tmp mv "/tmp/k6-${K6_VER}-linux-amd64/k6" /usr/local/bin/k6 chmod +x /usr/local/bin/k6 k6 version - name: Wait for OpenSearch # Same retry pattern as the integration job: the image needs well # over a minute on a busy runner and has no reliable health option. run: | for i in $(seq 1 60); do if curl -sf http://localhost:9200/_cluster/health >/dev/null 2>&1; then echo "opensearch is up after ${i} attempts" exit 0 fi sleep 5 done echo "opensearch did not start in 300s" >&2 exit 1 - name: Prisma generate + migrate run: pnpm db:generate && pnpm db:deploy - name: Archive-check the suite (k6 parse gate) run: pnpm --dir tests/performance check:search - name: Seed the 100,000-doc search fixture (2 ws, 200 users, 2 teams) # Idempotent (reset first, deterministic _id upserts); the seeder # verifies the landed count is exactly 100,000 before finishing. run: pnpm --dir tests/performance perf:seed:search - name: Boot the API (the search endpoint under test) # tsx dev runtime from apps/api (perf lane, not the prod build). # Job env (DATABASE_URL/REDIS_URL/SEARCH_URL/…/RATE_LIMIT_API) is # inherited by the background process. run: | cd apps/api nohup node_modules/.bin/tsx src/main.ts > /tmp/perf-search-api.log 2>&1 & echo $! > /tmp/perf-search-api.pid for i in $(seq 1 90); do if curl -sf http://localhost:4000/api/v1/health >/dev/null 2>&1; then echo "api is up on :4000 after ${i}s" exit 0 fi sleep 1 done echo "api did not come up in 90s" >&2 tail -n 100 /tmp/perf-search-api.log >&2 || true exit 1 - name: Search perf — FULL profile (100 VUs / 5 min / 50 req/s, DoD thresholds; nightly + manual) if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' run: pnpm --dir tests/performance search:100k - name: Search perf — REDUCED profile (10 VUs / 60 s / 10 req/s; merge-to-main gate) if: github.event_name == 'push' && github.ref == 'refs/heads/main' run: pnpm --dir tests/performance search:100k:reduced - name: Upload the perf report artifact if: always() uses: actions/upload-artifact@v4 with: name: perf-search-100k path: | tests/performance/reports/search-last.json tests/performance/reports/search-full.json tests/performance/reports/search-reduced.json tests/performance/search-summary.json if-no-files-found: ignore retention-days: 30 - name: Stop the API if: always() run: | if [ -f /tmp/perf-search-api.pid ]; then kill "$(cat /tmp/perf-search-api.pid)" 2>/dev/null || true fi pkill -f "apps/api" 2>/dev/null || true tail -n 30 /tmp/perf-search-api.log || true # ============================================================================ # perf-load — TASK-0135: the API + database 100k-record load lane. # # Self-contained (does NOT need [integration, security]): the k6 suites # (api-load.js, database-load.js) target ONLY the API's page-tree / record- # list / record-create endpoints + Postgres + Redis. The API boots with tsx # (dev runtime — this is a perf lane, not the production build, the same # pattern as perf-search) against the 100k-record fixture from # seed-100k.mjs (one database / 100,000 records + 200 hub pages + one # 10k-block page). The seeder writes a `seed-100k-v1` marker row into the # `test_seeds` table; extract-100k-env.mjs reads it back into K6_WORKSPACE_ID # / K6_DB_ID / K6_PAGE_IDS / K6_BIG_PAGE_ID — the ids the suites' setup() # requires (they throw when empty). # # Tiers (spec §26: "report-only in dev, hard in staging" — API 1k rps # p95<300 ms; database 100k table p95<500 ms + 100 rps create): # - THIS job (dev, ubuntu-24.04): SMOKE profiles, K6_STRICT=0 — the # report-only tier. It proves the suites + endpoints work end-to-end and # prints the real p95/p99/error values into the artifact; it NEVER # blocks on a threshold. The dev runner is not sized for the 1k-rps / # 9-min full profile, so dev numbers are mechanics, not budgets. # - perf-staging-hard (self-hosted, below): FULL profiles, K6_STRICT=1 — # the hard tier; a breach makes the step FAIL (k6 exit 99). # # No OpenSearch service: the API only talks to OpenSearch on the search # endpoint, and this lane skips that scenario (K6_SKIP_SEARCH=1) — the 100k # fixture is inserted via Prisma directly (no indexing events) and the # content index is only ever created by the worker, which this lane does not # run. Search-query perf at the 100k-doc scale is the perf-search job's job. # No MinIO either: the file endpoints are outside this lane's scenarios # (S3_* set to the tolerated dead-endpoint pattern, like perf-search). # ============================================================================ perf-load: name: Perf (k6 API+DB 100k — report-only) runs-on: ubuntu-24.04 timeout-minutes: 40 services: postgres: image: postgres:16-alpine env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: factory ports: - 5432:5432 options: >- --health-cmd "pg_isready -U postgres -d factory" --health-interval 5s --health-timeout 5s --health-retries 12 redis: image: redis:7-alpine ports: - 6379:6379 options: >- --health-cmd "redis-cli ping" --health-interval 5s --health-timeout 3s --health-retries 12 env: DATABASE_URL: postgresql://postgres:postgres@localhost:5432/factory?schema=public REDIS_URL: redis://localhost:6379 # Test-only secrets (never real values). K6_JWT_SECRET MUST equal the # API's JWT_SECRET: the k6 suites mint HS256 access tokens for the # fixture perf user under this shared secret. JWT_SECRET: ci-perf-load-jwt-secret REFRESH_TOKEN_SECRET: ci-perf-load-refresh-secret SESSION_SECRET: ci-perf-load-session-secret MASTER_ENCRYPTION_KEY: "ZmFjdG9yeS1zZWFyY2gtY2ktbWFzdGVyLWtleS0zMmJ5dGVzISE=" # High per-user API + auth limits for the load run: each VU authenticates # once (cached), but the full profile's 2000 VUs would trip the default # 10/min per IP auth limiter (same override as perf-search / e2e). RATE_LIMIT_API: "100000" RATE_LIMIT_AUTH: "100000" S3_ENDPOINT: http://localhost:9000 S3_ACCESS_KEY: minioadmin S3_SECRET_KEY: minioadmin S3_BUCKET: factory-uploads S3_REGION: us-east-1 S3_FORCE_PATH_STYLE: "true" K6_API_URL: http://localhost:4000 K6_JWT_SECRET: ci-perf-load-jwt-secret # The CI lane has no indexed corpus (see the header) → skip the search # scenario; the other three (tree-load / record-list / record-create) # carry the spec's API-load criteria. K6_SKIP_SEARCH: "1" steps: - <<: *setup - name: Install k6 v0.57.0 (pinned — parses the suite's numeric separators) run: | K6_VER=v0.57.0 curl -sfL "https://github.com/grafana/k6/releases/download/${K6_VER}/k6-${K6_VER}-linux-amd64.tar.gz" -o /tmp/k6.tgz tar xzf /tmp/k6.tgz -C /tmp mv "/tmp/k6-${K6_VER}-linux-amd64/k6" /usr/local/bin/k6 chmod +x /usr/local/bin/k6 k6 version - name: Prisma generate + migrate run: pnpm db:generate && pnpm db:deploy - name: Archive-check the suites (k6 parse gate) run: pnpm --dir tests/performance check:load - name: Boot the API (the endpoints under test) # tsx dev runtime from apps/api (perf lane, not the prod build). # Job env (DATABASE_URL/REDIS_URL/RATE_LIMIT_*/…) is inherited by the # background process. run: | cd apps/api nohup node_modules/.bin/tsx src/main.ts > /tmp/perf-load-api.log 2>&1 & echo $! > /tmp/perf-load-api.pid for i in $(seq 1 90); do if curl -sf http://localhost:4000/api/v1/health >/dev/null 2>&1; then echo "api is up on :4000 after ${i}s" exit 0 fi sleep 1 done echo "api did not come up in 90s" >&2 tail -n 100 /tmp/perf-load-api.log >&2 || true exit 1 - name: Seed the 100k-record fixture (1 DB / 100k records / 200 hubs / 10k-block page) # Idempotent (marker row seed-100k-v1 in test_seeds; a no-op when the # marker matches). ~4-5 min. The API must be up: the seeder # registers/logs in the perf user via the API, then inserts the 100k # records directly via Prisma. run: pnpm --dir tests/performance seed:100k - name: Extract the seeded ids → K6_* env (from the marker row) run: | source <(node tests/performance/extract-100k-env.mjs) echo "K6_WORKSPACE_ID=$K6_WORKSPACE_ID" >> "$GITHUB_ENV" echo "K6_DB_ID=$K6_DB_ID" >> "$GITHUB_ENV" echo "K6_PAGE_IDS=$K6_PAGE_IDS" >> "$GITHUB_ENV" echo "K6_BIG_PAGE_ID=$K6_BIG_PAGE_ID" >> "$GITHUB_ENV" - name: "API load — SMOKE (report-only: proves the endpoints + suite mechanics)" # K6_STRICT=0 (the :smoke script) — the summary prints the real p95/p99 # /error values but never fails the step. run: pnpm --dir tests/performance api-load:smoke - name: "Database load — SMOKE (report-only: 100k-table scroll/open + create)" run: pnpm --dir tests/performance database-load:smoke - name: Threshold-breach red-proof (a lowered threshold MUST turn the step red) # The spec requires the hard tier to actually BLOCK on a breach — so # the lane proves it: the demo suite runs against the live API with an # impossible p95 budget (K6_P95_MS=1) under K6_STRICT=1, so k6 exits # 99. This step inverts that: an exit OTHER than 99 (the demo # unexpectedly passing) fails the lane; the expected 99 keeps it green # and asserts the red tier is real. run: | set +e k6 run tests/performance/threshold-breach-demo.js \ --env K6_API_URL=http://localhost:4000 \ --env K6_STRICT=1 --env K6_P95_MS=1 rc=$? set -e if [ "$rc" -ne 99 ]; then echo "breach demo exited ${rc} (expected 99) — the hard tier is NOT blocking on a threshold breach!" >&2 exit 1 fi echo "breach demo exited 99 as expected — the hard tier blocks on a threshold breach." - name: Upload the perf report artifact if: always() uses: actions/upload-artifact@v4 with: name: perf-load-100k path: tests/performance/reports/ if-no-files-found: ignore retention-days: 30 - name: Stop the API if: always() run: | if [ -f /tmp/perf-load-api.pid ]; then kill "$(cat /tmp/perf-load-api.pid)" 2>/dev/null || true fi pkill -f "apps/api" 2>/dev/null || true tail -n 30 /tmp/perf-load-api.log || true # ============================================================================ # perf-editor — TASK-0135: the editor E2E interaction-latency lane. # # editor-local.mjs drives a real headless Chromium against the FULL dev # stack (turbo dev → web :3000 + api :4000 + realtime :4001, the e2e job's # boot pattern) and measures the LOCAL (client-side, no network) interaction # latency — spec §26 "editor E2E interaction p95 < 50 ms + block move < # 16 ms (local)": type/create p95<50 ms and block-move p95<16 ms (plus a # frame budget). Report-only here (K6_STRICT=0 — the suite prints the real # p95/p99 and never fails the step); the hard tier is # perf-staging-hard's full-profile editor run on the self-hosted runner. # # The full 100k fixture is seeded first so the page tree / permission # lookups run against production-scale data (marker → K6_WORKSPACE_ID). # # NOTE (committed defect, parent task): the security headers (TASK-0134, # packages/shared/src/security-headers.ts) send # script-src 'self' 'unsafe-inline' with NO 'unsafe-eval', and Next 14 DEV # builds emit their chunks via eval() (devtool eval-source-map) — so the dev # web is unbootable under its own headers ("Loading workspace…" forever, # zero API calls). editor-local bypasses CSP on the test context # (bypassCSP: true) so it can still MEASURE; the proper fix (a dev-only # devtool override / 'unsafe-eval' carve-out) is an app change, out of scope # for TASK-0135. The e2e job's fixture does NOT bypass CSP — it hits the # same wall until the parent fixes the headers. # ============================================================================ perf-editor: name: Perf (editor E2E — report-only) runs-on: ubuntu-24.04 timeout-minutes: 35 services: postgres: image: postgres:16-alpine env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: factory ports: - 5432:5432 options: >- --health-cmd "pg_isready -U postgres -d factory" --health-interval 5s --health-timeout 5s --health-retries 12 redis: image: redis:7-alpine ports: - 6379:6379 options: >- --health-cmd "redis-cli ping" --health-interval 5s --health-timeout 3s --health-retries 12 minio: image: minio/minio command: server /data --console-address ":9001" env: MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin ports: - 9000:9000 options: >- --health-cmd "curl -fsS http://localhost:9000/minio/health/live" --health-interval 5s --health-timeout 5s --health-retries 12 opensearch: image: opensearchproject/opensearch:2 env: discovery.type: single-node plugins.security.disabled: "true" OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m" ports: - 9200:9200 env: DATABASE_URL: postgresql://postgres:postgres@localhost:5432/factory?schema=public REDIS_URL: redis://localhost:6379 SEARCH_URL: http://localhost:9200 SEARCH_USERNAME: "" SEARCH_PASSWORD: "" SEARCH_INDEX_PREFIX: factory S3_ENDPOINT: http://localhost:9000 S3_ACCESS_KEY: minioadmin S3_SECRET_KEY: minioadmin S3_BUCKET: factory-uploads S3_REGION: us-east-1 S3_FORCE_PATH_STYLE: "true" JWT_SECRET: ci-perf-editor-jwt-secret REFRESH_TOKEN_SECRET: ci-perf-editor-refresh-secret SESSION_SECRET: ci-perf-editor-session-secret MASTER_ENCRYPTION_KEY: "Y2ktZTJlLW1hc3Rlci1rZXktMzJieXRlcyEh" INTERNAL_SERVICE_TOKEN: ci-perf-editor-internal-service-token INTERNAL_TEST_SEAM: "1" # The editor lane logs in (API + web cookie) and the dev stack's worker # is up; raise the auth limiter as in the e2e job. RATE_LIMIT_AUTH: "100000" # Deterministic mock AI provider (the full dev stack boots the AI # module; no real provider keys in CI) + 8-dim mock embedding space. AI_PROVIDER: mock SEARCH_VECTOR_DIMENSION: "8" steps: - <<: *setup - name: Install Playwright browser (tests/performance's pinned 1.49.1) run: pnpm --dir tests/performance exec playwright install --with-deps chromium - name: Wait for OpenSearch run: | for i in $(seq 1 60); do if curl -sf http://localhost:9200/_cluster/health >/dev/null 2>&1; then echo "opensearch is up after ${i} attempts" exit 0 fi sleep 5 done echo "opensearch did not start in 300s" >&2 exit 1 - name: Create MinIO bucket + OpenSearch index run: | aws s3api create-bucket \ --endpoint-url http://localhost:9000 \ --bucket factory-uploads \ --region us-east-1 || true curl -sf -X PUT "http://localhost:9200/factory-docs" \ -H 'Content-Type: application/json' -d '{}' || true - name: Prepare .env for the app stack run: | cp .env.example .env grep -v -e '^DATABASE_URL=' -e '^REDIS_URL=' -e '^SEARCH_URL=' \ -e '^S3_ENDPOINT=' -e '^S3_ACCESS_KEY=' -e '^S3_SECRET_KEY=' \ -e '^JWT_SECRET=' -e '^REFRESH_TOKEN_SECRET=' \ -e '^SESSION_SECRET=' -e '^MASTER_ENCRYPTION_KEY=' \ .env.example > /tmp/env.partial || true { cat /tmp/env.partial echo "DATABASE_URL=postgresql://postgres:postgres@localhost:5432/factory?schema=public" echo "REDIS_URL=redis://localhost:6379" echo "SEARCH_URL=http://localhost:9200" echo "S3_ENDPOINT=http://localhost:9000" echo "S3_ACCESS_KEY=minioadmin" echo "S3_SECRET_KEY=minioadmin" echo "JWT_SECRET=ci-perf-editor-jwt-secret" echo "REFRESH_TOKEN_SECRET=ci-perf-editor-refresh-secret" echo "SESSION_SECRET=ci-perf-editor-session-secret" echo "MASTER_ENCRYPTION_KEY=Y2ktZTJlLW1hc3Rlci1rZXktMzJieXRlcyEh" echo "INTERNAL_SERVICE_TOKEN=ci-perf-editor-internal-service-token" echo "INTERNAL_TEST_SEAM=1" echo "RATE_LIMIT_AUTH=100000" echo "AI_PROVIDER=mock" echo "SEARCH_VECTOR_DIMENSION=8" } > .env - name: Prisma generate + migrate + seed run: | pnpm db:generate pnpm db:deploy pnpm db:seed - name: Start full stack (turbo dev, background) run: | set -o pipefail nohup pnpm dev > /tmp/factory-dev-editor.log 2>&1 & echo $! > /tmp/factory-dev-editor.pid wait_for_port() { local port="$1" name="$2" for i in $(seq 1 120); do if (exec 3<>"/dev/tcp/127.0.0.1/${port}") 2>/dev/null; then exec 3>&- 3<&- || true echo "${name} is up on :${port} after ${i}s" return 0 fi sleep 1 done echo "${name} did not come up on :${port} in 120s" >&2 tail -n 100 /tmp/factory-dev-editor.log >&2 || true return 1 } wait_for_port 4000 api wait_for_port 3000 web wait_for_port 4001 realtime - name: Seed the 100k-record fixture (production-scale data for the editor) run: pnpm --dir tests/performance seed:100k - name: Extract the seeded workspace → K6_WORKSPACE_ID (from the marker row) run: | source <(node tests/performance/extract-100k-env.mjs) echo "K6_WORKSPACE_ID=$K6_WORKSPACE_ID" >> "$GITHUB_ENV" - name: "Editor-local — SMOKE (report-only: real p95/p99, never blocks)" # K6_STRICT=0 — the suite measures type/create/move against the dev # web and prints the report; it only fails on infra errors (no stack). # See the CSP note in the job header (bypassCSP inside the suite). run: K6_PROFILE=smoke K6_STRICT=0 pnpm --dir tests/performance editor-local - name: Upload the editor perf report artifact if: always() uses: actions/upload-artifact@v4 with: name: perf-editor-local path: | tests/performance/reports/editor-local.json if-no-files-found: ignore retention-days: 30 - name: Stop dev stack if: always() run: | if [ -f /tmp/factory-dev-editor.pid ]; then kill "$(cat /tmp/factory-dev-editor.pid)" 2>/dev/null || true pkill -f "turbo" 2>/dev/null || true pkill -f "next dev" 2>/dev/null || true fi tail -n 50 /tmp/factory-dev-editor.log || true # ============================================================================ # perf-staging-hard — TASK-0135: the HARD perf tier (spec: "hard in staging"). # # Runs ONLY via `workflow_dispatch` with perf_mode=staging-hard, on the # self-hosted staging runner (phase-15 §CI L47: "self-hosted runner (4 vCPU) # for the local editor perf test"). FULL profiles, K6_STRICT=1: a threshold # breach makes the step FAIL (k6 exit 99), and the realtime run keeps # K6_VERIFY=1 — the host-side zero-dropped-Yjs-ops check immediately after # the 10k run (the real DoD gate; it passes only once the apps/realtime # permission-resolver bug is fixed, so it is a meaningful blocker here). # # What this lane boots (self-contained on the 4-vCPU runner): # - infra via the repo's dev compose (infrastructure/docker/): postgres, # redis, minio, opensearch (+ their one-shot init). GitHub `services:` # are unavailable on self-hosted runners, so the canonical dev/staging- # like infra is started with `docker compose` (only the infra + one-shot # init services — the "apps"-profile services are NOT started; the apps # boot from source below, the e2e job's pattern). # - the full app stack from source (`pnpm dev`): api :4000 + web :3000 + # realtime :4001 + worker — so the editor measurement is CLIENT-SIDE # against the locally-served web (the spec's "local" / "self-hosted … # local editor perf test"), and the api/realtime/database suites load the # same production-shaped stack at full profile. # - the 100k-record fixture (seed-100k.mjs, K6_DBS=1 → ONE 100k table, the # spec's "database 100k records table" criterion) + the 100k-doc search # corpus (perf:seed:search → the factory-content index), so api-load runs # its TRUE 1k-rps matrix INCLUDING the 200-rps search scenario (no # K6_SKIP_SEARCH here — that is the report-only-lane corpus workaround). # # The four suites, all FULL + K6_STRICT=1 (a breach blocks the lane): # api-load 1k rps (400 tree / 300 list / 200 search / 100 create), # overall + per-scenario p95 < 300 ms, err < 0.1% # database-load 100k-table scroll/open + 100 rps create, p95 < 500 ms # realtime-load 10k conns / 200 msg/s, p95 < 200 ms + zero-op-loss verify # editor-local interaction p95 < 50 ms + block move < 16 ms (local) # # Provisioning (a self-hosted runner must exist before this job can run): # - a Linux runner labeled `factory-staging` (4 vCPU, per spec L47) with # Docker available (for the infra `docker compose` step). # # NOTE (Phase-15 release-pipeline target): the release pipeline's stage 8 # `performance` (infra/pipeline/release.yml — ABSENT in this repo) runs these # same four suites reading the DEPLOYED staging after the stage-6 rollout is # Ready (release-pipeline-only cluster; the dev/PR pipeline never touches # staging). This ci.yml job is the in-repo, self-contained equivalent: the # runnable hard tier on the self-hosted runner, and what release-pipeline # stage 8 invokes (or a target-staging variant of it). Until a runner with # the label above is provisioned, this job is wired-and-correct-in-shape but # fails fast at the infra step (no matching runner / no docker). # ============================================================================ perf-staging-hard: name: Perf (staging — HARD tier, self-hosted) runs-on: [self-hosted, factory-staging] # label per spec L47 — provision a 4-vCPU runner with this label + Docker timeout-minutes: 120 if: github.event_name == 'workflow_dispatch' && inputs.perf_mode == 'staging-hard' env: # Compose infra defaults (factory:factory/factory — see # infrastructure/docker/docker-compose.yml). The app stack reads the # prepared .env + these job env values. DATABASE_URL: postgresql://factory:factory@localhost:5432/factory?schema=public REDIS_URL: redis://localhost:6379 SEARCH_URL: http://localhost:9200 SEARCH_USERNAME: "" SEARCH_PASSWORD: "" SEARCH_INDEX_PREFIX: factory S3_ENDPOINT: http://localhost:9000 S3_ACCESS_KEY: minioadmin S3_SECRET_KEY: minioadmin S3_BUCKET: factory-uploads S3_REGION: us-east-1 S3_FORCE_PATH_STYLE: "true" # Test-only secrets (never real values). K6_JWT_SECRET MUST equal the app # JWT_SECRET: the k6 suites mint HS256 tokens for the perf users. JWT_SECRET: ci-perf-staging-hard-jwt-secret REFRESH_TOKEN_SECRET: ci-perf-staging-hard-refresh-secret SESSION_SECRET: ci-perf-staging-hard-session-secret MASTER_ENCRYPTION_KEY: "ZmFjdG9yeS1zZWFyY2gtY2ktbWFzdGVyLWtleS0zMmJ5dGVzISE=" INTERNAL_SERVICE_TOKEN: ci-perf-staging-hard-internal-service-token INTERNAL_TEST_SEAM: "1" REALTIME_TRUST_PROXY: "1" # per-VU X-Forwarded-For connect-limiter identity RATE_LIMIT_API: "100000" RATE_LIMIT_AUTH: "100000" AI_PROVIDER: mock SEARCH_VECTOR_DIMENSION: "8" K6_API_URL: http://localhost:4000 K6_REALTIME_URL: ws://localhost:4001 K6_JWT_SECRET: ci-perf-staging-hard-jwt-secret steps: - <<: *setup - name: Install k6 v0.57.0 (pinned — parses the suite's numeric separators) run: | K6_VER=v0.57.0 curl -sfL "https://github.com/grafana/k6/releases/download/${K6_VER}/k6-${K6_VER}-linux-amd64.tar.gz" -o /tmp/k6.tgz tar xzf /tmp/k6.tgz -C /tmp mv "/tmp/k6-${K6_VER}-linux-amd64/k6" /usr/local/bin/k6 chmod +x /usr/local/bin/k6 k6 version - name: "Start infra (dev compose: postgres, redis, minio, opensearch)" # GitHub `services:` are unavailable on self-hosted runners → start the # repo's canonical dev/staging-like infra with docker compose. Only the # infra + one-shot init services (minio-init / opensearch-init, # restart:"no") — the "apps"-profile services are NOT started here. run: | COMPOSE="docker compose -f infrastructure/docker/docker-compose.yml" $COMPOSE up -d postgres redis minio opensearch minio-init opensearch-init wait_http() { # url name tries for i in $(seq 1 "$3"); do if curl -sf "$1" >/dev/null 2>&1; then echo "$2 is up after ${i} checks"; return 0; fi sleep 2 done echo "$2 did not come up in $(( $3 * 2 ))s" >&2; return 1 } wait_http "http://localhost:9200/_cluster/health" "opensearch" 150 # ~5 min (slow JVM start) wait_http "http://localhost:9000/minio/health/live" "minio" 60 for i in $(seq 1 90); do if $COMPOSE exec -T postgres pg_isready -U factory -d factory >/dev/null 2>&1; then echo "postgres is ready after ${i}s"; break fi; sleep 1 done for i in $(seq 1 90); do if $COMPOSE exec -T redis redis-cli ping 2>/dev/null | grep -q PONG; then echo "redis is ready after ${i}s"; break fi; sleep 1 done $COMPOSE ps - name: Prepare .env for the app stack run: | cp .env.example .env grep -v -e '^DATABASE_URL=' -e '^REDIS_URL=' -e '^SEARCH_URL=' \ -e '^S3_ENDPOINT=' -e '^S3_ACCESS_KEY=' -e '^S3_SECRET_KEY=' \ -e '^JWT_SECRET=' -e '^REFRESH_TOKEN_SECRET=' \ -e '^SESSION_SECRET=' -e '^MASTER_ENCRYPTION_KEY=' \ .env.example > /tmp/env.partial || true { cat /tmp/env.partial echo "DATABASE_URL=postgresql://factory:factory@localhost:5432/factory?schema=public" echo "REDIS_URL=redis://localhost:6379" echo "SEARCH_URL=http://localhost:9200" echo "SEARCH_USERNAME=" echo "SEARCH_PASSWORD=" echo "SEARCH_INDEX_PREFIX=factory" echo "S3_ENDPOINT=http://localhost:9000" echo "S3_ACCESS_KEY=minioadmin" echo "S3_SECRET_KEY=minioadmin" echo "S3_BUCKET=factory-uploads" echo "S3_REGION=us-east-1" echo "S3_FORCE_PATH_STYLE=true" echo "JWT_SECRET=ci-perf-staging-hard-jwt-secret" echo "REFRESH_TOKEN_SECRET=ci-perf-staging-hard-refresh-secret" echo "SESSION_SECRET=ci-perf-staging-hard-session-secret" echo "MASTER_ENCRYPTION_KEY=ZmFjdG9yeS1zZWFyY2gtY2ktbWFzdGVyLWtleS0zMmJ5dGVzISE=" echo "INTERNAL_SERVICE_TOKEN=ci-perf-staging-hard-internal-service-token" echo "INTERNAL_TEST_SEAM=1" echo "REALTIME_TRUST_PROXY=1" echo "RATE_LIMIT_API=100000" echo "RATE_LIMIT_AUTH=100000" echo "AI_PROVIDER=mock" echo "SEARCH_VECTOR_DIMENSION=8" } > .env - name: Prisma generate + migrate + seed run: | pnpm db:generate pnpm db:deploy pnpm db:seed - name: Start full app stack from source (turbo dev, background) run: | set -o pipefail nohup pnpm dev > /tmp/factory-dev-staging.log 2>&1 & echo $! > /tmp/factory-dev-staging.pid wait_for_port() { local port="$1" name="$2" for i in $(seq 1 120); do if (exec 3<>"/dev/tcp/127.0.0.1/${port}") 2>/dev/null; then exec 3>&- 3<&- || true echo "${name} is up on :${port} after ${i}s" return 0 fi sleep 1 done echo "${name} did not come up on :${port} in 120s" >&2 tail -n 100 /tmp/factory-dev-staging.log >&2 || true return 1 } wait_for_port 4000 api wait_for_port 3000 web wait_for_port 4001 realtime - name: Seed the 100k-record fixture (K6_DBS=1 → one 100k table) # K6_DBS=1 keeps a single 100k-record table (the spec's "database 100k # records table" criterion) — NOT 5×20k (that would shrink every table # to 20k and break the 100k-table p95 budget). api-load's record-list # round-robins this single 100k DB. Idempotent (seed-100k-v1 marker). # Needs the API up (register/login the perf user) — it is. ~4-5 min. run: pnpm --dir tests/performance seed:100k - name: Seed the 100k-doc search corpus (factory-content index) # Populates the OpenSearch content index so api-load's 200-rps SEARCH # scenario queries a REAL corpus (the hard tier — no K6_SKIP_SEARCH). # Idempotent (deterministic _id upserts; verifies 100,000 landed). run: pnpm --dir tests/performance perf:seed:search - name: Extract the seeded ids → K6_* env (from the marker row) run: | source <(node tests/performance/extract-100k-env.mjs) echo "K6_WORKSPACE_ID=$K6_WORKSPACE_ID" >> "$GITHUB_ENV" echo "K6_DB_ID=$K6_DB_ID" >> "$GITHUB_ENV" echo "K6_PAGE_IDS=$K6_PAGE_IDS" >> "$GITHUB_ENV" echo "K6_BIG_PAGE_ID=$K6_BIG_PAGE_ID" >> "$GITHUB_ENV" - name: Archive-check the suites (k6 parse gate) run: pnpm --dir tests/performance check:load - name: API load — FULL (1k rps / 9 min, p95 < 300 ms; HARD — blocks on breach) # K6_STRICT=1 (the api-load script default). Search INCLUDED (the # factory-content corpus was seeded above) → the TRUE 1k-rps matrix. run: pnpm --dir tests/performance api-load - name: Database load — FULL (100k-table p95 < 500 ms + 100 rps create; HARD) run: pnpm --dir tests/performance database-load - name: Realtime load — FULL (10k conns / 200 msg/s, p95 < 200 ms; HARD) + zero-op-loss verify # K6_PROFILE=full (the runner default): 10k conns / 5-min ramp / 15-min # hold, K6_STRICT=1, K6_VERIFY=1 — the host-side doc-hash check runs # IMMEDIATELY after k6 (inside the room's ~5-min GC window). A breach # (k6 exit 99) or a dropped op (checker exit 1) fails the step. run: bash tests/performance/run-realtime-load.sh - name: Editor-local — FULL (interaction p95 < 50 ms + move < 16 ms; HARD) # The spec's "local editor perf test": the headless browser on the 4-vCPU # runner measures CLIENT-SIDE latency against the locally-served web # (:3000) — not network-bound. K6_STRICT=1 → a budget breach fails the # step. The suite bypasses CSP internally (the committed dev-web CSP # defect — see the perf-editor header note); until the parent fixes the # headers the non-bypassing e2e job still hits the "Loading workspace…" # wall. run: | WEB_BASE_URL=http://localhost:3000 K6_PROFILE=full K6_STRICT=1 \ pnpm --dir tests/performance editor-local - name: Upload the perf report artifact if: always() uses: actions/upload-artifact@v4 with: name: perf-staging-hard path: tests/performance/reports/ if-no-files-found: ignore retention-days: 30 - name: Stop the app stack + infra if: always() run: | if [ -f /tmp/factory-dev-staging.pid ]; then kill "$(cat /tmp/factory-dev-staging.pid)" 2>/dev/null || true pkill -f "turbo" 2>/dev/null || true pkill -f "next dev" 2>/dev/null || true fi docker compose -f infrastructure/docker/docker-compose.yml down -v --remove-orphans 2>/dev/null || true tail -n 50 /tmp/factory-dev-staging.log || true