release.yml · Source

release.yml.github/workflows/release.ymlraw ← workflows42,041 B← Back to the app
1# =============================================================================
2# AI Software Factory — Release v1.0 (TASK-0135, Phase 15)
3#
4# 12-stage release pipeline. The TAG IS THE LAST ACT: nothing in this file is
5# triggered by a tag (the old `push: tags:` trigger is REMOVED — the tag is
6# created BY stage 12, so a tag push can never re-enter this workflow).
7#
8#   Trigger ......... workflow_dispatch only:
9#                       from_stage      1–12 (default 1) — re-runnable per stage
10#                       release_version semver (default 1.0.0)
11#   Stage gate ...... a single `gate` job outputs `from` (the start stage);
12#                     stage N runs iff `needs.gate.outputs.from == N`. Every
13#                     stage also `needs` every predecessor, so a stage can
14#                     never start on a red predecessor (spec rule).
15#   Re-runnability .. `from_stage: K` re-runs stages K..12 only. Stage 12 is
16#                     idempotent: it re-pushes the same image tags/digests and
17#                     never moves an existing tag.
18#
19#   Stages:
20#     1  install+lint+typecheck   7  smoke            (Phase 0 smoke vs staging)
21#     2  unit                     8  performance      (4 k6/local suites, HARD)
22#     3  integration              9  backup-health    (26h/10min/26h, HARD gate)
23#     4  security                 10 final-e2e        (tests/e2e/final-flow.spec.ts)
24#     5  build                    11 docs-completeness (scripts/docs-completeness.mjs)
25#     6  deploy-staging           12 push+tag         (GHCR × 4, annotated tag,
26#                                                      release notes — LAST)
27#
28#   Permissions: contents: write (tag + release), packages: write (GHCR push),
29#   actions: read (run link for the release notes).
30# =============================================================================
31name: release
32 
33on:
34  workflow_dispatch:
35    inputs:
36      from_stage:
37        description: "Stage to start at (1-12; re-runnable per stage)"
38        required: false
39        default: "1"
40        type: string
41      release_version:
42        description: "Release version (semver, no leading v)"
43        required: false
44        default: "1.0.0"
45        type: string
46 
47# A release should never be canceled by a newer dispatch mid-publish.
48concurrency:
49  group: release-${{ github.workflow }}-${{ github.ref }}
50  cancel-in-progress: false
51 
52permissions:
53  contents: write
54  packages: write
55  actions: read
56 
57env:
58  REGISTRY: ghcr.io
59  IMAGE_OWNER: ${{ github.repository }}
60  PNPM_VERSION: "9.15.0" # keep in sync with "packageManager" in package.json
61 
62jobs:
63  # ---------------------------------------------------------------------------
64  # gate — stage orchestrator (validates inputs, computes the start stage).
65  # ---------------------------------------------------------------------------
66  gate:
67    name: "Gate: resolve version + start stage"
68    runs-on: ubuntu-24.04
69    timeout-minutes: 5
70    outputs:
71      version: ${{ steps.resolve.outputs.version }}
72      is_patch: ${{ steps.resolve.outputs.is_patch }}
73      from: ${{ steps.resolve.outputs.from }}
74    steps:
75      - name: Resolve version + start stage
76        id: resolve
77        env:
78          RAW_VERSION: ${{ github.event.inputs.release_version }}
79          RAW_FROM: ${{ github.event.inputs.from_stage }}
80        run: |
81          set -euo pipefail
82          V="${RAW_VERSION:-1.0.0}"
83          V="${V#v}"
84          case "$V" in
85            *[!0-9.]*|*..*|.*|*.*) echo "invalid semver version: ${RAW_VERSION:-1.0.0}" >&2; exit 1 ;;
86          esac
87          MAJOR="$(cut -d. -f1 <<<"$V")"
88          MINOR="$(cut -d. -f2 <<<"$V")"
89          PATCH="$(cut -d. -f3 <<<"$V")"
90          if ! [[ "$MAJOR$MINOR$PATCH" =~ ^[0-9]+$ ]]; then
91            echo "version components must be non-negative integers: $V" >&2; exit 1
92          fi
93          # "Patch bump" = a release on the vX.0.0 line (minor=0 AND patch=0)
94          # → those releases also publish the `latest` tag.
95          if [ "$MINOR" = "0" ] && [ "$PATCH" = "0" ]; then IS_PATCH=true; else IS_PATCH=false; fi
96          # from_stage: clamp into 1..12 (empty/invalid → 1 = full pipeline).
97          F="${RAW_FROM:-1}"
98          if ! [[ "$F" =~ ^[0-9]+$ ]] || [ "$F" -lt 1 ] || [ "$F" -gt 12 ]; then
99            echo "from_stage must be 1..12 (got '${RAW_FROM:-1}') — using 1" >&2
100            F=1
101          fi
102          echo "version=$V" >> "$GITHUB_OUTPUT"
103          echo "is_patch=$IS_PATCH" >> "$GITHUB_OUTPUT"
104          echo "from=$F" >> "$GITHUB_OUTPUT"
105          echo "::notice::release=$V (latest-tag: $IS_PATCH) — running stages ${F}..12"
106 
107  # ---------------------------------------------------------------------------
108  # Stage 1 — install+lint+typecheck (hard).
109  # ---------------------------------------------------------------------------
110  stage-1-lint-typecheck:
111    name: "[1/12] install + lint + typecheck"
112    runs-on: ubuntu-24.04
113    timeout-minutes: 30
114    needs: [gate]
115    if: needs.gate.outputs.from == 1
116    steps:
117      - name: Checkout
118        uses: actions/checkout@v4
119      - name: Setup pnpm
120        uses: pnpm/action-setup@v4
121        with:
122          version: ${{ env.PNPM_VERSION }}
123      - name: Setup Node.js
124        uses: actions/setup-node@v4
125        with:
126          node-version: 20
127          cache: pnpm
128      - name: Install (frozen lockfile)
129        run: pnpm install --frozen-lockfile
130      - name: ESLint
131        run: pnpm lint
132      - name: Typecheck (tsc strict via turbo)
133        run: pnpm typecheck
134 
135  # ---------------------------------------------------------------------------
136  # Stage 2 — unit (hard): Vitest (packages) + Jest (api).
137  # ---------------------------------------------------------------------------
138  stage-2-unit:
139    name: "[2/12] unit tests"
140    runs-on: ubuntu-24.04
141    timeout-minutes: 30
142    needs: [gate, stage-1-lint-typecheck]
143    if: needs.gate.outputs.from == 2
144    steps:
145      - name: Checkout
146        uses: actions/checkout@v4
147      - name: Setup pnpm
148        uses: pnpm/action-setup@v4
149        with:
150          version: ${{ env.PNPM_VERSION }}
151      - name: Setup Node.js
152        uses: actions/setup-node@v4
153        with:
154          node-version: 20
155          cache: pnpm
156      - name: Install (frozen lockfile)
157        run: pnpm install --frozen-lockfile
158      - name: "Unit tests (turbo: vitest packages + jest api)"
159        run: pnpm test:unit
160 
161  # ---------------------------------------------------------------------------
162  # Stage 3 — integration (hard): tests/integration against live services
163  # (same service block + env as the CI integration job).
164  # ---------------------------------------------------------------------------
165  stage-3-integration:
166    name: "[3/12] integration tests"
167    runs-on: ubuntu-24.04
168    timeout-minutes: 45
169    needs: [gate, stage-1-lint-typecheck, stage-2-unit]
170    if: needs.gate.outputs.from == 3
171    services:
172      postgres:
173        image: postgres:16-alpine
174        env:
175          POSTGRES_USER: postgres
176          POSTGRES_PASSWORD: postgres
177          POSTGRES_DB: factory
178        ports:
179          - 5432:5432
180        options: >-
181          --health-cmd "pg_isready -U postgres -d factory"
182          --health-interval 5s
183          --health-timeout 5s
184          --health-retries 12
185      redis:
186        image: redis:7-alpine
187        ports:
188          - 6379:6379
189        options: >-
190          --health-cmd "redis-cli ping"
191          --health-interval 5s
192          --health-timeout 3s
193          --health-retries 12
194      minio:
195        image: minio/minio
196        command: server /data --console-address ":9001"
197        env:
198          MINIO_ROOT_USER: minioadmin
199          MINIO_ROOT_PASSWORD: minioadmin
200        ports:
201          - 9000:9000
202        options: >-
203          --health-cmd "curl -fsS http://localhost:9000/minio/health/live"
204          --health-interval 5s
205          --health-timeout 5s
206          --health-retries 12
207      opensearch:
208        image: opensearchproject/opensearch:2
209        env:
210          discovery.type: single-node
211          plugins.security.disabled: "true"
212          OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m"
213        ports:
214          - 9200:9200
215        # OpenSearch is SLOW on GitHub runners — wait explicitly (below).
216    env:
217      DATABASE_URL: postgresql://postgres:postgres@localhost:5432/factory?schema=public
218      REDIS_URL: redis://localhost:6379
219      SEARCH_URL: http://localhost:9200
220      SEARCH_USERNAME: ""
221      SEARCH_PASSWORD: ""
222      SEARCH_INDEX_PREFIX: factory
223      S3_ENDPOINT: http://localhost:9000
224      S3_ACCESS_KEY: minioadmin
225      S3_SECRET_KEY: minioadmin
226      S3_BUCKET: factory-uploads
227      S3_REGION: us-east-1
228      S3_FORCE_PATH_STYLE: "true"
229      # Test-only secrets (never real values).
230      JWT_SECRET: release-test-jwt-secret
231      REFRESH_TOKEN_SECRET: release-test-refresh-secret
232      SESSION_SECRET: release-test-session-secret
233      MASTER_ENCRYPTION_KEY: "cmVsZWFzZS10ZXN0LW1hc3Rlci1rZXktMzJieXRlcyEh" # "release-test-master-key-32bytes!!"
234    steps:
235      - name: Checkout
236        uses: actions/checkout@v4
237      - name: Setup pnpm
238        uses: pnpm/action-setup@v4
239        with:
240          version: ${{ env.PNPM_VERSION }}
241      - name: Setup Node.js
242        uses: actions/setup-node@v4
243        with:
244          node-version: 20
245          cache: pnpm
246      - name: Install (frozen lockfile)
247        run: pnpm install --frozen-lockfile
248      - name: Wait for OpenSearch
249        run: |
250          for i in $(seq 1 60); do
251            if curl -sf http://localhost:9200/_cluster/health >/dev/null 2>&1; then
252              echo "opensearch is up after ${i} attempts"; exit 0
253            fi
254            sleep 5
255          done
256          echo "opensearch did not start in 300s" >&2; exit 1
257      - name: Create MinIO bucket + OpenSearch index
258        run: |
259          aws s3api create-bucket \
260            --endpoint-url http://localhost:9000 \
261            --bucket factory-uploads \
262            --region us-east-1 || true
263          curl -sf -X PUT "http://localhost:9200/factory-docs" \
264            -H 'Content-Type: application/json' -d '{}' || true
265          curl -sf -X POST "http://localhost:9200/factory-docs/_refresh" >/dev/null || true
266      - name: Prisma generate
267        run: pnpm db:generate
268      - name: Apply migrations
269        run: pnpm db:deploy
270      - name: Seed
271        run: pnpm db:seed
272      - name: Integration tests (turbo)
273        run: pnpm test:integration
274 
275  # ---------------------------------------------------------------------------
276  # Stage 4 — security (hard): TASK-0134's 10 suites + pnpm audit + gitleaks.
277  # Mirrors the CI security job (live dev-mode stack: the suites assert dev
278  # behavior — NODE_ENV must NOT be production).
279  # ---------------------------------------------------------------------------
280  stage-4-security:
281    name: "[4/12] security scan"
282    runs-on: ubuntu-24.04
283    timeout-minutes: 60
284    needs: [gate, stage-3-integration]
285    if: needs.gate.outputs.from == 4
286    services:
287      postgres:
288        image: postgres:16-alpine
289        env:
290          POSTGRES_USER: postgres
291          POSTGRES_PASSWORD: postgres
292          POSTGRES_DB: factory
293        ports:
294          - 5432:5432
295        options: >-
296          --health-cmd "pg_isready -U postgres -d factory"
297          --health-interval 5s
298          --health-timeout 5s
299          --health-retries 12
300      redis:
301        image: redis:7-alpine
302        ports:
303          - 6379:6379
304        options: >-
305          --health-cmd "redis-cli ping"
306          --health-interval 5s
307          --health-timeout 3s
308          --health-retries 12
309      minio:
310        image: minio/minio
311        command: server /data --console-address ":9001"
312        env:
313          MINIO_ROOT_USER: minioadmin
314          MINIO_ROOT_PASSWORD: minioadmin
315        ports:
316          - 9000:9000
317        options: >-
318          --health-cmd "curl -fsS http://localhost:9000/minio/health/live"
319          --health-interval 5s
320          --health-timeout 5s
321          --health-retries 12
322      opensearch:
323        image: opensearchproject/opensearch:2
324        env:
325          discovery.type: single-node
326          plugins.security.disabled: "true"
327          OPENSEARCH_JAVA_OPTS: "-Xms512m -Xmx512m"
328        ports:
329          - 9200:9200
330    env:
331      # Dev-mode stack (the live security suites assert dev behavior: no
332      # HSTS, no Secure cookie on plain-HTTP :4000) — NODE_ENV stays dev.
333      DATABASE_URL: postgresql://postgres:postgres@localhost:5432/factory?schema=public
334      REDIS_URL: redis://localhost:6379
335      SEARCH_URL: http://localhost:9200
336      SEARCH_USERNAME: ""
337      SEARCH_PASSWORD: ""
338      SEARCH_INDEX_PREFIX: factory
339      S3_ENDPOINT: http://localhost:9000
340      S3_ACCESS_KEY: minioadmin
341      S3_SECRET_KEY: minioadmin
342      S3_BUCKET: factory-uploads
343      S3_REGION: us-east-1
344      S3_FORCE_PATH_STYLE: "true"
345      JWT_SECRET: release-sec-jwt-secret
346      REFRESH_TOKEN_SECRET: release-sec-refresh-secret
347      SESSION_SECRET: release-sec-session-secret
348      MASTER_ENCRYPTION_KEY: "cmVsZWFzZS1zZWMtbWFzdGVyLWtleS0zMmJ5dGVzISE="
349      INTERNAL_SERVICE_TOKEN: release-sec-internal-service-token
350      INTERNAL_TEST_SEAM: "1"
351      AI_PROVIDER: mock
352      SEARCH_VECTOR_DIMENSION: "8"
353      AI_MOCK_DELTA_DELAY_MS: "60"
354      # Live-app security suites: a down stack FAILS the build (not skips).
355      SEC_REQUIRE_LIVE: "1"
356      # Global rate-limit buckets raised (as in CI) so the other suites don't
357      # trip them; the two fixed buckets (webhook 60/min, form 5/min) are
358      # env-independent and stay live-testable.
359      RATE_LIMIT_API: "100000"
360      RATE_LIMIT_AUTH: "10000"
361    steps:
362      - name: Checkout
363        uses: actions/checkout@v4
364      - name: Setup pnpm
365        uses: pnpm/action-setup@v4
366        with:
367          version: ${{ env.PNPM_VERSION }}
368      - name: Setup Node.js
369        uses: actions/setup-node@v4
370        with:
371          node-version: 20
372          cache: pnpm
373      - name: Install (frozen lockfile)
374        run: pnpm install --frozen-lockfile
375      - name: Wait for OpenSearch
376        run: |
377          for i in $(seq 1 60); do
378            if curl -sf http://localhost:9200/_cluster/health >/dev/null 2>&1; then
379              echo "opensearch is up after ${i} attempts"; exit 0
380            fi
381            sleep 5
382          done
383          echo "opensearch did not start in 300s" >&2; exit 1
384      - name: Create MinIO bucket + OpenSearch index
385        run: |
386          aws s3api create-bucket \
387            --endpoint-url http://localhost:9000 \
388            --bucket factory-uploads \
389            --region us-east-1 || true
390          curl -sf -X PUT "http://localhost:9200/factory-docs" \
391            -H 'Content-Type: application/json' -d '{}' || true
392      - name: Prisma generate + migrate + seed
393        run: |
394          pnpm db:generate
395          pnpm db:deploy
396          pnpm db:seed
397      - name: Start dev stack (background)
398        run: |
399          set -o pipefail
400          nohup pnpm dev > /tmp/factory-dev.log 2>&1 &
401          echo $! > /tmp/factory-dev.pid
402          wait_for_port() {
403            local port="$1" name="$2"
404            for i in $(seq 1 120); do
405              if (exec 3<>"/dev/tcp/127.0.0.1/${port}") 2>/dev/null; then
406                exec 3>&- 3<&- || true
407                echo "${name} is up on :${port} after ${i}s"; return 0
408              fi
409              sleep 1
410            done
411            echo "${name} did not come up on :${port} in 120s" >&2
412            tail -n 100 /tmp/factory-dev.log >&2 || true
413            return 1
414          }
415          wait_for_port 4000 api
416          wait_for_port 3000 web
417          wait_for_port 4001 realtime
418      - name: pnpm audit (high+ vulnerabilities fail the build)
419        run: pnpm audit --audit-level=high
420      - name: Security suites (10 suites, live stack required)
421        run: pnpm test:security
422      - name: Stop dev stack
423        if: always()
424        run: |
425          if [ -f /tmp/factory-dev.pid ]; then
426            kill "$(cat /tmp/factory-dev.pid)" 2>/dev/null || true
427            pkill -f "turbo" 2>/dev/null || true
428            pkill -f "next dev" 2>/dev/null || true
429          fi
430          tail -n 50 /tmp/factory-dev.log || true
431      - name: Install gitleaks (pinned 8.18.4)
432        run: |
433          curl -sSfL "https://github.com/gitleaks/gitleaks/releases/download/v8.18.4/gitleaks_8.18.4_linux_x64.tar.gz" | tar -xz -C /usr/local/bin gitleaks
434          gitleaks version
435      - name: Gitleaks — full git history (new secret fails the build)
436        # protect = full history. --baseline-path suppresses exactly the
437        # pre-existing findings documented in gitleaks-baseline.json; ANY
438        # other finding exits non-zero.
439        run: |
440          gitleaks protect \
441            --source=. \
442            --config gitleaks.toml \
443            --baseline-path gitleaks-baseline.json \
444            --report-format json \
445            --report-path gitleaks-report.json \
446            --exit-code 1
447          echo "full-history scan clean (baseline applied)"
448 
449  # ---------------------------------------------------------------------------
450  # Stage 5 — build (hard): pnpm build (turbo) + the four Docker images built
451  # (NOT yet pushed — stage 12 pushes). Build outputs artifact for stages
452  # 4-artifact consumers and the release notes.
453  # ---------------------------------------------------------------------------
454  stage-5-build:
455    name: "[5/12] build (turbo + docker images)"
456    runs-on: ubuntu-24.04
457    timeout-minutes: 60
458    needs: [gate, stage-4-security]
459    if: needs.gate.outputs.from == 5
460    strategy:
461      fail-fast: false
462      matrix:
463        # Keep in sync with the apps/ directories and the kustomization
464        # images list (infrastructure/kubernetes/kustomization.yaml).
465        app: [api, web, worker, realtime]
466    steps:
467      - name: Checkout
468        uses: actions/checkout@v4
469      - name: Setup pnpm
470        uses: pnpm/action-setup@v4
471        with:
472          version: ${{ env.PNPM_VERSION }}
473      - name: Setup Node.js
474        uses: actions/setup-node@v4
475        with:
476          node-version: 20
477          cache: pnpm
478      - name: Install (frozen lockfile)
479        run: pnpm install --frozen-lockfile
480      - name: pnpm build (turbo — all apps + packages)
481        run: pnpm build
482        if: matrix.app == 'api' # once per workflow run (matrix fans the docker part)
483      - name: Set up Docker Buildx
484        uses: docker/setup-buildx-action@v3
485      - name: Build image (not pushed)
486        uses: docker/build-push-action@v6
487        with:
488          context: .
489          file: apps/${{ matrix.app }}/Dockerfile
490          push: false
491          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_OWNER }}/${{ matrix.app }}:${{ needs.gate.outputs.version }}
492          cache-from: type=gha,scope=${{ matrix.app }}
493          cache-to: type=gha,scope=${{ matrix.app }}
494          labels: |
495            org.opencontainers.image.source=${{ github.repositoryUrl }}
496            org.opencontainers.image.revision=${{ github.sha }}
497            org.opencontainers.image.version=${{ needs.gate.outputs.version }}
498      - name: Image build digest
499        run: echo "${{ env.REGISTRY }}/${{ env.IMAGE_OWNER }}/${{ matrix.app }}:${{ needs.gate.outputs.version }} built (push happens in stage 12)"
500      - name: Upload build outputs
501        if: matrix.app == 'api'
502        uses: actions/upload-artifact@v4
503        with:
504          name: build-outputs
505          path: |
506            apps/api/dist
507            apps/worker/dist
508            apps/realtime/dist
509          if-no-files-found: warn
510 
511  # ---------------------------------------------------------------------------
512  # Stage 6 — deploy-staging (hard: apply + readiness).
513  #
514  # PLACEHOLDER — wire to your cluster before a real release. To activate:
515  # set the repo variable DEPLOY_STAGING=true and a KUBECONFIG (or ArgoCD
516  # token) on the "staging" environment. Until then the step is an explicit
517  # no-op (and fails closed if DEPLOY_STAGING=true but no credentials exist).
518  # ---------------------------------------------------------------------------
519  stage-6-deploy-staging:
520    name: "[6/12] deploy staging (placeholder)"
521    runs-on: ubuntu-24.04
522    timeout-minutes: 15
523    needs: [gate, stage-5-build]
524    if: needs.gate.outputs.from == 6
525    environment:
526      name: staging
527    env:
528      DEPLOY_STAGING: ${{ vars.DEPLOY_STAGING }}
529    steps:
530      - name: Checkout
531        uses: actions/checkout@v4
532      - name: Check deployment wiring
533        run: |
534          if [ "$DEPLOY_STAGING" != "true" ]; then
535            echo "DEPLOY_STAGING is not 'true' — skipping deployment (placeholder)."
536            echo "To enable: set the repo variable DEPLOY_STAGING=true, add a KUBECONFIG"
537            echo "(or ArgoCD token) to the 'staging' environment, and wire the deploy step."
538            exit 0
539          fi
540          if [ -z "${KUBECONFIG:-}" ] && [ -z "${ARGOCD_TOKEN:-}" ]; then
541            echo "DEPLOY_STAGING=true but no KUBECONFIG/ARGOCD_TOKEN secret found." >&2
542            echo "Refusing to guess: add cluster credentials to the staging environment." >&2
543            exit 1
544          fi
545          # ── Wire your deployment here ─────────────────────────────────
546          # Example (kustomize): bump the tag and apply:
547          #   kustomize edit set image ghcr.io/<org>/api=${{ needs.gate.outputs.version }} \
548          #     infrastructure/kubernetes/
549          #   kubectl apply -k infrastructure/kubernetes/
550          # Example (Argo CD):
551          #   argocd app sync factory-staging --force
552          # ─────────────────────────────────────────────────────────────
553          echo "Deployment step not yet wired — see release.yml header. (no-op)"
554      - name: Wait for rollout (10 min budget)
555        # Runs only once deployment is wired: stages 7/8/10 read staging
556        # AFTER the rollout is Ready (spec rule).
557        if: env.DEPLOY_STAGING == 'true'
558        run: |
559          for i in $(seq 1 60); do
560            if kubectl -n factory get deploy api web worker realtime \
561                 -o jsonpath='{range .items[*]}{.status.readyReplicas}/{.status.replicas}{"\n"}{end}' \
562                 | awk -F/ '$1 != $2' | grep -q . ; then
563              sleep 10
564            else
565              echo "all deployments ready after ${i}0s"; exit 0
566            fi
567          done
568          echo "rollout not ready in 600s" >&2; exit 1
569 
570  # ---------------------------------------------------------------------------
571  # Stage 7 — smoke (hard): the Phase 0 smoke suite against staging —
572  # register → login → create page → search finds it. Budget: < 2 min.
573  # ---------------------------------------------------------------------------
574  stage-7-smoke:
575    name: "[7/12] smoke (staging)"
576    runs-on: ubuntu-24.04
577    timeout-minutes: 10
578    needs: [gate, stage-6-deploy-staging]
579    if: needs.gate.outputs.from == 7
580    environment:
581      name: staging
582    env:
583      # Staging base URLs (the smoke hits the deployed stack, not CI services).
584      E2E_BASE_URL: ${{ vars.STAGING_WEB_URL }}
585      E2E_API_URL: ${{ vars.STAGING_API_URL }}
586    steps:
587      - name: Checkout
588        uses: actions/checkout@v4
589      - name: Setup pnpm
590        uses: pnpm/action-setup@v4
591        with:
592          version: ${{ env.PNPM_VERSION }}
593      - name: Setup Node.js
594        uses: actions/setup-node@v4
595        with:
596          node-version: 20
597          cache: pnpm
598      - name: Install (frozen lockfile)
599        run: pnpm install --frozen-lockfile
600      - name: "Smoke: login + page create + search (< 2 min)"
601        env:
602          DEPLOY_STAGING: ${{ vars.DEPLOY_STAGING }}
603        run: |
604          set -euo pipefail
605          API="${E2E_API_URL:-http://staging.example.com/api}"
606          echo "smoke target: api=$API"
607          if [ "${DEPLOY_STAGING:-}" != "true" ]; then
608            echo "SKIPPED (loud): staging is not wired (DEPLOY_STAGING != true) — smoke cannot run."
609            exit 1
610          fi
611          START=$(date +%s)
612          RUN_ID="$(date +%s)-$RANDOM"
613          EMAIL="smoke-${RUN_ID}@factory.local"
614          PASSW0RD='Smoke!2345'
615          jqget() { node -e "let d='';process.stdin.on('data',c=>d+=c).on('end',()=>{try{const j=JSON.parse(d);const v=eval('j$1');console.log(v??'')}catch{console.log('')}})"; }
616          # 1 — register (201: user + personal workspace + tokens; httpOnly refresh cookie)
617          REG=$(curl -fsS -c /tmp/smoke.cookies -X POST "${API}/v1/auth/register" \
618            -H 'content-type: application/json' \
619            -d "{\"name\":\"Smoke ${RUN_ID}\",\"email\":\"${EMAIL}\",\"password\":\"${PASSW0RD}\"}")
620          WS_ID="$(printf '%s' "$REG" | jqget '.personalWorkspace.id')"
621          [ -n "$WS_ID" ] || { echo "register response missing personalWorkspace.id: ${REG}" >&2; exit 1; }
622          # 2 — login (fresh session + fresh cookie)
623          LOG=$(curl -fsS -c /tmp/smoke.cookies -X POST "${API}/v1/auth/login" \
624            -H 'content-type: application/json' \
625            -d "{\"email\":\"${EMAIL}\",\"password\":\"${PASSW0RD}\"}")
626          printf '%s' "$LOG" | grep -q 'accessToken' || { echo "login did not return a token" >&2; exit 1; }
627          # 3 — create a page in the auto-created personal workspace
628          PAGE=$(curl -fsS -b /tmp/smoke.cookies -X POST "${API}/v1/workspaces/${WS_ID}/pages" \
629            -H 'content-type: application/json' -d "{\"title\":\"Smoke page ${RUN_ID}\"}")
630          PAGE_ID="$(printf '%s' "$PAGE" | jqget '.id')"
631          [ -n "$PAGE_ID" ] || { echo "page create failed: ${PAGE}" >&2; exit 1; }
632          # 4 — search finds the page (GET /search/query = the command-palette
633          #    page search, a bare array of {id,type,title}; allow indexing
634          #    lag: 3 retries × 5 s)
635          FOUND=""
636          for i in 1 2 3; do
637            sleep 5
638            FOUND=$(curl -fsS -b /tmp/smoke.cookies "${API}/v1/search/query?q=Smoke+page+${RUN_ID}" || true)
639            printf '%s' "$FOUND" | grep -q "Smoke page ${RUN_ID}" && break
640          done
641          printf '%s' "$FOUND" | grep -q "Smoke page ${RUN_ID}" || { echo "search did not find the page: ${FOUND}" >&2; exit 1; }
642          END=$(date +%s)
643          ELAPSED=$((END - START))
644          echo "smoke GREEN in ${ELAPSED}s (budget 120s)"
645          [ "$ELAPSED" -lt 120 ] || { echo "smoke exceeded the 2-minute budget (${ELAPSED}s)" >&2; exit 1; }
646 
647  # ---------------------------------------------------------------------------
648  # Stage 8 — performance (hard thresholds): the 4 TASK-0135 suites
649  # (api-load, database-load, realtime-load, editor-local) + the 100k seed.
650  # The dev/PR run of the same suites is report-only; HERE thresholds are
651  # hard (K6_STRICT=1) — a breach blocks the release.
652  # ---------------------------------------------------------------------------
653  stage-8-performance:
654    name: "[8/12] performance (hard)"
655    runs-on: ubuntu-24.04
656    timeout-minutes: 180
657    needs: [gate, stage-7-smoke]
658    if: needs.gate.outputs.from == 8
659    environment:
660      name: staging
661    env:
662      DEPLOY_STAGING: ${{ vars.DEPLOY_STAGING }}
663      # Staging load target (the perf suites run against the deployed stack).
664      K6_API_URL: ${{ vars.STAGING_API_URL }}
665      K6_REALTIME_URL: ${{ vars.STAGING_WS_URL }}
666      K6_EMAIL: demo@factory.local
667      K6_PASSWORD: Factory!2345
668      # Hard thresholds: a breach FAILS the run (dev/PR is report-only).
669      K6_STRICT: "1"
670      # Seed fixture connects to staging Postgres (least-priv seed identity).
671      DATABASE_URL: ${{ secrets.STAGING_DATABASE_URL }}
672    steps:
673      - name: Checkout
674        uses: actions/checkout@v4
675      - name: Setup pnpm
676        uses: pnpm/action-setup@v4
677        with:
678          version: ${{ env.PNPM_VERSION }}
679      - name: Setup Node.js
680        uses: actions/setup-node@v4
681        with:
682          node-version: 20
683          cache: pnpm
684      - name: Install (frozen lockfile)
685        run: pnpm install --frozen-lockfile
686      - name: Install k6 (pinned)
687        run: |
688          K6_VERSION="0.50.0"
689          curl -sSfL "https://github.com/grafana/k6/releases/download/v${K6_VERSION}/k6-v${K6_VERSION}-linux-amd64.tar.gz" \
690            | tar -xz -C /usr/local/bin k6
691          k6 version
692      - name: Resolve perf fixtures (workspace / page / db ids)
693        if: env.DEPLOY_STAGING == 'true'
694        run: |
695          echo "Perf fixture ids come from the staging seed (demo user's workspace,"
696          echo "the seeded 100k database, and the 200 smoke pages). Exported:"
697          echo "  K6_WORKSPACE_ID=${{ vars.STAGING_PERF_WORKSPACE_ID }}"
698          echo "  K6_DB_ID=${{ vars.STAGING_PERF_DB_ID }}"
699          echo "  K6_PAGE_IDS=${{ vars.STAGING_PERF_PAGE_IDS }}"
700      - name: Seed the 100k fixture (idempotent — marker table test_seeds)
701        if: env.DEPLOY_STAGING == 'true'
702        run: pnpm --dir tests/performance seed:100k
703      - name: api-load (1k rps, p95 < 300 ms, errors < 0.1%)
704        if: env.DEPLOY_STAGING == 'true'
705        run: pnpm --dir tests/performance api-load
706      - name: database-load (100k records, table p95 < 500 ms, 100 rps creates)
707        if: env.DEPLOY_STAGING == 'true'
708        run: pnpm --dir tests/performance database-load
709      - name: realtime-load (10k conns / 200 msg/s, update p95 < 200 ms)
710        if: env.DEPLOY_STAGING == 'true'
711        run: pnpm --dir tests/performance realtime-load
712      - name: editor-local (interaction p95 < 50 ms, block move < 16 ms)
713        if: env.DEPLOY_STAGING == 'true'
714        run: |
715          WEB_BASE_URL="${{ vars.STAGING_WEB_URL }}" \
716          API_BASE_URL="${{ vars.STAGING_API_URL }}" \
717          pnpm --dir tests/performance editor-local
718      - name: Upload perf reports
719        if: always()
720        uses: actions/upload-artifact@v4
721        with:
722          name: perf-reports
723          path: tests/performance/reports
724          if-no-files-found: warn
725      - name: Placeholder verdict
726        if: env.DEPLOY_STAGING != 'true'
727        run: |
728          echo "SKIPPED (loud): staging is not wired — performance cannot run against"
729          echo "CI services (the suites target the deployed stack). The DEV/PR pipeline"
730          echo "runs the same suites report-only; this stage is hard-once-staging-exists."
731          exit 1
732 
733  # ---------------------------------------------------------------------------
734  # Stage 9 — backup-health (hard): the TASK-0135 backup/DR gate (runbook §8.7).
735  # Reads the REAL backup artifacts (no synthetic markers):
736  #   - newest object LastModified under pg/basebackups_005/  < 26 h
737  #   - newest object LastModified under pg/wal_005/          < 10 min
738  #   - newest SUCCESS snapshot start_time in repo factory-s3 < 26 h
739  # Semantics (scripts/backup-health.ts): reachable-but-stale or auth errors
740  # ALWAYS fail; an unreachable/empty endpoint is a loud report-only SKIP
741  # (exit 0) in dev, and a hard FAIL when DEPLOY_STAGING=true /
742  # BACKUP_HEALTH_STRICT=1 — so in staging the gate bites where the backup
743  # stack actually exists. A release whose rollback backup is stale must not
744  # ship.
745  # ---------------------------------------------------------------------------
746  stage-9-backup-health:
747    name: "[9/12] backup health (PITR/OS gate)"
748    runs-on: ubuntu-24.04
749    timeout-minutes: 10
750    needs: [gate, stage-8-performance]
751    if: needs.gate.outputs.from == 9
752    environment:
753      name: staging
754    env:
755      CI: true
756      # Endpoints of the BACKUP stack (the S3 the CronJobs write to + the
757      # OpenSearch the snapshot CronJob targets). Defaults match local dev;
758      # set them in the staging environment to the real values.
759      S3_ENDPOINT: ${{ vars.BACKUP_S3_ENDPOINT }}
760      BACKUP_S3_BUCKET: ${{ vars.BACKUP_S3_BUCKET }}
761      OS_URL: ${{ vars.BACKUP_OS_URL }}
762      # Hard-gate in staging (unreachable = FAIL); report-only in dev.
763      DEPLOY_STAGING: ${{ vars.DEPLOY_STAGING }}
764    steps:
765      - name: Checkout
766        uses: actions/checkout@v4
767      - name: Setup pnpm
768        uses: pnpm/action-setup@v4
769        with:
770          version: ${{ env.PNPM_VERSION }}
771      - name: Setup Node.js
772        uses: actions/setup-node@v4
773        with:
774          node-version: 20
775          cache: pnpm
776      - name: Install dependencies
777        run: pnpm install --frozen-lockfile
778      - name: Assert backup freshness (26 h / 10 min / 26 h)
779        env:
780          # Credentials for the backup identity (least-priv, see
781          # infrastructure/kubernetes/postgres/walg-iam-policy.yaml).
782          S3_ACCESS_KEY: ${{ secrets.BACKUP_S3_ACCESS_KEY }}
783          S3_SECRET_KEY: ${{ secrets.BACKUP_S3_SECRET_KEY }}
784          S3_REGION: ${{ vars.BACKUP_S3_REGION }}
785          BACKUP_S3_PG_PREFIX: pg
786          OS_SNAPSHOT_REPO: factory-s3
787        run: pnpm backup:health
788 
789  # ---------------------------------------------------------------------------
790  # Stage 10 — final-e2e (hard): tests/e2e/final-flow.spec.ts — the 13-step
791  # sequential final flow, against the staging URL. All green = the release
792  # is authorized (the tag step, 12, may now run).
793  # ---------------------------------------------------------------------------
794  stage-10-final-e2e:
795    name: "[10/12] final E2E (13-step flow)"
796    runs-on: ubuntu-24.04
797    timeout-minutes: 45
798    needs: [gate, stage-9-backup-health]
799    if: needs.gate.outputs.from == 10
800    environment:
801      name: staging
802    env:
803      E2E_BASE_URL: ${{ vars.STAGING_WEB_URL }}
804      E2E_API_URL: ${{ vars.STAGING_API_URL }}
805      E2E_INTERNAL_SERVICE_TOKEN: ${{ secrets.STAGING_INTERNAL_SERVICE_TOKEN }}
806      INTERNAL_TEST_SEAM: "1"
807      AI_PROVIDER: mock
808      AI_MOCK_DELTA_DELAY_MS: "60"
809    steps:
810      - name: Checkout
811        uses: actions/checkout@v4
812      - name: Setup pnpm
813        uses: pnpm/action-setup@v4
814        with:
815          version: ${{ env.PNPM_VERSION }}
816      - name: Setup Node.js
817        uses: actions/setup-node@v4
818        with:
819          node-version: 20
820          cache: pnpm
821      - name: Install (frozen lockfile)
822        run: pnpm install --frozen-lockfile
823      - name: Install Playwright browsers
824        run: pnpm --dir tests/e2e exec playwright install --with-deps chromium
825      - name: Run the 13-step final flow (dedicated config)
826        run: pnpm --dir tests/e2e exec playwright test --config playwright.final-flow.config.ts
827      - name: Upload E2E report
828        if: always()
829        uses: actions/upload-artifact@v4
830        with:
831          name: final-e2e-report
832          path: tests/e2e/playwright-report
833          if-no-files-found: warn
834 
835  # ---------------------------------------------------------------------------
836  # Stage 11 — docs-completeness (hard): scripts/docs-completeness.mjs
837  # (READMEs per package, OpenAPI + docs/api coverage, ROADMAP Phase 15
838  # boxes). Release mode: --require-phase-15-ticked — the boxes are ticked
839  # last (by the coordinator, after this release is green), so this stage is
840  # re-run (from_stage: 11) once they land.
841  # ---------------------------------------------------------------------------
842  stage-11-docs-completeness:
843    name: "[11/12] docs completeness"
844    runs-on: ubuntu-24.04
845    timeout-minutes: 10
846    needs: [gate, stage-10-final-e2e]
847    if: needs.gate.outputs.from == 11
848    steps:
849      - name: Checkout
850        uses: actions/checkout@v4
851      - name: Setup Node.js
852        uses: actions/setup-node@v4
853        with:
854          node-version: 20
855      - name: Run docs-completeness (release mode — Phase 15 boxes ticked)
856        run: node scripts/docs-completeness.mjs --require-phase-15-ticked
857 
858  # ---------------------------------------------------------------------------
859  # Stage 12 — push+tag (hard; THE LAST ACT).
860  #   1. Push the four GHCR images: tag = <version>, plus `latest` for
861  #      vX.0.0 releases (is_patch), plus the commit sha. Idempotent —
862  #      re-pushing the same tags/digests is a no-op.
863  #   2. Create the ANNOTATED tag v<version> (idempotent: never moves an
864  #      existing tag).
865  #   3. Publish the GitHub release (notes: CHANGELOG section + digests +
866  #      run links) as a DRAFT (the human publishes — v1.0.0 is deliberate).
867  # ---------------------------------------------------------------------------
868  stage-12-push-tag:
869    name: "[12/12] push + tag + release"
870    runs-on: ubuntu-24.04
871    timeout-minutes: 45
872    needs: [gate, stage-11-docs-completeness]
873    if: needs.gate.outputs.from == 12
874    steps:
875      - name: Checkout
876        uses: actions/checkout@v4
877        with:
878          fetch-depth: 0 # full history → the annotated tag can live anywhere
879      - name: Log in to GitHub Container Registry
880        uses: docker/login-action@v3
881        with:
882          registry: ${{ env.REGISTRY }}
883          username: ${{ github.actor }}
884          password: ${{ secrets.GITHUB_TOKEN }}
885      - name: Set up Docker Buildx
886        uses: docker/setup-buildx-action@v3
887      - name: Push the four images (version + [latest] + sha)
888        env:
889          VERSION: ${{ needs.gate.outputs.version }}
890          IS_PATCH: ${{ needs.gate.outputs.is_patch }}
891        run: |
892          set -euo pipefail
893          : "${VERSION:?}"
894          for app in api web worker realtime; do
895            IMAGE="${REGISTRY}/${IMAGE_OWNER}/${app}"
896            TAGS="${IMAGE}:${VERSION}"
897            if [ "$IS_PATCH" = "true" ]; then
898              TAGS="${TAGS},${IMAGE}:latest"
899            fi
900            echo "=== ${IMAGE}:${VERSION} ${IS_PATCH:+"(+ latest)"} ==="
901            docker buildx build \
902              --file "apps/${app}/Dockerfile" \
903              --tag "$TAGS" \
904              --push \
905              --cache-from type=gha,scope="${app}" \
906              --cache-to type=gha,scope="${app}" \
907              --label "org.opencontainers.image.source=${GITHUB_REPOSITORY_URL}" \
908              --label "org.opencontainers.image.revision=${GITHUB_SHA}" \
909              --label "org.opencontainers.image.version=${VERSION}" \
910              .
911          done
912          echo "all four images pushed"
913      - name: Registry check (tag + latest exist)
914        env:
915          VERSION: ${{ needs.gate.outputs.version }}
916          IS_PATCH: ${{ needs.gate.outputs.is_patch }}
917        run: |
918          set -euo pipefail
919          for app in api web worker realtime; do
920            IMAGE="${REGISTRY}/${IMAGE_OWNER}/${app}"
921            # `docker manifest inspect` proves the tag is pullable from GHCR.
922            docker manifest inspect "${IMAGE}:${VERSION}" >/dev/null
923            [ "$IS_PATCH" = "true" ] && docker manifest inspect "${IMAGE}:latest" >/dev/null
924            echo "${IMAGE}:${VERSION} present"
925          done
926          echo "registry check: OK"
927      - name: Digests to job summary
928        env:
929          VERSION: ${{ needs.gate.outputs.version }}
930          IS_PATCH: ${{ needs.gate.outputs.is_patch }}
931          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
932        run: |
933          set -euo pipefail
934          echo "### Published images — v${VERSION}" >> "$GITHUB_STEP_SUMMARY"
935          echo "" >> "$GITHUB_STEP_SUMMARY"
936          for app in api web worker realtime; do
937            IMAGE="${REGISTRY}/${IMAGE_OWNER}/${app}"
938            DIGEST="$(docker buildx imagetools inspect "${IMAGE}:${VERSION}" 2>/dev/null \
939              | awk -F': ' '/^  digest:/{print $2}' | head -1)"
940            echo "- \`${IMAGE}:${VERSION}\` — \`${DIGEST:-see build log}\`" >> "$GITHUB_STEP_SUMMARY"
941            [ "$IS_PATCH" = "true" ] && echo "- \`${IMAGE}:latest\` → v${VERSION}" >> "$GITHUB_STEP_SUMMARY"
942          done
943          echo "" >> "$GITHUB_STEP_SUMMARY"
944          echo "commit: \`${GITHUB_SHA}\`" >> "$GITHUB_STEP_SUMMARY"
945      - name: Create annotated tag (idempotent — never moves an existing tag)
946        env:
947          VERSION: ${{ needs.gate.outputs.version }}
948        run: |
949          set -euo pipefail
950          TAG="v${VERSION}"
951          if git ls-remote --tags origin "refs/tags/${TAG}" | grep -q .; then
952            echo "tag ${TAG} already exists — leaving it exactly where it is (idempotent re-run)"
953            exit 0
954          fi
955          # The tag is created BY this workflow on the tested commit
956          # (this dispatch's head — the code all 11 preceding stages ran).
957          # The release notes carry the run link, so provenance is explicit.
958          git tag -a "${TAG}" -m "Release v${VERSION}" "${GITHUB_SHA}"
959          git push origin "refs/tags/${TAG}"
960          echo "tagged ${GITHUB_SHA} as ${TAG}"
961      - name: Publish GitHub release (draft)
962        env:
963          VERSION: ${{ needs.gate.outputs.version }}
964          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
965          GH_REPO: ${{ github.repository }}
966        run: |
967          set -euo pipefail
968          TAG="v${VERSION}"
969          RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
970          # Notes: the CHANGELOG section for this version (if present) + the
971          # run link + the runbook links.
972          NOTES=""
973          if [ -f CHANGELOG.md ]; then
974            SECTION="$(awk -v v="## [${VERSION}]" 'index($0,v)==1{f=1;next} f&&/^## \[/{exit} f' CHANGELOG.md || true)"
975            NOTES="${SECTION:-_CHANGELOG.md has no [${VERSION}] section — add it._}"
976          fi
977          NOTES="${NOTES}
978 
979          ---
980          - Release run: ${RUN_URL}
981          - Runbooks: [deployment](docs/development/deployment.md) · [admin guide](docs/development/admin-guide.md) · [support matrix](docs/support-matrix.md)
982          "
983          if gh release view "${TAG}" >/dev/null 2>&1; then
984            echo "release ${TAG} exists — leaving it untouched (idempotent re-run)"
985          else
986            gh release create "${TAG}" --title "v${VERSION}" --notes "${NOTES}" --draft
987            echo "draft release ${TAG} created"
988          fi
989