/** * T-176 / T-177 / T-178 — org-admin HEALTH tabs (jobs + search + notifications). * * The three new org-admin routes are all `assertOrgAdmin`-gated server-side * (unknown org → 404 no-leak; org MEMBER/GUEST/non-member → 403 * `requires_org_admin`). This spec proves the guard matrix on all three, the * happy-path shape of each, the audited requeue mutation, and the * "search down → healthy:false (200, never 500)" contract. * * GET /orgs/:orgId/admin/jobs — per-queue counts (cluster scope) * GET /orgs/:orgId/admin/jobs/failed — recent failed jobs (DLQ view) * POST /orgs/:orgId/admin/jobs/requeue — re-run a failed job (+audit) * GET /orgs/:orgId/admin/search/health — doc count / lag / last reconcile * GET /orgs/:orgId/admin/notifications/health — in-app + email delivery * * Search is proven against the dev/CI OpenSearch the API already targets * (SEARCH_URL, default localhost:9200) — the house pattern * (search.integration.test.ts). When the stack is absent (a Docker-less * authoring host) the whole suite self-SKIPS (a top-level stack gate), it * never fails. On the dev stack the `factory-content` alias already resolves * → we write through it (never touch it); on a fresh CI stack we create the * plain index. Either way we seed 2 docs with a run-unique workspaceId and * countDocs is TENANT-FILTERED → docCount === sourceCount === 2 exactly. */ import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { execSync } from "node:child_process"; import { Queue, Worker } from "bullmq"; import Redis from "ioredis"; import { SearchClient, contentMapping } from "@factory/search"; import { ErrorCodes } from "@factory/shared"; import { getRedisUrl } from "./helpers/containers"; import { createTestApp, type TestAppHandle } from "./helpers/api"; import { authHeaders, createWorkspace, createUser } from "./helpers/seed"; import type { PrismaClient } from "@prisma/client"; /** The OpenSearch the API talks to (API's SEARCH_URL default). House pattern: * the full dev/CI stack's search engine, NOT a container this spec controls * (the host can be disk-constrained — a fresh container would crash). */ const SEARCH_URL = (process.env.SEARCH_URL ?? "http://localhost:9200").replace(/\/+$/, ""); function dockerAvailable(): boolean { try { execSync("docker info", { stdio: "ignore", timeout: 8_000 }); return true; } catch { return false; } } async function opensearchUp(url: string): Promise { try { const res = await fetch(`${url}/_cluster/health`, { signal: AbortSignal.timeout(5_000) }); return res.ok; } catch { return false; } } /** Collection-time stack gate (Docker + OpenSearch) — self-skip when the dev * stack is absent (a Docker-less authoring host), never a fake pass. */ const stackReady = dockerAvailable() && (await opensearchUp(SEARCH_URL)); let app: TestAppHandle; // The BullMQ Redis + queue handles are created in beforeAll (NOT at module // import time): createTestApp() sets process.env.REDIS_URL, so a module-level // `new Queue` would fall back to a local :6379 and hit a DIFFERENT Redis than // the app (the app's queues live in the testcontainers Redis). let redis: Redis; let qEmail: Queue; /** Direct client on the SAME OpenSearch the API targets (SEARCH_URL). */ let os: SearchClient; let createdIndex = false; const json = async (res: Response): Promise> => (await res.json().catch(() => ({}))) as Record; const reasonOf = (body: Record): string | undefined => ((body.error as { details?: Array<{ reason?: string }> })?.details?.[0]?.reason); async function makeOrg(tok: string): Promise { const res = await app.request("POST", "/api/v1/orgs", { headers: authHeaders(tok), body: { name: `Health Org ${Date.now()}-${Math.random().toString(36).slice(2, 6)}` } }); const body = (await json(res)) as { id?: string; organization?: { id?: string } }; const orgId = body.id ?? body.organization?.id; if (!orgId) throw new Error(`makeOrg failed: ${res.status} ${await res.text()}`); return orgId; } async function addOrgMember(prisma: PrismaClient, orgId: string, userId: string, role: "ADMIN" | "MEMBER" | "GUEST") { await prisma.organizationMember.upsert({ where: { organizationId_userId: { organizationId: orgId, userId } }, update: { role }, create: { organizationId: orgId, userId, role } }); } const RUN = Date.now().toString(36) + Math.random().toString(36).slice(2, 6); let owner: Awaited>; let admin: Awaited>; let member: Awaited>; let orgId = ""; let wsId = ""; let failedJobId = ""; // the UUID BullMQ assigned (the `email:{id}` dedup key is rejected by BullMQ 5.x — see the T-178 note) let seedNotifId = ""; // A whole doc (type page) — chunk docs are excluded from countDocs. function pageDoc(workspaceId: string, id: string): Record { return { docId: `page:${id}`, type: "page", id, workspaceId, title: `Health page ${id}`, body: "health doc body", grants: { public: false, workspace: true, user: false, teams: [], users: [] }, archived: false, url: `/w/${workspaceId}/page/${id}`, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString() }; } describe.skipIf(!stackReady)("T-176/177/178: org-admin health tabs", () => { beforeAll(async () => { // The API targets the dev/CI OpenSearch at SEARCH_URL (default // localhost:9200) — the house pattern (search.integration.test.ts). Our // direct client targets the same instance. os = new SearchClient({ baseUrl: SEARCH_URL }); app = await createTestApp(); // Queue handles on the SAME Redis the app uses (REDIS_URL is now set). const url = await getRedisUrl(); redis = new Redis(url, { maxRetriesPerRequest: null }); qEmail = new Queue("email", { connection: redis }); owner = await createUser(app.request, { name: "H Owner" }); admin = await createUser(app.request, { name: "H Admin" }); member = await createUser(app.request, { name: "H Member" }); orgId = await makeOrg(owner.accessToken!); await addOrgMember(app.prisma, orgId, admin.userId, "ADMIN"); await addOrgMember(app.prisma, orgId, member.userId, "MEMBER"); const ws = await createWorkspace(app.request, authHeaders(owner.accessToken!), `H WS ${RUN}`); wsId = ws.workspaceId; await app.prisma.workspace.update({ where: { id: wsId }, data: { organizationId: orgId } }); // Index bootstrap (house pattern, search.integration.test.ts): on the DEV // stack the `factory-content` alias already resolves (to factory-content-vN) // → write through it, NEVER touch the alias. On a FRESH CI stack it is // absent → create the plain index with the canonical mapping. Either way we // seed with a run-unique workspaceId, and countDocs is TENANT-FILTERED, so // docCount is exactly the docs WE seed regardless of the shared index's // other content. if (!(await os.indexExists("factory-content"))) { const map = contentMapping(); await os.createIndexIfMissing("factory-content", { settings: map.settings as Record, mappings: map.mappings as Record }); createdIndex = true; } // A real FAILED email job — a short-lived failing worker consumes it // (attempts:1 → failed). The email queue has no other integration writer, // so this is interference-free and realistic. The job's payload mirrors the // worker's `buildInstantEmailData` output (`notificationIds: [id]`); the // service correlates on the PAYLOAD, and BullMQ assigns a UUID id (the // worker's `email:{id}` dedup key is rejected by BullMQ 5.x — T-178 note). const notif = await app.prisma.notification.create({ data: { workspaceId: wsId, userId: member.userId, type: "COMMENT", channel: "IN_APP", title: "Health fail probe" } }); seedNotifId = notif.id; const w = new Worker("email", () => Promise.reject(new Error("boom-t178")), { connection: redis, concurrency: 1 }); const job = await qEmail.add("email-notification", { notificationIds: [notif.id] }, { attempts: 1 }); await w.waitUntilReady(); // Poll for the failed state (a throwaway worker is the only thing that can // move a waiting job to failed — moveToFailed requires the active list). const deadline = Date.now() + 15_000; while (Date.now() < deadline) { const state = await (await qEmail.getJob(job.id!))?.getState(); if (state === "failed") break; if (state === "completed") throw new Error(`seed job ${job.id} completed, expected failed`); await new Promise((r) => setTimeout(r, 50)); } expect(await (await qEmail.getJob(job.id!))?.getState()).toBe("failed"); await w.close(); failedJobId = job.id!; }, 180_000); afterAll(async () => { await qEmail?.close(); await redis?.quit().catch(() => undefined); // Index cleanup: a FRESH-stack plain index (we created it) is dropped // whole; a LIVE dev-stack index is only pruned of THIS run's tenant docs // (delete-by-query on our unique workspaceId) — the shared alias is never // touched. if (createdIndex) { await fetch(`${SEARCH_URL}/factory-content?expand_wildcards=all`, { method: "DELETE" }).catch( () => undefined ); } else if (wsId) { await fetch(`${SEARCH_URL}/factory-content/_delete_by_query`, { method: "POST", body: JSON.stringify({ query: { term: { workspaceId: wsId } } }), headers: { "content-type": "application/json" } }).catch(() => undefined); } // Remove the org (cascades its workspaces / notifications / search docs). if (app && orgId) { const orgWorkspaces = await app.prisma.workspace.findMany({ where: { organizationId: orgId }, select: { id: true } }); await app.prisma.organization.deleteMany({ where: { id: orgId } }); await app.prisma.workspace.deleteMany({ where: { id: { in: orgWorkspaces.map((w) => w.id) } } }); } await app?.close(); }, 30_000); it("guard matrix: owner+admin pass; member/non-member → 403; unknown org → 404 (all 3 routes)", async () => { const paths = ["/jobs", "/search/health", "/notifications/health"]; for (const p of paths) { const ok = await app.request("GET", `/api/v1/orgs/${orgId}/admin${p}`, { headers: authHeaders(owner.accessToken!) }); expect(ok.status).toBe(200); const okA = await app.request("GET", `/api/v1/orgs/${orgId}/admin${p}`, { headers: authHeaders(admin.accessToken!) }); expect(okA.status).toBe(200); const denied = await app.request("GET", `/api/v1/orgs/${orgId}/admin${p}`, { headers: authHeaders(member.accessToken!) }); expect(denied.status).toBe(403); expect(reasonOf(await json(denied))).toBe("requires_org_admin"); } // non-member (no org membership at all) → 403 on one route const outsider = await createUser(app.request, { name: "H Outsider" }); const no403 = await app.request("GET", `/api/v1/orgs/${orgId}/admin/jobs`, { headers: authHeaders(outsider.accessToken!) }); expect(no403.status).toBe(403); expect(reasonOf(await json(no403))).toBe("requires_org_admin"); // unknown org → 404 (no-leak) const unknown = await app.request("GET", `/api/v1/orgs/does-not-exist/admin/jobs`, { headers: authHeaders(owner.accessToken!) }); expect(unknown.status).toBe(404); }); it("GET jobs: cluster scope, 7 queues each with 4 state counters", async () => { const res = await app.request("GET", `/api/v1/orgs/${orgId}/admin/jobs`, { headers: authHeaders(owner.accessToken!) }); expect(res.status).toBe(200); const body = (await json(res)) as { scope: string; queues: Array<{ name: string; waiting: number; active: number; completed: number; failed: number }>; }; expect(body.scope).toBe("cluster"); expect(body.queues.map((q) => q.name).sort()).toEqual( ["ai", "automations", "email", "files", "maintenance", "search-index", "webhooks"] ); for (const q of body.queues) { for (const k of ["waiting", "active", "completed", "failed"]) { expect(typeof q[k]).toBe("number"); } } }); it("GET search/health: sourceCount, docCount (delta = seeded), lag + lastIndexed", async () => { // Seed 2 mirror rows + 2 whole docs in the controlled index. const docA = `pgA-${RUN}`; const docB = `pgB-${RUN}`; await app.prisma.searchDocument.createMany({ data: [ { workspaceId: wsId, type: "page", refId: docA, title: `A`, permissions: { public: false, workspace: true } as object }, { workspaceId: wsId, type: "page", refId: docB, title: `B`, permissions: { public: false, workspace: true } as object } ] }); await os.indexDocs([ pageDoc(wsId, docA) as never, pageDoc(wsId, docB) as never ]); // Force a refresh so the just-indexed docs are visible to _search (OpenSearch // auto-refreshes only every ~1s; a deterministic count needs it immediate). await os.refresh(); const res = await app.request("GET", `/api/v1/orgs/${orgId}/admin/search/health`, { headers: authHeaders(owner.accessToken!) }); expect(res.status).toBe(200); const body = (await json(res)) as { healthy: boolean; error: string | null; workspaces: number; sourceCount: number; docCount: number | null; drift: number | null; lagMs: number | null; lastIndexedAt: string | null; lastReconcileAt: string | null; }; expect(body.healthy).toBe(true); expect(body.error).toBeNull(); expect(body.workspaces).toBe(1); expect(body.sourceCount).toBe(2); // countDocs is TENANT-FILTERED (terms workspaceId=[wsId]) and our wsId is // run-unique, so the count is exactly the 2 whole docs we seeded — even on // the shared dev index that holds other tenants' docs. expect(body.docCount).toBe(2); expect(body.drift).toBe(0); expect(body.lastIndexedAt).toBeTruthy(); expect(typeof body.lagMs).toBe("number"); expect(body.lagMs as number).toBeGreaterThanOrEqual(0); }); it("GET search/health with OpenSearch down → healthy:false, 200 (never 500)", async () => { // Same module instance the booted AppModule uses (the relative path // resolves to the file the app's SearchModule imports — the audit-completeness // spec uses this exact `app.get(Service)` + relative-import pattern). const { SearchService } = await import( /* webpackIgnore: true */ "../../apps/api/src/modules/search/search.service.ts" ); const ss = app.get!(SearchService) as { setClientForTesting: (c: unknown) => void; } | null; expect(ss).toBeTruthy(); // A client whose every call throws SEARCH_UNAVAILABLE (a dead endpoint). const dead = new SearchClient({ baseUrl: "http://127.0.0.1:1", fetchImpl: (async () => { throw new TypeError("fetch failed"); }) as typeof fetch }); ss!.setClientForTesting(dead); try { const res = await app.request("GET", `/api/v1/orgs/${orgId}/admin/search/health`, { headers: authHeaders(owner.accessToken!) }); expect(res.status).toBe(200); const body = (await json(res)) as { healthy: boolean; error: string; docCount: number | null }; expect(body.healthy).toBe(false); expect(body.error).toBe(ErrorCodes.SEARCH_UNAVAILABLE); expect(body.docCount).toBeNull(); } finally { ss!.setClientForTesting(null); // restore the real client } }); it("GET notifications/health: in-app volume + email failure correlated to org notif", async () => { const res = await app.request("GET", `/api/v1/orgs/${orgId}/admin/notifications/health`, { headers: authHeaders(owner.accessToken!) }); expect(res.status).toBe(200); const body = (await json(res)) as { workspaces: number; inAppTotal: number; inAppUnread: number; emailQueued: number; emailFailed: number; failedJobs: Array<{ notificationId: string; jobId: string; failedReason: string | null; attemptsMade: number }>; }; expect(body.workspaces).toBe(1); expect(body.inAppTotal).toBeGreaterThanOrEqual(1); expect(body.inAppUnread).toBeGreaterThanOrEqual(1); expect(typeof body.emailQueued).toBe("number"); expect(body.emailFailed).toBeGreaterThanOrEqual(1); // The email queue is SHARED (a prior run may have left failed jobs), so // locate OUR job by its payload notificationId (the service correlates on // the payload, not the jobId) — never assume it is first. const f = body.failedJobs.find((j) => j.notificationId === seedNotifId); expect(f).toBeTruthy(); expect(f!.jobId).toBe(failedJobId); // the UUID BullMQ assigned expect(f!.notificationId).toBe(seedNotifId); expect(f!.failedReason).toContain("boom-t178"); expect(f!.attemptsMade).toBe(1); }); it("GET jobs/failed (queue=email) lists the failed email job", async () => { const res = await app.request("GET", `/api/v1/orgs/${orgId}/admin/jobs/failed?queue=email`, { headers: authHeaders(owner.accessToken!) }); expect(res.status).toBe(200); const body = (await json(res)) as { scope: string; jobs: Array<{ queue: string; jobId: string }> }; expect(body.scope).toBe("cluster"); expect(body.jobs.some((j) => j.queue === "email" && j.jobId === failedJobId)).toBe(true); }); it("POST jobs/requeue: non-admin 403; admin 200 + job leaves failed + audit row", async () => { // non-admin → 403 const denied = await app.request("POST", `/api/v1/orgs/${orgId}/admin/jobs/requeue`, { headers: authHeaders(member.accessToken!), body: { queue: "email", jobId: failedJobId } }); expect(denied.status).toBe(403); expect(reasonOf(await json(denied))).toBe("requires_org_admin"); // admin → 200; the job leaves the failed state (back to waiting). const ok = await app.request("POST", `/api/v1/orgs/${orgId}/admin/jobs/requeue`, { headers: authHeaders(admin.accessToken!), body: { queue: "email", jobId: failedJobId } }); expect(ok.status).toBe(200); expect((await json(ok))).toMatchObject({ queued: true, queue: "email", jobId: failedJobId }); const state = await (await qEmail.getJob(failedJobId))?.getState(); expect(state).not.toBe("failed"); // The requeue is a sensitive mutation → audited. Org routes are // workspace-independent, so filter by action + the org in metadata (the // house pattern the reindex spec uses; the row's resourceType is "queue"). const audits = await app.prisma.auditLog.findMany({ where: { action: "jobs.failed_job_requeued", metadata: { path: ["orgId"], equals: orgId } } }); expect(audits.length).toBe(1); expect(audits[0]!.metadata).toMatchObject({ queue: "email", jobId: failedJobId }); }); });