/** * T-176 — org-admin JOBS tab (background job status + failures). * * BullMQ is CLUSTER-wide (every org shares the seven worker queues), so this * tab reports cluster queue state — the org-admin gate is the access control * (an org admin is a trusted platform operator; the response says * `scope: "cluster"` so the UI never pretends the numbers are org-scoped). * * Surface: * GET /orgs/:orgId/admin/jobs — per-queue state counts * GET /orgs/:orgId/admin/jobs/failed — recent failed jobs (the DLQ view) * POST /orgs/:orgId/admin/jobs/requeue — re-run ONE failed job (audited) * * The queue name is a server-side enum (never caller-supplied string → * `new Queue()`). Requeue resets the job's attempt counter and * returns it to waiting (BullMQ's `Job.retry`) — the standard DLQ re-run. */ import { Inject, Injectable } from "@nestjs/common"; import { z } from "zod"; import type { Request } from "express"; import { PrismaService } from "../../common/prisma.service.js"; import { RedisService } from "../../common/redis.service.js"; import { AuditService } from "../../common/audit.service.js"; import { ApiError } from "../../common/errors.js"; import { requireUser } from "../../common/types.js"; import { assertOrgAdmin } from "./guards.js"; /** * The seven worker queues (mirrors `apps/worker/src/queues.ts::QUEUES` — kept * as a literal here so the API does not import the worker package; the names * are a stable contract, and a name added to the worker without adding it here * simply stays hidden from this tab). */ const QUEUE_NAMES = ["files", "search-index", "webhooks", "automations", "email", "ai", "maintenance"] as const; export type AdminQueueName = (typeof QUEUE_NAMES)[number]; /** The failed-jobs list cap (one page of the DLQ view). */ const FAILED_LIST_CAP = 100; /** How much of a job's payload to surface (DLQ payloads can be big). */ const PAYLOAD_PREVIEW_CHARS = 2000; const RequeueBody = z.object({ queue: z.enum(QUEUE_NAMES), jobId: z.string().min(1).max(128) }); @Injectable() export class OrgJobsService { constructor( @Inject(PrismaService) private readonly prisma: PrismaService, @Inject(RedisService) private readonly redis: RedisService, @Inject(AuditService) private readonly audit: AuditService ) {} /** Per-queue state counts (waiting/active/completed/failed). */ async overview(orgId: string, req: Request) { const userId = requireUser(req); await assertOrgAdmin(this.prisma, orgId, userId); const queues: Array<{ name: AdminQueueName; waiting: number; active: number; completed: number; failed: number }> = []; // One queue handle at a time (the house pattern — the API never keeps // long-lived BullMQ connections; each handle is closed after use). for (const name of QUEUE_NAMES) { const q = await this.openQueue(name); try { const [waiting, active, completed, failed] = await Promise.all([ q.getJobCountByTypes("waiting", "delayed"), q.getJobCountByTypes("active"), q.getJobCountByTypes("completed"), q.getJobCountByTypes("failed") ]); queues.push({ name, waiting, active, completed, failed }); } finally { await q.close(); } } return { // BullMQ is cluster-wide (all orgs share the worker queues) — the // org-admin gate is the access control; the label keeps the UI honest. scope: "cluster" as const, queues }; } /** Recent failed jobs (the DLQ view) — all queues or one. */ async failed( orgId: string, req: Request, queue?: string, limit = 20 ): Promise<{ scope: "cluster"; jobs: Array> }> { const userId = requireUser(req); await assertOrgAdmin(this.prisma, orgId, userId); const cap = Math.max(1, Math.min(limit, FAILED_LIST_CAP)); const names = queue ? [queue as AdminQueueName] : [...QUEUE_NAMES]; const jobs: Array> = []; for (const name of names) { const q = await this.openQueue(name); try { const failed = await q.getFailed(0, cap - 1); for (const j of failed) { jobs.push(this.jobView(name, j)); } } finally { await q.close(); } } jobs.sort((a, b) => String(b.failedAt).localeCompare(String(a.failedAt))); return { scope: "cluster", jobs: jobs.slice(0, cap) }; } /** * Re-run ONE failed job (resets its attempt counter, back to waiting). * The job must actually be in the failed state — requeueing a live job * would be a no-op surprise, so it is a 409 instead. */ async requeue( orgId: string, req: Request, body: { queue: AdminQueueName; jobId: string } ): Promise<{ queued: true; queue: AdminQueueName; jobId: string }> { const userId = requireUser(req); await assertOrgAdmin(this.prisma, orgId, userId); const q = await this.openQueue(body.queue); try { const job = await q.getJob(body.jobId); if (!job) { // No-leak: a job id from another queue (or a vanished id) is a 404. throw ApiError.notFound("Job not found in this queue."); } const state = await job.getState(); if (state !== "failed") { throw ApiError.conflict("Only failed jobs can be requeued.", [ { reason: "job_not_failed", state } ]); } await job.retry(); await this.audit.audit(req, req.auth, { action: "jobs.failed_job_requeued", resourceType: "queue", resourceId: body.queue, metadata: { orgId, queue: body.queue, jobId: body.jobId, jobName: job.name } }); return { queued: true, queue: body.queue, jobId: body.jobId }; } finally { await q.close(); } } /** One failed-job row for the DLQ view (payload truncated). */ private jobView(name: AdminQueueName, j: { id?: string; name: string; data?: unknown; failedReason?: string | null; /** BullMQ surfaces the failure timestamp as `finishedOn` (ms since epoch). */ finishedOn?: number; attemptsMade: number; }) { const payload = (() => { try { const s = JSON.stringify(j.data ?? null); return s.length > PAYLOAD_PREVIEW_CHARS ? `${s.slice(0, PAYLOAD_PREVIEW_CHARS)}…` : s; } catch { return null; } })(); return { queue: name, jobId: j.id ?? null, name: j.name, payload, failedReason: j.failedReason ? String(j.failedReason).slice(0, 500) : null, // BullMQ surfaces the failure timestamp as `finishedOn` (ms since epoch). failedAt: j.finishedOn ? new Date(j.finishedOn).toISOString() : null, attemptsMade: j.attemptsMade }; } private async openQueue(name: AdminQueueName) { const { Queue } = await import("bullmq"); return new Queue(name, { connection: this.redis.getRedis().options as never }); } }