automation.spec.ts

automation.spec.tstests/e2e/specs/automation.spec.tsraw ← specs35,492 B← Back to the app
1/**
2 * AI Software Factory — Phase 11 automations E2E (TASK-0114).
3 *
4 * FULL STACK REQUIRED — this spec drives the REAL end-to-end path:
5 *   browser (Next.js :3000) + API (:4000) + worker (tsx, real BullMQ
6 *   `automations` worker + `factory:events` stream consumer + 5-min
7 *   schedule reconciler) + Postgres + Redis.
8 *
9 *   pnpm docker:up && pnpm dev   (api :4000, web :3000, worker)
10 *   DATABASE_URL=... pnpm test:e2e -- automation
11 *
12 * There is intentionally NO `webServer` block (the whole stack must be
13 * running — see playwright.config.ts header).
14 *
15 * Scenarios (ROADMAP Phase 11 exit criteria):
16 *   1. record.created → UPDATE_RECORD + CREATE_NOTIFICATION → SUCCEEDED
17 *      with the property changed in < 10 s; the run + its FULL LOGS are
18 *      visible in the UI (settings → Automations → row → Runs → expand).
19 *   2. condition gate — a property condition (option NAME) skips the
20 *      non-matching record (output.skipped = "conditions_not_met") and
21 *      executes the matching one.
22 *   3. schedule (cron) trigger fires → a run with triggerType "schedule"
23 *      (the worker-side reconciler registers the repeatable; we invoke it
24 *      directly via the worker source so the test does not wait for the
25 *      5-minute cycle — the cron resolution still happens in real BullMQ).
26 *   4. pause stops execution — PAUSED automations match nothing;
27 *      re-activating resumes matching.
28 *   5. failure → FAILED run with the error surfaced in the UI run history,
29 *      and the UI Retry button re-runs the run. (The 8×-retry →
30 *      DEAD_LETTER path is covered hermetically in the integration spec —
31 *      no real network target in this environment produces a retryable
32 *      transport error, so a full E2E DLQ cycle cannot be hit
33 *      deterministically; recorded in the Phase 11 gap report.)
34 *   6. self-trigger loop prevention through the REAL stream + consumer +
35 *      worker: an UPDATE_RECORD emits record.updated, which re-matches the
36 *      same automation → the second run is skipped with
37 *      output.skipped = "self_trigger" and no third run follows.
38 *   7. manual run (DRAFT automation, POST /automations/:id/run → 202)
39 *      executes and its full per-action logs render in the UI.
40 *
41 * Data-testid contract (verified against apps/web, TASK-0114 UI):
42 *   automation-list, automation-row-{id}, automation-toggle-{id}
43 *   (role=switch), automation-runs-{id}, automation-run-{runId}
44 *   (status text lowercase, "dead-letter" for DEAD_LETTER),
45 *   automation-run-log-{runId}, automation-retry-run-{runId}.
46 *
47 * API note: databases are created through the page path
48 * (POST /workspaces/:id/pages { type: "database" } → { id, databaseId }) —
49 * the workspace-scoped database-create route does not exist in the API
50 * (pre-existing gap, also breaks the pre-existing databases integration
51 * suite; recorded in the Phase 11 gap report).
52 */
53import { spawn } from "node:child_process";
54import { randomBytes } from "node:crypto";
55import { writeFileSync, rmSync } from "node:fs";
56import path from "node:path";
57import type { APIRequestContext, Page } from "@playwright/test";
58import { test, expect } from "../fixtures";
59 
60// ---------------------------------------------------------------------------
61// Helpers (spec-local: tolerant of envelope shapes, exact on what is asserted)
62// ---------------------------------------------------------------------------
63 
64type Json = Record<string, unknown>;
65interface Run {
66  id: string;
67  status: string;
68  triggerType: string | null;
69  retryCount: number;
70  error: { code?: string; message?: string } | null;
71  createdAt: string;
72  startedAt: string | null;
73  finishedAt: string | null;
74  output?: unknown;
75}
76 
77const j = async (res: { json(): Promise<unknown> }): Promise<Json> =>
78  (await res.json().catch(() => ({}))) as Json;
79 
80async function expectOk(res: { status(): number; text(): Promise<string> }, what: string): Promise<Json> {
81  const status = res.status();
82  if (status < 200 || status >= 300) throw new Error(`${what} → ${status}: ${await res.text()}`);
83  return j(res);
84}
85 
86async function createDatabasePage(ctx: APIRequestContext, wsId: string, title: string): Promise<{ pageId: string; databaseId: string }> {
87  // The REAL database-creation surface: a page with type "database"
88  // (page.service.ts creates the Database row + seeds the Title property +
89  // embeds the database block). Returns the BARE page with databaseId.
90  const res = await ctx.post(`/api/v1/workspaces/${wsId}/pages`, { data: { title, type: "database" } });
91  const body = await expectOk(res, `create database page "${title}"`);
92  const databaseId = body.databaseId;
93  if (typeof databaseId !== "string" || !databaseId) throw new Error(`createDatabasePage: no databaseId in response: ${JSON.stringify(body)}`);
94  return { pageId: String(body.id), databaseId };
95}
96 
97async function addPropertyRaw(ctx: APIRequestContext, dbId: string, body: { name: string; type: string; config?: unknown }): Promise<void> {
98  await expectOk(
99    await ctx.post(`/api/v1/databases/${dbId}/properties`, { data: body }),
100    `add property ${body.name}`
101  );
102}
103 
104/** Create a property then look up its id by name (the list envelope is tolerated). */
105async function findPropertyId(ctx: APIRequestContext, dbId: string, name: string): Promise<string> {
106  const res = await ctx.get(`/api/v1/databases/${dbId}/properties`);
107  const body = await expectOk(res, `list properties of ${dbId}`);
108  const list = (Array.isArray(body) ? body : (body.data ?? body.properties ?? [])) as Json[];
109  const p = list.find((x) => x.name === name);
110  if (!p?.id) throw new Error(`property "${name}" not found in ${dbId}: ${JSON.stringify(list.map((x) => x.name))}`);
111  return String(p.id);
112}
113 
114async function createRecordRaw(ctx: APIRequestContext, dbId: string, properties: Record<string, unknown>): Promise<Json> {
115  const res = await ctx.post(`/api/v1/databases/${dbId}/records`, { data: { properties } });
116  const body = await expectOk(res, `create record in ${dbId}`);
117  const rec = (body.record ?? body) as Json;
118  if (!rec.id) throw new Error(`createRecordRaw: no record id: ${JSON.stringify(body)}`);
119  return rec;
120}
121 
122async function getRecord(ctx: APIRequestContext, recordId: string): Promise<Json> {
123  const res = await ctx.get(`/api/v1/records/${recordId}`);
124  return expectOk(res, `get record ${recordId}`);
125}
126 
127/** The SELECT value stored on a record property: the optionId (envelope-tolerant). */
128function selectValue(record: Json, propertyId: string): string | null {
129  const props = record.properties as Record<string, unknown> | undefined;
130  const v = props?.[propertyId];
131  if (v == null) return null;
132  if (typeof v === "object" && "optionId" in (v as Json)) return String((v as Json).optionId);
133  return String(v);
134}
135 
136/**
137 * On narrow (mobile) viewports the app shell keeps a fixed 256px sidebar and
138 * the automations table (min-width ~460px) overflows UNDER it, so the
139 * run-history rows render partially beneath the sidebar and their click point
140 * is intercepted by the aside. The intended mobile operation is the topbar
141 * "Toggle sidebar" button (collapses the aside to nothing, freeing the full
142 * width) — perform it on narrow viewports so the run rows are clickable.
143 * No-op on desktop (the sidebar stays expanded).
144 *
145 * NOTE: the app shell not auto-collapsing the sidebar on narrow viewports (and
146 * letting wide tables overflow under it) is a PRE-EXISTING app-shell
147 * mobile-layout limitation, not an automations bug — recorded in the Phase 11
148 * gap report. This helper exercises the intended collapse rather than masking
149 * the layout with a JS click.
150 */
151async function collapseSidebarIfNarrow(page: Page): Promise<void> {
152  const vp = page.viewportSize();
153  if (!vp || vp.width >= 768) return; // desktop — leave the sidebar as-is
154  if ((await page.locator("aside").count()) === 0) return; // already collapsed
155  await page.getByRole("button", { name: "Toggle sidebar" }).click();
156  await page.waitForTimeout(300);
157}
158 
159// --- automation API ---------------------------------------------------------
160 
161interface AutomationDetail {
162  id: string;
163  name: string;
164  status: string;
165  databaseId: string | null;
166  triggers: string[];
167  hasCondition: boolean;
168  actions: string[];
169}
170 
171async function createAutomation(ctx: APIRequestContext, wsId: string, input: Json): Promise<AutomationDetail> {
172  const res = await ctx.post(`/api/v1/workspaces/${wsId}/automations`, { data: input });
173  const body = await expectOk(res, `create automation "${input.name}"`);
174  if (!body.id) throw new Error(`createAutomation: no id: ${JSON.stringify(body)}`);
175  return body as unknown as AutomationDetail;
176}
177 
178async function setTriggers(ctx: APIRequestContext, autoId: string, triggers: Json[]): Promise<void> {
179  await expectOk(await ctx.put(`/api/v1/automations/${autoId}/triggers`, { data: { triggers } }), `set triggers on ${autoId}`);
180}
181 
182async function setCondition(ctx: APIRequestContext, autoId: string, condition: Json | null): Promise<void> {
183  await expectOk(await ctx.put(`/api/v1/automations/${autoId}/conditions`, { data: { condition } }), `set condition on ${autoId}`);
184}
185 
186async function setActions(ctx: APIRequestContext, autoId: string, actions: Json[]): Promise<void> {
187  await expectOk(await ctx.put(`/api/v1/automations/${autoId}/actions`, { data: { actions } }), `set actions on ${autoId}`);
188}
189 
190async function setStatus(ctx: APIRequestContext, autoId: string, status: string): Promise<void> {
191  await expectOk(await ctx.post(`/api/v1/automations/${autoId}/status`, { data: { status } }), `set status ${status} on ${autoId}`);
192}
193 
194async function listRuns(ctx: APIRequestContext, autoId: string): Promise<Run[]> {
195  const res = await ctx.get(`/api/v1/automations/${autoId}/runs?limit=200`);
196  const body = await expectOk(res, `list runs of ${autoId}`);
197  return (body.data as Run[]) ?? [];
198}
199 
200/**
201 * listRuns that is tolerant of 429 RATE_LIMITED: the API rate-limits per
202 * user/min (RATE_LIMIT_API, default 100), and this suite polls the runs
203 * endpoint in a tight loop — a 429 must not fail the test, it must just
204 * wait out the window (Retry-After) and re-poll. Returns null on a 429 so
205 * the caller can skip this poll cycle.
206 */
207async function listRunsTolerant(ctx: APIRequestContext, autoId: string): Promise<Run[] | null> {
208  const res = await ctx.get(`/api/v1/automations/${autoId}/runs?limit=200`);
209  if (res.status() === 429) {
210    const retryAfter = Number(res.headers()["retry-after"] ?? 60);
211    const waitMs = Math.min(Math.max(retryAfter, 1) * 1000, 65_000);
212    await sleep(waitMs);
213    return null;
214  }
215  const body = await expectOk(res, `list runs of ${autoId}`);
216  return (body.data as Run[]) ?? [];
217}
218 
219async function getRun(ctx: APIRequestContext, autoId: string, runId: string): Promise<Json> {
220  const res = await ctx.get(`/api/v1/automations/${autoId}/runs/${runId}`);
221  return expectOk(res, `get run ${runId}`);
222}
223 
224const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
225 
226/** Poll listRuns until pred() matches; returns the matching run (or the last state on timeout). */
227async function waitForRun(
228  ctx: APIRequestContext,
229  autoId: string,
230  pred: (runs: Run[]) => Run | undefined,
231  opts: { timeoutMs?: number; intervalMs?: number; what?: string } = {}
232): Promise<Run> {
233  const { timeoutMs = 30_000, intervalMs = 1_000 } = opts;
234  const t0 = Date.now();
235  for (;;) {
236    const runs = await listRunsTolerant(ctx, autoId);
237    if (runs === null) continue; // 429 — waited out Retry-After, re-poll
238    const m = pred(runs);
239    if (m) return m;
240    if (Date.now() - t0 > timeoutMs) {
241      throw new Error(
242        `${opts.what ?? "waitForRun"}: timeout after ${timeoutMs} ms; runs = ${JSON.stringify(runs.map((r) => ({ id: r.id.slice(-6), status: r.status })))}`
243      );
244    }
245    await sleep(intervalMs);
246  }
247}
248 
249// --- schedule reconciler (worker source, via a child tsx process) ------------
250//
251// The worker registers SCHEDULE repeatables at boot + every 5 minutes. The
252// test invokes the REAL reconciler so a cron `* * * * *` automation is
253// registered immediately; the cron resolution + fire still happen inside the
254// real BullMQ queue (no time manipulation).
255//
256// The reconciler lives in the worker package (TS source, no build for the
257// monorepo test path). Playwright's strip-only TS transform cannot load it
258// into the spec process (parameter properties etc.), so — exactly like the
259// integration spec's SIGKILL/crash-recovery test — we run it in a CHILD tsx
260// process whose script is written next to the spec (so @prisma/client,
261// ioredis and bullmq resolve from the e2e package's node_modules). The child
262// imports the worker source via file:// URL, runs ONE reconcile, prints
263// `{added, removed}` and exits.
264async function reconcileSchedules(): Promise<{ added: number; removed: number }> {
265  const dsn = process.env.DATABASE_URL;
266  const redisUrl = process.env.REDIS_URL ?? "redis://localhost:6379";
267  if (!dsn) throw new Error("reconcileSchedules: DATABASE_URL is not set (the schedule E2E test needs the DB DSN).");
268  // CJS spec (no "type":"module") — use __dirname (matches files.spec.ts).
269  const here = __dirname;
270  const workerSrc = path.resolve(here, "..", "..", "..", "apps", "worker", "src");
271  const childScript = path.join(here, `.reconcile-child-${randomBytes(4).toString("hex")}.mts`);
272  // tsx is a repo-root devDependency (root node_modules), 3 levels up from here.
273  const tsxCli = path.resolve(here, "..", "..", "..", "node_modules", "tsx", "dist", "cli.mjs");
274  writeFileSync(
275    childScript,
276    [
277      "import { PrismaClient } from \"@prisma/client\";",
278      "import { Redis } from \"ioredis\";",
279      "const dir = process.env.WORKER_SRC_DIR!;",
280      "const { reconcileScheduleAutomations } = await import(\"file://\" + dir + \"/processors/automations/schedule.ts\");",
281      "const prisma = new PrismaClient({ datasources: { db: { url: process.env.DATABASE_URL! } } });",
282      "const redis = new Redis(process.env.REDIS_URL!, { maxRetriesPerRequest: null });",
283      "const r = await reconcileScheduleAutomations(redis, prisma);",
284      // Marker-prefixed line: the worker source pulls in the pino-pretty
285      // logger, whose ASYNC flush can interleave indented field lines AFTER
286      // a synchronous console.log, so the last stdout line is unreliable.
287      // The parent scans for the marker prefix instead.
288      "console.log(\"RECONCILE_DONE \" + JSON.stringify(r));",
289      "await redis.quit();",
290      "await prisma.$disconnect();",
291      ""
292    ].join("\n"),
293    "utf8"
294  );
295  try {
296    const out: string[] = [];
297    const err: string[] = [];
298    await new Promise<void>((resolve, reject) => {
299      const child = spawn(process.execPath, [tsxCli, childScript], {
300        cwd: here,
301        stdio: ["ignore", "pipe", "pipe"],
302        env: { ...process.env, DATABASE_URL: dsn, REDIS_URL: redisUrl, WORKER_SRC_DIR: workerSrc }
303      });
304      const t = setTimeout(() => {
305        child.kill("SIGKILL");
306        reject(new Error(`reconcileSchedules: child timed out; stderr=${err.join("")}`));
307      }, 30_000);
308      child.stdout.on("data", (d) => out.push(String(d)));
309      child.stderr.on("data", (d) => err.push(String(d)));
310      child.on("close", (code) => {
311        clearTimeout(t);
312        if (code === 0) resolve();
313        else reject(new Error(`reconcileSchedules: child exited ${code}; stderr=${err.join("")}`));
314      });
315    });
316    const stdout = out.join("");
317    const line = stdout.split("\n").find((l) => l.startsWith("RECONCILE_DONE "));
318    if (!line) {
319      throw new Error(`reconcileSchedules: no RECONCILE_DONE marker in child stdout; stdout=${JSON.stringify(stdout.slice(-500))}; stderr=${err.join("").slice(-500)}`);
320    }
321    return JSON.parse(line.slice("RECONCILE_DONE ".length)) as { added: number; removed: number };
322  } finally {
323    rmSync(childScript, { force: true });
324  }
325}
326 
327/** GET /notifications (per-user) — find one matching body. */
328async function findNotification(ctx: APIRequestContext, bodyIncludes: string): Promise<Json | null> {
329  const res = await ctx.get(`/api/v1/notifications?limit=100`);
330  const body = await expectOk(res, "list notifications");
331  const list = (Array.isArray(body) ? body : (body.data ?? [])) as Json[];
332  return list.find((n) => typeof n.body === "string" && n.body.includes(bodyIncludes)) ?? null;
333}
334 
335// ---------------------------------------------------------------------------
336// 1. record.created → UPDATE_RECORD + CREATE_NOTIFICATION → SUCCEEDED < 10 s
337// ---------------------------------------------------------------------------
338 
339test("record.created triggers UPDATE_RECORD + notification; property changed in < 10 s; run + logs visible in UI", async ({
340  api,
341  authPage,
342  freshWorkspace
343}) => {
344  const ws = await freshWorkspace("Auto E2E A");
345  const { databaseId } = await createDatabasePage(api, ws.id, "Tickets");
346  await addPropertyRaw(api, databaseId, {
347    name: "Stage",
348    type: "SELECT",
349    config: { options: [{ name: "A" }, { name: "B" }] }
350  });
351  const statusPropId = await findPropertyId(api, databaseId, "Stage");
352 
353  const auto = await createAutomation(api, ws.id, {
354    name: "E2E: promote new tickets",
355    databaseId,
356    triggers: [{ type: "RECORD_CREATED", config: { databaseId } }],
357    actions: [
358      { type: "UPDATE_RECORD", config: { recordId: "from_trigger", propertyValues: { Stage: "B" } } },
359      { type: "CREATE_NOTIFICATION", config: { to: "automation_creator", type: "DATABASE_CHANGE", body: "E2E auto: ticket promoted to B" } }
360    ]
361  });
362  await setStatus(api, auto.id, "ACTIVE");
363 
364  // The record that fires the automation (created with A; the automation sets B).
365  const createdMs = Date.now();
366  const rec = await createRecordRaw(api, databaseId, { [statusPropId]: "A" });
367  const recordId = String(rec.id);
368 
369  const run = await waitForRun(
370    api,
371    auto.id,
372    (runs) => runs.find((r) => r.status === "SUCCEEDED" || r.status === "FAILED" || r.status === "DEAD_LETTER"),
373    { timeoutMs: 10_000, what: "terminal run within the 10 s exit criterion" }
374  );
375  expect(run.status, `run ${run.id} terminal status (error=${JSON.stringify(run.error)})`).toBe("SUCCEEDED");
376  const elapsedMs = Date.now() - createdMs;
377  expect(elapsedMs, `record-created → SUCCEEDED must be < 10 s (was ${elapsedMs} ms)`).toBeLessThan(10_000);
378 
379  // The effect: Status flipped to B on the same record.
380  const after = await getRecord(api, recordId);
381  expect(selectValue(after, statusPropId)).toBe("B");
382 
383  // The notification reached the automation's creator (the demo user owns the workspace).
384  expect(await findNotification(api, "E2E auto: ticket promoted to B")).not.toBeNull();
385 
386  // --- UI: list → row → run history → full logs -----------------------------
387  // Navigate by WORKSPACE **id** (not slug): the API's GET /workspaces/:id
388  // resolves by id only (findUnique on id) and 403s on a slug, which would
389  // bounce the [workspace] layout back to /w/default (matching the pre-existing
390  // spec convention — files/databases.spec navigate with ws.id).
391  await authPage.goto(`/w/${ws.id}/settings/automations`);
392  await expect(authPage.getByTestId("automation-list")).toBeVisible({ timeout: 15_000 });
393  await expect(authPage.getByTestId(`automation-row-${auto.id}`)).toBeVisible({ timeout: 15_000 });
394  // Latest-run cell on the row shows "succeeded".
395  await expect(authPage.getByTestId(`automation-row-${auto.id}`).getByText("succeeded")).toBeVisible();
396 
397  await collapseSidebarIfNarrow(authPage);
398  await authPage.getByTestId(`automation-runs-${auto.id}`).click();
399  await expect(authPage.getByTestId(`automation-run-${run.id}`)).toBeVisible({ timeout: 15_000 });
400  await authPage.getByTestId(`automation-run-${run.id}`).click();
401  const logPanel = authPage.getByTestId(`automation-run-log-${run.id}`);
402  await expect(logPanel).toBeVisible({ timeout: 15_000 });
403  // Full run logs: per-action lines + the terminal line.
404  await expect(logPanel).toContainText("action 1/2 UPDATE_RECORD started");
405  await expect(logPanel).toContainText("action 2/2 CREATE_NOTIFICATION started");
406  await expect(logPanel).toContainText("run completed");
407});
408 
409// ---------------------------------------------------------------------------
410// 2. condition gate (option NAME semantics)
411// ---------------------------------------------------------------------------
412 
413test("condition gate: non-matching record skipped, matching record executes", async ({ api, freshWorkspace }) => {
414  const ws = await freshWorkspace("Auto E2E B");
415  const { databaseId } = await createDatabasePage(api, ws.id, "Leads");
416  await addPropertyRaw(api, databaseId, {
417    name: "Stage",
418    type: "SELECT",
419    config: { options: [{ name: "A" }, { name: "B" }] }
420  });
421  const statusPropId = await findPropertyId(api, databaseId, "Stage");
422 
423  const auto = await createAutomation(api, ws.id, {
424    name: "E2E: gate on Stage = B",
425    databaseId,
426    triggers: [{ type: "RECORD_CREATED", config: { databaseId } }],
427    condition: { field: { ref: "property", name: "Stage" }, operator: "eq", value: "B" },
428    actions: [{ type: "UPDATE_RECORD", config: { recordId: "from_trigger", propertyValues: { Stage: "A" } } }]
429  });
430  await setStatus(api, auto.id, "ACTIVE");
431 
432  // Non-matching (A) → a run is still created (the matcher matched the
433  // trigger) but SUCCEEDS with output.skipped = "conditions_not_met".
434  await createRecordRaw(api, databaseId, { [statusPropId]: "A" });
435  const skipped = await waitForRun(api, auto.id, (runs) => runs.find((r) => r.status === "SUCCEEDED"), {
436    timeoutMs: 15_000,
437    what: "skipped run"
438  });
439  const skippedDetail = await getRun(api, auto.id, skipped.id);
440  expect(skippedDetail.output, "output.skipped for the gated run").toEqual({ skipped: "conditions_not_met" });
441 
442  // Matching (B) → the action runs.
443  await createRecordRaw(api, databaseId, { [statusPropId]: "B" });
444  const ran = await waitForRun(api, auto.id, (runs) => runs.find((r) => r.id !== skipped.id && r.status === "SUCCEEDED"), {
445    timeoutMs: 15_000,
446    what: "executed run"
447  });
448  const ranDetail = await getRun(api, auto.id, ran.id);
449  expect((ranDetail.output as Json)?.skipped, "the matching run must NOT be skipped").toBeUndefined();
450  const actions = (ranDetail.output as Json)?.actions as Json[];
451  expect(actions?.length).toBe(1);
452  expect(actions?.[0]?.type).toBe("UPDATE_RECORD");
453});
454 
455// ---------------------------------------------------------------------------
456// 3. schedule (cron) trigger fires
457// ---------------------------------------------------------------------------
458 
459test("schedule trigger: cron automation fires and creates a schedule run", async ({ api, freshWorkspace }) => {
460  test.setTimeout(120_000);
461  const ws = await freshWorkspace("Auto E2E C");
462  const auto = await createAutomation(api, ws.id, {
463    name: "E2E: hourly ping",
464    triggers: [{ type: "SCHEDULE", config: { cron: "* * * * *" } }],
465    actions: [{ type: "CREATE_NOTIFICATION", config: { to: "automation_creator", type: "AUTOMATION", body: "E2E cron tick" } }]
466  });
467  await setStatus(api, auto.id, "ACTIVE");
468 
469  // Register the repeatable via the real worker reconciler (normally boot /
470  // every 5 min — not worth waiting in a test).
471  const { added } = await reconcileSchedules();
472  expect(added, "the reconciler must register the new repeatable").toBeGreaterThanOrEqual(1);
473 
474  const run = await waitForRun(
475    api,
476    auto.id,
477    (runs) => runs.find((r) => r.triggerType === "schedule" && (r.status === "SUCCEEDED" || r.status === "FAILED" || r.status === "DEAD_LETTER")),
478    { timeoutMs: 95_000, intervalMs: 2_000, what: "cron fire (≤ 60 s resolution + execution)" }
479  );
480  expect(run.status).toBe("SUCCEEDED");
481  expect(await findNotification(api, "E2E cron tick")).not.toBeNull();
482 
483  // Cleanup: pause + reconcile again so the repeatable is removed (the
484  // workspace teardown would otherwise leave a harmless orphan that skips
485  // on the missing automation).
486  await setStatus(api, auto.id, "PAUSED");
487  await reconcileSchedules();
488});
489 
490// ---------------------------------------------------------------------------
491// 4. pause stops execution; re-activate resumes
492// ---------------------------------------------------------------------------
493 
494test("PAUSED automation stops matching; re-activating resumes", async ({ api, freshWorkspace }) => {
495  const ws = await freshWorkspace("Auto E2E D");
496  const { databaseId } = await createDatabasePage(api, ws.id, "Items");
497  const auto = await createAutomation(api, ws.id, {
498    name: "E2E: pausable",
499    databaseId,
500    triggers: [{ type: "RECORD_CREATED", config: { databaseId } }],
501    actions: [{ type: "CREATE_NOTIFICATION", config: { to: "automation_creator", type: "AUTOMATION", body: "E2E pause test tick" } }]
502  });
503  await setStatus(api, auto.id, "ACTIVE");
504 
505  // Active: the first record matches.
506  await createRecordRaw(api, databaseId, {});
507  const first = await waitForRun(api, auto.id, (runs) => runs.find((r) => r.status === "SUCCEEDED"), {
508    timeoutMs: 15_000,
509    what: "first (active) run"
510  });
511 
512  // Paused: a second record must NOT create a new run (10 s of consumer time).
513  await setStatus(api, auto.id, "PAUSED");
514  await sleep(1_000); // let the status settle before the event is published
515  await createRecordRaw(api, databaseId, {});
516  await sleep(10_000);
517  const whilePaused = await listRuns(api, auto.id);
518  expect(whilePaused, "no new run while PAUSED").toHaveLength(1);
519  expect(whilePaused[0]?.id).toBe(first.id);
520 
521  // Re-activated: a third record matches again.
522  await setStatus(api, auto.id, "ACTIVE");
523  await createRecordRaw(api, databaseId, {});
524  await waitForRun(api, auto.id, (runs) => runs.find((r) => r.id !== first.id && r.status === "SUCCEEDED"), {
525    timeoutMs: 15_000,
526    what: "run after re-activation"
527  });
528});
529 
530// ---------------------------------------------------------------------------
531// 5. failure → FAILED + UI error + retry re-runs
532// ---------------------------------------------------------------------------
533 
534test("failing action (dead TEST-NET webhook) → FAILED run with error in UI; retry re-runs the run", async ({
535  api,
536  authPage,
537  freshWorkspace
538}) => {
539  test.setTimeout(90_000);
540  const ws = await freshWorkspace("Auto E2E E");
541  const { databaseId } = await createDatabasePage(api, ws.id, "Outbox");
542  const auto = await createAutomation(api, ws.id, {
543    name: "E2E: failing webhook",
544    databaseId,
545    triggers: [{ type: "RECORD_CREATED", config: { databaseId } }],
546    actions: [{ type: "SEND_WEBHOOK", config: { url: "https://192.0.2.1/hook" } }]
547  });
548  await setStatus(api, auto.id, "ACTIVE");
549 
550  // 192.0.2.1 is TEST-NET (public-range, passes the SSRF guard) but
551  // unreachable: undici gives up after its 10 s connect timeout and the
552  // pipeline classifies the transport error as fatal → FAILED, no retries.
553  await createRecordRaw(api, databaseId, {});
554  const run = await waitForRun(
555    api,
556    auto.id,
557    (runs) => runs.find((r) => r.status === "FAILED" || r.status === "SUCCEEDED" || r.status === "DEAD_LETTER"),
558    { timeoutMs: 45_000, what: "terminal (failed) run" }
559  );
560  expect(run.status, `terminal status (error=${JSON.stringify(run.error)})`).toBe("FAILED");
561  expect(run.error, "the run error must be stored").not.toBeNull();
562 
563  // --- UI: the failed run + its error + the Retry button --------------------
564  // Navigate by WORKSPACE **id** (not slug): the API's GET /workspaces/:id
565  // resolves by id only (findUnique on id) and 403s on a slug, which would
566  // bounce the [workspace] layout back to /w/default (matching the pre-existing
567  // spec convention — files/databases.spec navigate with ws.id).
568  await authPage.goto(`/w/${ws.id}/settings/automations`);
569  await expect(authPage.getByTestId(`automation-row-${auto.id}`)).toBeVisible({ timeout: 15_000 });
570  await collapseSidebarIfNarrow(authPage);
571  await authPage.getByTestId(`automation-runs-${auto.id}`).click();
572  await expect(authPage.getByTestId(`automation-run-${run.id}`)).toBeVisible({ timeout: 15_000 });
573  await expect(authPage.getByTestId(`automation-run-${run.id}`).getByText("failed")).toBeVisible();
574  await authPage.getByTestId(`automation-run-${run.id}`).click();
575  const logPanel = authPage.getByTestId(`automation-run-log-${run.id}`);
576  await expect(logPanel).toBeVisible({ timeout: 15_000 });
577  await expect(logPanel).toContainText("failed");
578  const retryButton = authPage.getByTestId(`automation-retry-run-${run.id}`);
579  await expect(retryButton).toBeVisible();
580 
581  // Retry re-queues the SAME run row (retryCount reset) → it runs again.
582  const before = (await getRun(api, auto.id, run.id)) as Json;
583  const firstFinishedAt = before.finishedAt as string;
584  await retryButton.click();
585  // The run goes back to PENDING/RUNNING, then fails again (the target is
586  // still dead) — with a FRESH startedAt (proof it actually re-ran).
587  await waitForRun(
588    api,
589    auto.id,
590    (runs) => runs.find((r) => r.id === run.id && r.status === "FAILED" && (r.finishedAt ?? "") > firstFinishedAt),
591    { timeoutMs: 60_000, what: "re-run after UI retry" }
592  );
593});
594 
595// ---------------------------------------------------------------------------
596// 6. self-trigger loop prevention through the real stream + worker
597// ---------------------------------------------------------------------------
598 
599test("self-trigger: an automation's own record.updated re-match is skipped, no loop", async ({ api, freshWorkspace }) => {
600  test.setTimeout(90_000);
601  const ws = await freshWorkspace("Auto E2E F");
602  const { databaseId } = await createDatabasePage(api, ws.id, "Loop");
603  await addPropertyRaw(api, databaseId, {
604    name: "Stage",
605    type: "SELECT",
606    config: { options: [{ name: "A" }, { name: "B" }] }
607  });
608  const statusPropId = await findPropertyId(api, databaseId, "Stage");
609 
610  const auto = await createAutomation(api, ws.id, {
611    name: "E2E: self-trigger guard",
612    databaseId,
613    // NOTE: a BARE RECORD_UPDATED trigger (no `properties` filter). The
614    // matcher only narrows on `changedProperties` when the trigger carries a
615    // non-empty `properties` list; user-originated record.updated events carry
616    // the property **ids** (query.service.ts `changedFields` → Object.keys of
617    // the id-keyed patch), whereas the trigger filter / condition / executor
618    // contract is **property names** — so a property-scoped trigger would fail
619    // closed on a user edit. Without the filter, both the user edit and the
620    // automation's own echo match, and the loop guard skips the echo
621    // (self_trigger). The id/name mismatch for property-scoped user triggers
622    // is a pre-existing contract gap (recorded in the Phase 11 gap report).
623    triggers: [{ type: "RECORD_UPDATED", config: { databaseId } }],
624    actions: [{ type: "UPDATE_RECORD", config: { recordId: "from_trigger", propertyValues: { Stage: "A" } } }]
625  });
626  await setStatus(api, auto.id, "ACTIVE");
627 
628  // Run 1 (depth 0): a USER edit publishes record.updated — the matcher
629  // enqueues the run. The action flips Status back to A, which emits ANOTHER
630  // record.updated — stamped with THIS automationId at depth 1. The consumer
631  // re-matches the same automation (run 2), and the loop guard must skip it
632  // with "self_trigger". No run 3 may follow (a skipped run executes no
633  // actions → emits nothing).
634  const rec = await createRecordRaw(api, databaseId, { [statusPropId]: "A" });
635  await expectOk(
636    await api.patch(`/api/v1/records/${String(rec.id)}`, { data: { properties: { [statusPropId]: "B" } } }),
637    "user edit (B) that starts the chain"
638  );
639  // Wait until BOTH runs exist AND the newest (the echo) is terminal — a
640  // two-run count alone would let us read the echo's output before it ran.
641  const isTerminal = (r: Run) =>
642    r.status === "SUCCEEDED" || r.status === "FAILED" || r.status === "DEAD_LETTER";
643  await waitForRun(api, auto.id, (runs) => (runs.length >= 2 && isTerminal(runs[0]) ? runs[0] : undefined), {
644    timeoutMs: 30_000,
645    what: "the second run (the re-matched record.updated echo) reaching terminal"
646  });
647  const runsNow = await listRuns(api, auto.id);
648  expect(runsNow.length, "exactly 2 runs: the update ran, its own echo was skipped").toBe(2);
649  // The echo run is the newest one (createdAt desc); it must be the skip.
650  const skipped = await getRun(api, auto.id, runsNow[0].id);
651  expect(skipped.output, "self_trigger skip output").toEqual({ skipped: "self_trigger" });
652 
653  // Let the consumer prove there is no third run.
654  await sleep(8_000);
655  expect((await listRuns(api, auto.id)).length, "no third run (loop prevented)").toBe(2);
656});
657 
658// ---------------------------------------------------------------------------
659// 7. manual run (DRAFT) + full logs in the UI
660// ---------------------------------------------------------------------------
661 
662test("manual run on a DRAFT automation executes; full per-action logs render in the UI", async ({
663  api,
664  authPage,
665  freshWorkspace
666}) => {
667  const ws = await freshWorkspace("Auto E2E G");
668  const { databaseId } = await createDatabasePage(api, ws.id, "Manual");
669  await addPropertyRaw(api, databaseId, {
670    name: "Stage",
671    type: "SELECT",
672    config: { options: [{ name: "A" }, { name: "B" }] }
673  });
674  const statusPropId = await findPropertyId(api, databaseId, "Stage");
675  const seed = await createRecordRaw(api, databaseId, { [statusPropId]: "A" });
676 
677  // DRAFT (created, never activated) — manual runs are allowed on drafts.
678  const auto = await createAutomation(api, ws.id, {
679    name: "E2E: manual draft",
680    databaseId,
681    actions: [{ type: "UPDATE_RECORD", config: { recordId: String(seed.id), propertyValues: { Stage: "B" } } }]
682  });
683  expect(auto.status).toBe("DRAFT");
684 
685  const res = await api.post(`/api/v1/automations/${auto.id}/run`, { data: {} });
686  expect(res.status(), "manual run → 202").toBe(202);
687  const body = await j(res);
688  const runId = String(body.runId);
689  expect(runId).toBeTruthy();
690 
691  const run = await waitForRun(
692    api,
693    auto.id,
694    (runs) => runs.find((r) => r.id === runId && (r.status === "SUCCEEDED" || r.status === "FAILED")),
695    { timeoutMs: 30_000, what: "manual run terminal" }
696  );
697  expect(run.status).toBe("SUCCEEDED");
698  expect(selectValue(await getRecord(api, String(seed.id)), statusPropId)).toBe("B");
699 
700  // UI: the DRAFT row exists, its run history shows the manual run + logs.
701  // Navigate by WORKSPACE **id** (not slug): the API's GET /workspaces/:id
702  // resolves by id only (findUnique on id) and 403s on a slug, which would
703  // bounce the [workspace] layout back to /w/default (matching the pre-existing
704  // spec convention — files/databases.spec navigate with ws.id).
705  await authPage.goto(`/w/${ws.id}/settings/automations`);
706  await expect(authPage.getByTestId(`automation-row-${auto.id}`)).toBeVisible({ timeout: 15_000 });
707  await collapseSidebarIfNarrow(authPage);
708  await authPage.getByTestId(`automation-runs-${auto.id}`).click();
709  await expect(authPage.getByTestId(`automation-run-${runId}`)).toBeVisible({ timeout: 15_000 });
710  await authPage.getByTestId(`automation-run-${runId}`).click();
711  const logPanel = authPage.getByTestId(`automation-run-log-${runId}`);
712  await expect(logPanel).toBeVisible({ timeout: 15_000 });
713  await expect(logPanel).toContainText("action 1/1 UPDATE_RECORD started");
714  await expect(logPanel).toContainText("action 1/1 UPDATE_RECORD done");
715  await expect(logPanel).toContainText("run completed");
716});
717