diff --git a/apps/server/src/engine/service.ts b/apps/server/src/engine/service.ts index 8b49d254f..fece1f507 100644 --- a/apps/server/src/engine/service.ts +++ b/apps/server/src/engine/service.ts @@ -754,6 +754,8 @@ export class AgentService { if (error instanceof LostLeaseError || context.signal.aborted) throw error; await context.guard(); const failures = Number(task.state.failures ?? 0) + 1; + // Each streak of failures (after a success or a resume) gets its own alerts. + const failureStreak = Number(task.state.failureStreak ?? 0) + (failures === 1 ? 1 : 0); const detail = error instanceof Error ? error.message : "Page check failed"; const nextCheckAt = new Date( Date.now() + Math.min(60, 2 ** failures) * 60000, @@ -778,10 +780,11 @@ export class AgentService { ...task.state, failures, resumingMonitor: false, + failureStreak, notice: { title: "Watch needs attention", body: detail, - key: `watch-error:${task.id}:${failures >= 5 ? "paused" : "retry"}`, + key: `watch-error:${task.id}:${failureStreak}:${failures >= 5 ? "paused" : "retry"}`, }, }, }; diff --git a/tests/monitor-recovery.test.ts b/tests/monitor-recovery.test.ts index 023cc6ffe..8d40eb234 100644 --- a/tests/monitor-recovery.test.ts +++ b/tests/monitor-recovery.test.ts @@ -191,7 +191,7 @@ test("the paused alert is delivered when the final failure outcome is lost", asy assert.equal(task?.status, "paused"); assert.equal((await db.get(owner, "monitors", monitor.id))?.status, "paused"); const pausedId = createHash("sha256") - .update(`watch-error:${monitor.taskId}:paused`) + .update(`watch-error:${monitor.taskId}:1:paused`) .digest("hex"); const paused = (await read("/notifications")).filter( (item) => item.id === pausedId, diff --git a/tests/workflows.test.ts b/tests/workflows.test.ts index 404b11abb..9a3d254fa 100644 --- a/tests/workflows.test.ts +++ b/tests/workflows.test.ts @@ -158,6 +158,40 @@ test("failed page checks back off, expose the error, and pause after repeated fa assert.equal((await db.get(owner, "monitors", monitor.id))?.status, "paused"); }); +test("each failure streak of a watch raises its own alerts after it is resumed", async () => { + const monitor = await server.agent.createMonitor(owner, { + title: "Resumed availability", + url: "https://example.com", + }); + const failUntilPaused = async () => { + for (let i = 1; i <= 5; i++) { + await server.agent.worker.tick(); + const task = await server.agent.getTask(owner, monitor.taskId); + if (i < 5) + await db.compareAndSwap( + owner, + "tasks", + task.id, + { status: "scheduled" }, + { nextRunAt: "2020-01-01T00:00:00Z" }, + ); + } + assert.equal((await server.agent.getTask(owner, monitor.taskId)).status, "paused"); + }; + const alerts = async () => + (await db.list(owner, "notifications")).filter( + (n) => n.taskId === monitor.taskId && n.title === "Watch needs attention", + ); + await failUntilPaused(); + assert.equal((await alerts()).length, 2, "one retry alert and one paused alert"); + await server.agent.controlMonitor(owner, monitor.id, "resume"); + await failUntilPaused(); + assert.equal((await alerts()).length, 4, "the second streak alerts again"); + // Replaying the same outcome must not duplicate the alert. + await server.agent.worker.tick(); + assert.equal((await alerts()).length, 4); +}); + test("dismissal racing acceptance never creates work for a dismissed idea", async () => { for (let i = 0; i < 4; i++) { const idea: Idea = {