feat(signals): auto-pause scouts that keep failing, and name the wall

frontend/src/scenes/inbox/components/config/scouts/ScoutBadges.tsx

@@ -2,6 +2,28 @@ import { LemonTag, Tooltip } from '@posthog/lemon-ui'
 
 import type { ScoutOriginEnumApi } from 'products/signals/frontend/generated/api.schemas'
 
+/**
+ * Shown when a scout paused itself after failing repeatedly (`status=paused_by_system`,
+ * `pause_reason=repeated_failures`). Without it the pause is only visible in the API
+ * response, and a scout that has silently stopped looks the same as one a person turned off.
+ */
+export function ScoutPausedBadge(): JSX.Element {
+    return (
+        <Tooltip
+            title={
+                <span>
+                    This scout paused itself because its last few runs all failed. It retries about once a day and
+                    resumes its normal schedule on the first successful run. Turn it on to resume it right away.
+                </span>
+            }
+        >
+            <LemonTag type="warning" size="small">
+                Paused
+            </LemonTag>
+        </Tooltip>
+    )
+}
+
 /** Canonical (PostHog-maintained) vs Custom (team-authored) scout badge. */
 export function ScoutOriginBadge({ origin }: { origin: ScoutOriginEnumApi }): JSX.Element {
     return (

frontend/src/scenes/inbox/components/config/scouts/ScoutConfigControls.test.tsx

@@ -20,6 +20,7 @@ const config: SignalScoutConfigApi = {
     run_cron_schedule: '0 9 * * *',
     output_destinations: {},
     last_run_at: null,
+    consecutive_failure_count: 0,
     created_at: '2026-07-21T12:00:00Z',
 }
 

frontend/src/scenes/inbox/components/config/scouts/ScoutRowCard.tsx

@@ -21,7 +21,7 @@ import {
     ScoutRollup,
 } from '../../../utils/scoutRunsWindow'
 import { agentSetupModalLogic } from '../../shell/agentSetupModalLogic'
-import { ScoutOriginBadge } from './ScoutBadges'
+import { ScoutOriginBadge, ScoutPausedBadge } from './ScoutBadges'
 import { ScoutConfigForm, ScoutEnabledSwitch } from './ScoutConfigControls'
 import { ScoutRunBoxes } from './ScoutRunBoxes'
 
@@ -129,6 +129,9 @@ export function ScoutRowCard({
                                 </Link>
                             </Tooltip>
                             <ScoutOriginBadge origin={config.scout_origin} />
+                            {config.status === 'paused_by_system' && config.pause_reason === 'repeated_failures' ? (
+                                <ScoutPausedBadge />
+                            ) : null}
                         </div>
                     </div>
                     <div className="flex items-center gap-1 whitespace-nowrap text-[11px] text-muted">

frontend/src/scenes/inbox/logics/scoutCreateModalLogic.test.ts

@@ -40,6 +40,7 @@ const CREATED_SCOUT: SignalScoutCreateResponseApi = {
         run_cron_schedule: null,
         output_destinations: {},
         last_run_at: null,
+        consecutive_failure_count: 0,
         created_at: '2026-07-24T00:00:00Z',
     },
 }

frontend/src/scenes/inbox/logics/scoutFleetLogic.test.ts

@@ -45,6 +45,7 @@ const BASE_CONFIG: SignalScoutConfigApi = {
     run_cron_schedule: null,
     output_destinations: {},
     last_run_at: null,
+    consecutive_failure_count: 0,
     created_at: '2026-07-22T00:00:00Z',
 }
 

posthog/models/activity_logging/activity_log.py

@@ -396,11 +396,13 @@ class Meta:
     "Subscription": [
         "next_delivery_date",
     ],
-    # `last_run_at` is written by the scout coordinator on every tick (~every 15 min per scout).
-    # When that is the only change, suppress the activity signal entirely so run bookkeeping
-    # never spams the audit log.
+    # `last_run_at` is written by the scout coordinator on every tick (~every 15 min per scout),
+    # and the failure streak by the runner on every run outcome. When those are the only
+    # change, suppress the activity signal entirely so run bookkeeping never spams the audit
+    # log. A breaker trip is NOT suppressed: it moves `status`, which logs like any pause.
     "SignalScoutConfig": [
         "last_run_at",
+        "consecutive_failure_count",
     ],
 }
 
@@ -760,6 +762,7 @@ class Meta:
         # Run bookkeeping, not user intent — keep it out of change detection even when it
         # rides along with a real change (belt-and-suspenders with signal_exclusions above).
         "last_run_at",
+        "consecutive_failure_count",
         # Companion bookkeeping that rides along with every logged `status` change; the
         # activity log entry itself already carries who and when.
         "status_changed_at",

products/signals/backend/admin.py

@@ -116,6 +116,7 @@ class SignalScoutConfigAdmin(admin.ModelAdmin):
         "pause_reason",
         "status_changed_at",
         "status_changed_by",
+        "consecutive_failure_count",
     )
     list_select_related = ("team", "team__organization")
     show_full_result_count = False

products/signals/backend/migrations/0079_signalscoutconfig_consecutive_failure_count.py

@@ -0,0 +1,13 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [("signals", "0078_backfill_scout_status")]
+
+    operations = [
+        migrations.AddField(
+            model_name="signalscoutconfig",
+            name="consecutive_failure_count",
+            field=models.PositiveIntegerField(db_default=0, default=0),
+        ),
+    ]

products/signals/backend/migrations/0080_reconcile_scout_status_enabled.py

@@ -0,0 +1,33 @@
+from django.db import migrations
+
+
+def reconcile_enabled_wins(apps, schema_editor):
+    # Re-runs 0078's enabled-wins reconciliation over rows old instances drifted during the
+    # 0077 rollout window (enabled-only writes from pods that predated the dual-write land
+    # after the backfill). `enabled` carries the operator's intent for any mismatch: no
+    # system writer existed before the constraints in 0081, so a disagreeing row can only be
+    # a human toggle that missed the `status` side. Small table, plain UPDATEs.
+    SignalScoutConfig = apps.get_model("signals", "SignalScoutConfig")
+    manager = SignalScoutConfig._default_manager
+    manager.filter(enabled=False, status__in=["active", "pending_pause"]).update(
+        status="paused_by_user", pause_reason=None
+    )
+    manager.filter(enabled=True, status__in=["paused_by_system", "paused_by_user"]).update(
+        status="active", pause_reason=None
+    )
+    # Reason coherence for 0081's second constraint: a reason on a runnable/user-paused row is
+    # stray; a reasonless warning is meaningless (runnable anyway); a reasonless system pause
+    # can only predate system writers, so it reads as a human's off-switch.
+    manager.filter(status__in=["active", "paused_by_user"], pause_reason__isnull=False).update(pause_reason=None)
+    manager.filter(status="pending_pause", pause_reason__isnull=True).update(status="active")
+    manager.filter(status="paused_by_system", pause_reason__isnull=True).update(status="paused_by_user")
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("signals", "0079_signalscoutconfig_consecutive_failure_count"),
+    ]
+
+    operations = [
+        migrations.RunPython(reconcile_enabled_wins, reverse_code=migrations.RunPython.noop),
+    ]

products/signals/backend/migrations/0081_scout_status_constraints.py

@@ -0,0 +1,28 @@
+from django.db import migrations, models
+
+from posthog.migration_helpers import AddConstraintNotValid
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("signals", "0080_reconcile_scout_status_enabled"),
+    ]
+
+    operations = [
+        AddConstraintNotValid(
+            model_name="signalscoutconfig",
+            constraint=models.CheckConstraint(
+                name="scout_config_enabled_matches_status",
+                condition=models.Q(enabled=True, status__in=["active", "pending_pause"])
+                | models.Q(enabled=False, status__in=["paused_by_system", "paused_by_user"]),
+            ),
+        ),
+        AddConstraintNotValid(
+            model_name="signalscoutconfig",
+            constraint=models.CheckConstraint(
+                name="scout_config_pause_reason_matches_status",
+                condition=models.Q(status__in=["pending_pause", "paused_by_system"], pause_reason__isnull=False)
+                | models.Q(status__in=["active", "paused_by_user"], pause_reason__isnull=True),
+            ),
+        ),
+    ]

products/signals/backend/migrations/0082_validate_scout_status_constraints.py

@@ -0,0 +1,14 @@
+from django.db import migrations
+
+from posthog.migration_helpers import ValidateConstraint
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("signals", "0081_scout_status_constraints"),
+    ]
+
+    operations = [
+        ValidateConstraint(model_name="signalscoutconfig", name="scout_config_enabled_matches_status"),
+        ValidateConstraint(model_name="signalscoutconfig", name="scout_config_pause_reason_matches_status"),
+    ]

products/signals/backend/migrations/max_migration.txt

@@ -1 +1 @@
-0078_backfill_scout_status
+0082_validate_scout_status_constraints

products/signals/backend/models.py

@@ -1237,6 +1237,14 @@ class PauseReason(models.TextChoices):
     # Stamped by the coordinator after each dispatch; drives the due-check. Written every
     # run, so it is excluded from activity logging (see field_exclusions below).
     last_run_at = models.DateTimeField(null=True, blank=True)
+    # Failure-streak circuit breaker over this lane's run outcomes, maintained by the runner:
+    # bumped on a failed run, zeroed on a successful one. At `FAILURE_STREAK_PAUSE_THRESHOLD`
+    # the runner pauses the lane (`transition_status_by_system`, `repeated_failures`) and the
+    # coordinator holds it to one probe per `AUTO_PAUSE_PROBE_INTERVAL_S`. Without it a lane
+    # that can never succeed re-dispatches forever, taking a full-length sandbox lease per
+    # interval to produce nothing. Written on every run, so it is excluded from activity
+    # logging like `last_run_at`; the pause itself logs through `status` like any other.
+    consecutive_failure_count = models.PositiveIntegerField(default=0, db_default=0)
     created_at = models.DateTimeField(auto_now_add=True)
     updated_at = models.DateTimeField(auto_now=True)
     created_by = models.ForeignKey(
@@ -1261,13 +1269,20 @@ class Meta:
         default_manager_name = "all_teams"
         constraints = [
             models.UniqueConstraint(fields=["team", "skill_name"], name="unique_scout_config_per_team_skill"),
-            # A CHECK constraint tying `enabled` to `status` is deliberately deferred to a
-            # follow-up migration: enforcing it in the same deploy that introduces the
-            # dual-write breaks rolling deploys, because not-yet-replaced instances still
-            # write `enabled` alone and a NOT VALID constraint already checks new writes.
-            # That follow-up must first re-run the enabled-wins reconciliation over drifted
-            # rows (enabled-only writes from old instances during the rollout window land
-            # after the 0075 backfill), then add and validate the constraint.
+            # Backstop for the dual-write in `save`: added NOT VALID + validated (0080–0082)
+            # only after the 0077 deploy fully rolled, because enforcing it in the same deploy
+            # that introduced the dual-write would break not-yet-replaced instances that still
+            # write `enabled` alone.
+            models.CheckConstraint(
+                name="scout_config_enabled_matches_status",
+                condition=models.Q(enabled=True, status__in=["active", "pending_pause"])
+                | models.Q(enabled=False, status__in=["paused_by_system", "paused_by_user"]),
+            ),
+            models.CheckConstraint(
+                name="scout_config_pause_reason_matches_status",
+                condition=models.Q(status__in=["pending_pause", "paused_by_system"], pause_reason__isnull=False)
+                | models.Q(status__in=["active", "paused_by_user"], pause_reason__isnull=True),
+            ),
         ]
 
     def save(self, *args: Any, **kwargs: Any) -> None:
@@ -1329,7 +1344,18 @@ def transition_status_by_system(
         if new_status == self.Status.PAUSED_BY_USER:
             raise ValueError("Only a user write may set paused_by_user.")
         with transaction.atomic():
-            locked = type(self).all_teams.select_for_update().get(pk=self.pk)
+            # One ordered query locks the whole team's rows, not just ours: the cap check below
+            # counts sibling rows, so two concurrent resumes locking only their own rows would
+            # each read the other as still paused and both slip under the cap. A single ordered
+            # lock set also keeps concurrent transitions on one team deadlock-free. Team config
+            # counts are small (capped) and transitions rare, so the wider lock is cheap.
+            team_rows = {
+                row.pk: row
+                for row in type(self).all_teams.select_for_update().filter(team_id=self.team_id).order_by("pk")
+            }
+            locked = team_rows.get(self.pk)
+            if locked is None:
+                return False
             if locked.status == self.Status.PAUSED_BY_USER:
                 return False
             if locked.status != self.Status.ACTIVE and locked.pause_reason != pause_reason:
@@ -1351,7 +1377,7 @@ def transition_status_by_system(
             )
 
             if new_status in self.RUNNABLE_STATUSES and locked.status not in self.RUNNABLE_STATUSES:
-                peers = type(self).all_teams.filter(team_id=locked.team_id, enabled=True).exclude(pk=locked.pk).count()
+                peers = sum(1 for row in team_rows.values() if row.enabled and row.pk != locked.pk)
                 if peers >= MAX_ENABLED_SCOUTS_PER_TEAM:
                     return False
             recorded_reason = None if new_status == self.Status.ACTIVE else pause_reason
@@ -1362,17 +1388,28 @@ def transition_status_by_system(
             locked.status_changed_at = timezone.now()
             locked.status_changed_by = None
             locked.enabled = new_status in self.RUNNABLE_STATUSES
-            locked.save(
-                update_fields=[
-                    "status",
-                    "pause_reason",
-                    "status_changed_at",
-                    "status_changed_by",
-                    "enabled",
-                    "updated_at",
-                ]
-            )
-        for field in ("status", "pause_reason", "status_changed_at", "status_changed_by", "enabled"):
+            update_fields = [
+                "status",
+                "pause_reason",
+                "status_changed_at",
+                "status_changed_by",
+                "enabled",
+                "updated_at",
+            ]
+            if new_status == self.Status.ACTIVE:
+                # A resume always starts with a clean failure streak — otherwise the very next
+                # failed run would re-trip the breaker off stale evidence.
+                locked.consecutive_failure_count = 0
+                update_fields.append("consecutive_failure_count")
+            locked.save(update_fields=update_fields)
+        for field in (
+            "status",
+            "pause_reason",
+            "status_changed_at",
+            "status_changed_by",
+            "enabled",
+            "consecutive_failure_count",
+        ):
             setattr(self, field, getattr(locked, field))
         return True