Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions products/tasks/backend/logic/services/loop_runs.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,13 @@
LOOP_AUTO_PAUSE_THRESHOLD = 5
TRIGGER_CONTEXT_MAX_BYTES = 64 * 1024

# Stored on Loop.disabled_reason when the kill-switch pauses a loop, so clients can render the
# cause (and a billing CTA for usage_limited) instead of a bare "Paused". Lifecycle pause codes
# (owner deactivated/removed, GitHub disconnected) live in loop_lifecycle.py; re-enabling clears
# the field in facade/loops.py::update_loop either way.
DISABLED_REASON_USAGE_LIMITED = "usage_limited"
DISABLED_REASON_REPEATED_FAILURES = "repeated_failures"

_NON_TERMINAL_TASK_RUN_STATUSES = (TaskRun.Status.NOT_STARTED, TaskRun.Status.QUEUED, TaskRun.Status.IN_PROGRESS)
_TERMINAL_TASK_RUN_STATUSES = (TaskRun.Status.COMPLETED, TaskRun.Status.FAILED, TaskRun.Status.CANCELLED)

Expand Down Expand Up @@ -279,7 +286,9 @@ def fire_loop(
# advisory lock across it would stall every other fire for the team.
gate_owner_id = loop.created_by_id
if _usage_gate_blocked(loop):
_increment_consecutive_failures_and_maybe_pause(loop, error="cloud usage limit exceeded")
_increment_consecutive_failures_and_maybe_pause(
loop, error="cloud usage limit exceeded", disabled_reason=DISABLED_REASON_USAGE_LIMITED
)
observe_loop_fire(reason="gate_blocked")
dispatch_loop_event(loop, "needs_attention", {"reason": "gate_blocked"})
return LoopFireResult(created=False, reason="gate_blocked", task_id=None, task_run_id=None)
Expand Down Expand Up @@ -614,7 +623,7 @@ def _execute_task_processing_workflow_for_loop(
)


def _increment_consecutive_failures_and_maybe_pause(loop: Loop, *, error: str) -> None:
def _increment_consecutive_failures_and_maybe_pause(loop: Loop, *, error: str, disabled_reason: str) -> None:
should_pause = False
with transaction.atomic():
locked_loop = Loop.objects.for_team(loop.team_id, canonical=True).select_for_update().get(id=loop.id)
Expand All @@ -623,7 +632,8 @@ def _increment_consecutive_failures_and_maybe_pause(loop: Loop, *, error: str) -
update_fields = ["consecutive_failures", "last_error", "updated_at"]
if locked_loop.consecutive_failures >= LOOP_AUTO_PAUSE_THRESHOLD and locked_loop.enabled:
locked_loop.enabled = False
update_fields.append("enabled")
locked_loop.disabled_reason = disabled_reason
update_fields.extend(["enabled", "disabled_reason"])
should_pause = True
locked_loop.save(update_fields=update_fields)

Expand Down Expand Up @@ -672,7 +682,8 @@ def handle_loop_run_terminal(task_run: TaskRun) -> None:
update_fields = ["last_run_at", "last_run_status", "last_error", "consecutive_failures", "updated_at"]
if not is_success and loop.consecutive_failures >= LOOP_AUTO_PAUSE_THRESHOLD and loop.enabled:
loop.enabled = False
update_fields.append("enabled")
loop.disabled_reason = DISABLED_REASON_REPEATED_FAILURES
update_fields.extend(["enabled", "disabled_reason"])
should_pause = True
loop.save(update_fields=update_fields)

Expand Down
6 changes: 6 additions & 0 deletions products/tasks/backend/tests/test_loop_runs.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
from posthog.models.user import User

from products.tasks.backend.logic.services.loop_runs import (
DISABLED_REASON_REPEATED_FAILURES,
DISABLED_REASON_USAGE_LIMITED,
LOOP_AUTO_PAUSE_THRESHOLD,
LOOP_RATE_CAP_PER_DAY,
LOOP_TEAM_RATE_CAP_PER_DAY,
Expand Down Expand Up @@ -217,6 +219,7 @@ def test_usage_gate_blocked_records_failure_and_flags_attention_without_creating
loop.refresh_from_db()
self.assertEqual(loop.consecutive_failures, 1)
self.assertEqual(loop.last_error, "cloud usage limit exceeded")
self.assertIsNone(loop.disabled_reason)
self.assertEqual(Task.objects.filter(team=self.team).count(), 0)
mock_dispatch.assert_called_once_with(loop, "needs_attention", {"reason": "gate_blocked"})

Expand All @@ -233,6 +236,7 @@ def test_usage_gate_blocked_pauses_loop_after_reaching_failure_threshold(

loop.refresh_from_db()
self.assertFalse(loop.enabled)
self.assertEqual(loop.disabled_reason, DISABLED_REASON_USAGE_LIMITED)
self.assertEqual(loop.consecutive_failures, LOOP_AUTO_PAUSE_THRESHOLD)
mock_pause.assert_called_once()
mock_dispatch.assert_any_call(
Expand Down Expand Up @@ -767,6 +771,7 @@ def test_failed_run_increments_consecutive_failures_and_dispatches_run_failed(se
self.assertEqual(loop.consecutive_failures, 1)
self.assertEqual(loop.last_error, "boom")
self.assertTrue(loop.enabled)
self.assertIsNone(loop.disabled_reason)
mock_dispatch.assert_called_once_with(
loop,
"run_failed",
Expand All @@ -783,6 +788,7 @@ def test_failed_run_reaching_threshold_auto_pauses_the_loop(self, mock_pause, mo

loop.refresh_from_db()
self.assertFalse(loop.enabled)
self.assertEqual(loop.disabled_reason, DISABLED_REASON_REPEATED_FAILURES)
self.assertEqual(loop.consecutive_failures, LOOP_AUTO_PAUSE_THRESHOLD)
mock_pause.assert_called_once()
mock_dispatch.assert_any_call(
Expand Down
Loading