From 2ab0fc07987fc6ac4a597b99a571f0fa9351c52c Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:44:42 +0000 Subject: [PATCH 1/3] fix(lab): close the first runtime manager before the start retry raes 7.0.0 lets one RuntimeManager own a target's components until it is closed. The automatic backend retry kept the first attempt's manager alive and built a second one for the same target, so the retry raised "backend already has a runtime owner" and `lilrae lab start --json` printed nothing. The retry now closes the first manager before the admitted hook runs. If RAES refuses to release it, the retry is skipped and the first attempt's error is kept with the reason. If the retry also fails, its error ends with "Initial attempt:" and the first attempt's diagnostics. The retryable set is unchanged. --- docs/sdl/runtime-architecture.md | 13 +- src/aptl/backends/_raes_start_retry.py | 65 +++++++ src/aptl/backends/raes.py | 18 +- tests/test_raes_backend.py | 3 + tests/test_raes_start_retry.py | 237 +++++++++++++++++++++++++ 5 files changed, 320 insertions(+), 16 deletions(-) create mode 100644 src/aptl/backends/_raes_start_retry.py create mode 100644 tests/test_raes_start_retry.py diff --git a/docs/sdl/runtime-architecture.md b/docs/sdl/runtime-architecture.md index 163e88c14..a7c4729b6 100644 --- a/docs/sdl/runtime-architecture.md +++ b/docs/sdl/runtime-architecture.md @@ -18,10 +18,15 @@ The supported runtime handoff is now: 5. Start the lab through `aptl.backends.raes.start_raes_scenario()`. If the deployment backend returns its existing retryable start diagnostic and -the admitted plan selected the `soc` profile, the SOC recovery hook runs and -APTL applies that same admitted `ExecutionPlan` one more time. It does not parse -the SDL or invoke the planner again. The retry decision therefore follows the -realized profile set, not the global container configuration. +the scenario's startup adapter admits the `BEFORE_BACKEND_RETRY` hook, lab start +waits 60 seconds, runs that hook, and APTL applies that same admitted +`ExecutionPlan` one more time. It does not parse the SDL or invoke the planner +again. The retry decision therefore follows the admitted adapter, not the global +container configuration. RAES lets one runtime manager own the target at a time, +so APTL closes the first attempt's manager before the hook runs. If that manager +can't be closed, APTL skips the retry and reports the first attempt's error. If +the retry also fails, its error ends with `Initial attempt:` and the first +attempt's diagnostics. Runtime bindings are an in-process admission input, not APTL configuration or secrets. APTL does not source them from the environment, forward them to diff --git a/src/aptl/backends/_raes_start_retry.py b/src/aptl/backends/_raes_start_retry.py new file mode 100644 index 000000000..8f70342f0 --- /dev/null +++ b/src/aptl/backends/_raes_start_retry.py @@ -0,0 +1,65 @@ +"""Lab start's single backend retry under RAES single runtime ownership.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import replace + +from aptl.backends.raes_start_model import AcesStartOutcome +from aptl.utils.logging import get_logger +from aptl.utils.redaction import redact + +log = get_logger("raes-backend") + +_RETRY_SKIPPED = ( + "Automatic retry skipped: the first runtime owner could not be released" +) + + +def apply_with_single_retry( + run_attempt: Callable[[], AcesStartOutcome], + before_retry: Callable[[], None] | None, +) -> AcesStartOutcome: + """Run one attempt, and one retry when it failed retryably and a hook exists. + + raes 7 lets one ``RuntimeManager`` own a target's components until it is + closed, and each attempt builds its own manager, so the first attempt's + manager is closed before the hook runs. If it cannot be released, the retry + is skipped and the first diagnostic is kept. A failed retry also carries the + first attempt's diagnostic. + """ + + first = run_attempt() + if not first.retryable or before_retry is None: + return first + release_failure = _release_runtime_owner(first) + if release_failure is not None: + log.warning("Automatic backend retry skipped: %s", release_failure) + return _with_error( + first, f"{first.lab_result.error}\n{_RETRY_SKIPPED} ({release_failure})." + ) + before_retry() + retry = run_attempt() + if retry.lab_result.success: + return retry + return _with_error( + retry, f"{retry.lab_result.error}\nInitial attempt: {first.lab_result.error}" + ) + + +def _release_runtime_owner(outcome: AcesStartOutcome) -> str | None: + """Close the attempt's manager; return why it stayed open, if it did.""" + + manager = outcome.runtime_manager + if manager is None: + return None + try: + manager.close() + except (RuntimeError, TimeoutError) as exc: + # RAES refuses while callbacks run or its clock driver does not stop. + return redact(f"{type(exc).__name__}: {exc}") + return None + + +def _with_error(outcome: AcesStartOutcome, error: str) -> AcesStartOutcome: + return replace(outcome, lab_result=replace(outcome.lab_result, error=error)) diff --git a/src/aptl/backends/raes.py b/src/aptl/backends/raes.py index 0a8e8bbcd..ac7f1a697 100644 --- a/src/aptl/backends/raes.py +++ b/src/aptl/backends/raes.py @@ -40,6 +40,7 @@ INSTANTIATION_FAILURE_MESSAGE as INSTANTIATION_FAILURE_MESSAGE, start_failure_outcome as _start_failure_outcome, ) +from aptl.backends._raes_start_retry import apply_with_single_retry from aptl.backends.raes_diagnostics import render_raes_diagnostics from aptl.backends.raes_execution_helpers import ( evaluation_results as collect_evaluation_results, @@ -357,23 +358,16 @@ def _apply_with_backend_retry( run_store = run_target.run_store if run_target is not None else None run_id = run_target.run_id if run_target is not None else None - outcome = _run_execution_plan( - target, - execution_plan, - scenario_path, - run_store=run_store, - run_id=run_id, - ) - if outcome.retryable and before_backend_retry is not None: - before_backend_retry() - return _run_execution_plan( + return apply_with_single_retry( + lambda: _run_execution_plan( target, execution_plan, scenario_path, run_store=run_store, run_id=run_id, - ) - return outcome + ), + before_backend_retry, + ) def _run_execution_plan( diff --git a/tests/test_raes_backend.py b/tests/test_raes_backend.py index c565f63b3..c29fb50e4 100644 --- a/tests/test_raes_backend.py +++ b/tests/test_raes_backend.py @@ -576,6 +576,9 @@ def apply(self, execution_plan): self.target, execution_plan, self._initial_snapshot ) + def close(self): + """Release nothing: unlike the real manager, the double claims no component.""" + def test_create_aptl_manifest_is_canonical_backend_manifest_v2(): from raes_backend_protocols.capabilities import BackendManifest diff --git a/tests/test_raes_start_retry.py b/tests/test_raes_start_retry.py new file mode 100644 index 000000000..85c8c66de --- /dev/null +++ b/tests/test_raes_start_retry.py @@ -0,0 +1,237 @@ +"""Lab start's single backend retry against the real RAES runtime owner. + +raes 7 lets one ``RuntimeManager`` own a target's components until it is +closed, so the retry must release the first attempt's manager before it builds +the second. The deployment backend and the retry hook are modelled. The runtime +manager, its RAES claim, and ``close()`` are real; the skip test subclasses the +real manager only to leave a clock driver that does not stop. The CLI test +replaces only the lab steps around the handoff. +""" + +import json +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +from typer.testing import CliRunner + +from aptl.backends import raes +from aptl.backends.raes_planning_compat import AptlRuntimeManager +from aptl.core.config import AptlConfig +from aptl.core.lab_types import LabResult +from tests.test_raes_backend import ( + _bundle, + _execution_plan_with_realization_requirements, + _RealizedBackend, + _write_compose, +) + +_START_FAILED = "aptl.provisioner.backend-start-failed" +_FIRST_ERROR = "Node wazuh-dashboard exited after compose start" +_RETRY_ERROR = "Node shuffle-backend exited after compose start" +_CONFIG = AptlConfig(lab={"name": "test"}, containers={"victim": True}) + + +def _realized_backend(*realize_results: LabResult) -> _RealizedBackend: + """Model a daemon that runs ``vm`` and answers each apply in turn.""" + + backend = _RealizedBackend(containers=("vm",), platform="linux") + if realize_results: + backend.realize.side_effect = list(realize_results) + return backend + + +def _target(tmp_path: Path, backend: _RealizedBackend): + _write_compose(tmp_path, {"vm": ["victim"]}) + return raes.create_aptl_runtime_target( + project_dir=tmp_path, + bundle=_bundle(tmp_path), + config=_CONFIG, + backend=backend, + ) + + +def _start(tmp_path: Path, target, before_retry): + return raes._apply_with_backend_retry( + target, + _execution_plan_with_realization_requirements(), + tmp_path / "scenarios" / "demo.sdl.yaml", + None, + before_retry, + ) + + +def _failed(error: str) -> LabResult: + return LabResult(success=False, error=error) + + +def _started() -> LabResult: + return LabResult(success=True, message="ok") + + +def _assert_still_owned(target) -> None: + """RAES refuses a second manager while an open one owns the target.""" + + with pytest.raises(RuntimeError, match="already has a runtime owner"): + AptlRuntimeManager(target) + + +def test_retry_releases_the_first_runtime_owner_and_starts(tmp_path): + """The live crash: raes 7 refused the retry's second runtime manager.""" + + backend = _realized_backend(_failed(_FIRST_ERROR), _started()) + applies_before_hook = [] + before_retry = MagicMock( + side_effect=lambda: applies_before_hook.append(backend.realize.call_count) + ) + + target = _target(tmp_path, backend) + + result = _start(tmp_path, target, before_retry) + + assert result.lab_result.success is True, result.lab_result.error + assert backend.realize.call_count == 2 + assert applies_before_hook == [1] + assert isinstance(result.runtime_manager, AptlRuntimeManager) + _assert_still_owned(target) + + +def test_failed_retry_reports_both_attempts(tmp_path): + backend = _realized_backend(_failed(_FIRST_ERROR), _failed(_RETRY_ERROR)) + before_retry = MagicMock() + + result = _start(tmp_path, _target(tmp_path, backend), before_retry) + + error = result.lab_result.error + assert result.lab_result.success is False + assert backend.realize.call_count == 2 + before_retry.assert_called_once_with() + assert error.count(_START_FAILED) == 2 + retry_part, initial_part = error.split("\nInitial attempt: ") + assert _RETRY_ERROR in retry_part and _FIRST_ERROR not in retry_part + assert _FIRST_ERROR in initial_part and _START_FAILED in initial_part + + +class _StuckClockDriver: + """A participant clock driver that does not stop when asked.""" + + active = True + failure = None + + def stop(self) -> bool: + return False + + +class _ManagerWithStuckClock(AptlRuntimeManager): + """The real manager, left with a clock driver that outlives its apply.""" + + def apply(self, execution_plan): + result = super().apply(execution_plan) + self._participant_clock_driver = _StuckClockDriver() + return result + + +def test_unreleasable_first_owner_skips_the_retry(mocker, tmp_path): + mocker.patch("aptl.backends.raes.RuntimeManager", _ManagerWithStuckClock) + backend = _realized_backend(_failed(_FIRST_ERROR), _started()) + before_retry = MagicMock() + + result = _start(tmp_path, _target(tmp_path, backend), before_retry) + + error = result.lab_result.error + assert result.lab_result.success is False + assert backend.realize.call_count == 1 + before_retry.assert_not_called() + assert _START_FAILED in error and _FIRST_ERROR in error + assert "Automatic retry skipped" in error + assert "runtime manager clock driver remains active" in error + assert "Initial attempt" not in error + + +@pytest.mark.parametrize( + ("backend_factory", "hook", "started"), + [ + pytest.param( + lambda: _realized_backend(_started()), + MagicMock(), + True, + id="first-attempt-starts", + ), + pytest.param( + lambda: _RealizedBackend(containers=()), + MagicMock(), + False, + id="non-retryable-first-failure", + ), + pytest.param( + lambda: _realized_backend(_failed(_FIRST_ERROR)), + None, + False, + id="no-admitted-retry-hook", + ), + ], +) +def test_first_attempt_without_retry_keeps_its_owner( + tmp_path, backend_factory, hook, started +): + backend = backend_factory() + target = _target(tmp_path, backend) + + result = _start(tmp_path, target, hook) + + error = result.lab_result.error or "" + assert result.lab_result.success is started, error + assert backend.realize.call_count == 1 + assert "Initial attempt" not in error + assert "Automatic retry skipped" not in error + assert isinstance(result.runtime_manager, AptlRuntimeManager) + _assert_still_owned(target) + if hook is not None: + hook.assert_not_called() + + +def test_lab_start_json_reports_both_attempts_after_a_failed_retry(mocker, tmp_path): + """``lilrae lab start --json`` printed nothing when the retry crashed. + + Only the lab steps around the RAES handoff and ``time.sleep`` are replaced; + the handoff, lab start's retry callback and the runtime manager are real. + """ + from aptl.backends.raes_start_model import AdmittedScenarioStart + from aptl.backends.scenario_startup import ScenarioStartupPlan, StartupHook + from aptl.cli.main import app + from aptl.core.lab import _LabStartContext, _step_start_containers + + backend = _realized_backend(_failed(_FIRST_ERROR), _failed(_RETRY_ERROR)) + bundle = _bundle(tmp_path) + ctx = _LabStartContext(project_dir=tmp_path, skip_seed=False) + ctx.config = _CONFIG + ctx.backend = backend + ctx.admitted_start = AdmittedScenarioStart( + bundle=bundle, + target=_target(tmp_path, backend), + execution_plan=_execution_plan_with_realization_requirements(), + realization=None, + ) + ctx.scenario_startup = ScenarioStartupPlan( + seed_script="scripts/seed.sh", + required_profiles=(), + activation_profiles=(), + startup_hooks=frozenset({StartupHook.BEFORE_BACKEND_RETRY}), + ) + sleep = mocker.patch("time.sleep") + mocker.patch( + "aptl.cli.lab.orchestrate_lab_start", + side_effect=lambda *_args, **_kwargs: _step_start_containers(ctx), + ) + + result = CliRunner().invoke(app, ["lab", "start", "--json"]) + + payload = json.loads(result.stdout) + error = payload["error"] + assert result.exit_code == 1 + assert payload["ok"] is False + assert error.count(_START_FAILED) == 2 + assert _RETRY_ERROR in error and _FIRST_ERROR in error + assert f"Initial attempt: RAES runtime handoff failed: {_START_FAILED}" in error + assert backend.realize.call_count == 2 + sleep.assert_called_once_with(60) From 5feac72ca675af52ec3f29a164f2e909cb2acf9c Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Sun, 25 Oct 2026 09:55:13 +0000 Subject: [PATCH 2/3] test: list the start-retry module and its tests in the tracked-file ledger --- docs/reviews/962-lilrae-readiness/tracked-file-inventory.tsv | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/reviews/962-lilrae-readiness/tracked-file-inventory.tsv b/docs/reviews/962-lilrae-readiness/tracked-file-inventory.tsv index c6b81e4b3..86d07bc87 100644 --- a/docs/reviews/962-lilrae-readiness/tracked-file-inventory.tsv +++ b/docs/reviews/962-lilrae-readiness/tracked-file-inventory.tsv @@ -870,6 +870,7 @@ src/aptl/backends/_raes_scenario_resolution.py core-candidate Migrate only after src/aptl/backends/_raes_service_index_placement.py core-candidate Migrate only after public-import, history and scenario-independence verification src/aptl/backends/_raes_shuffle_implementation_catalog.py backend-owned-runtime Retain Shuffle implementation profiles selected only under open authority src/aptl/backends/_raes_start_failure.py core-candidate Keep redacted scenario-start failure translation with generic RAES admission +src/aptl/backends/_raes_start_retry.py core-candidate Keep the generic start-retry runtime-owner release with RAES orchestration src/aptl/backends/_raes_stateful_observation.py core-candidate Migrate only after public-import, history and scenario-independence verification src/aptl/backends/_raes_stateful_values.py backend-owned-runtime Retain authored value readers used by stateful realization src/aptl/backends/_runtime_concern_disclosure.py core-candidate Migrate only after public-import, history and scenario-independence verification @@ -1605,6 +1606,7 @@ tests/test_raes_runtime_observation.py tests-follow-owner Retain meaningful boun tests/test_raes_runtime_orchestration.py tests-follow-owner Post-baseline regression coverage for the RAES orchestration boundary tests/test_raes_service_index_schema.py tests-follow-owner Retain meaningful boundary and parity tests with migrated owner tests/test_raes_service_index_schema_lowering.py tests-follow-owner Retain meaningful boundary and parity tests with migrated owner +tests/test_raes_start_retry.py tests-follow-owner Retain start-retry runtime-owner release and result regressions tests/test_raes_stateful_realization.py tests-follow-owner Retain meaningful boundary and parity tests with migrated owner tests/test_range_integration.py tests-follow-owner Retain meaningful boundary and parity tests with migrated owner tests/test_realization_declaration_gate.py tests-follow-owner Retain meaningful boundary and parity tests with migrated owner From dd4d13369e8edd470469d5121e3d1dc2da206d42 Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Sun, 25 Oct 2026 10:52:46 +0000 Subject: [PATCH 3/3] refactor: split the start retry for the return limit and type its outcome copy --- src/aptl/backends/_raes_start_retry.py | 16 +++++++++++++++- tests/test_raes_start_retry.py | 12 ++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/aptl/backends/_raes_start_retry.py b/src/aptl/backends/_raes_start_retry.py index 8f70342f0..ddd6b8f40 100644 --- a/src/aptl/backends/_raes_start_retry.py +++ b/src/aptl/backends/_raes_start_retry.py @@ -4,6 +4,7 @@ from collections.abc import Callable from dataclasses import replace +from typing import cast from aptl.backends.raes_start_model import AcesStartOutcome from aptl.utils.logging import get_logger @@ -38,6 +39,16 @@ def apply_with_single_retry( return _with_error( first, f"{first.lab_result.error}\n{_RETRY_SKIPPED} ({release_failure})." ) + return _retry_after_release(first, run_attempt, before_retry) + + +def _retry_after_release( + first: AcesStartOutcome, + run_attempt: Callable[[], AcesStartOutcome], + before_retry: Callable[[], None], +) -> AcesStartOutcome: + """Run the hook and the retry; a failed retry also carries the first error.""" + before_retry() retry = run_attempt() if retry.lab_result.success: @@ -62,4 +73,7 @@ def _release_runtime_owner(outcome: AcesStartOutcome) -> str | None: def _with_error(outcome: AcesStartOutcome, error: str) -> AcesStartOutcome: - return replace(outcome, lab_result=replace(outcome.lab_result, error=error)) + """Return the outcome with its lab result's error replaced.""" + + lab_result = replace(outcome.lab_result, error=error) + return cast(AcesStartOutcome, replace(outcome, lab_result=lab_result)) diff --git a/tests/test_raes_start_retry.py b/tests/test_raes_start_retry.py index 85c8c66de..5c7a8b5f6 100644 --- a/tests/test_raes_start_retry.py +++ b/tests/test_raes_start_retry.py @@ -108,8 +108,10 @@ def test_failed_retry_reports_both_attempts(tmp_path): before_retry.assert_called_once_with() assert error.count(_START_FAILED) == 2 retry_part, initial_part = error.split("\nInitial attempt: ") - assert _RETRY_ERROR in retry_part and _FIRST_ERROR not in retry_part - assert _FIRST_ERROR in initial_part and _START_FAILED in initial_part + assert _RETRY_ERROR in retry_part + assert _FIRST_ERROR not in retry_part + assert _FIRST_ERROR in initial_part + assert _START_FAILED in initial_part class _StuckClockDriver: @@ -142,7 +144,8 @@ def test_unreleasable_first_owner_skips_the_retry(mocker, tmp_path): assert result.lab_result.success is False assert backend.realize.call_count == 1 before_retry.assert_not_called() - assert _START_FAILED in error and _FIRST_ERROR in error + assert _START_FAILED in error + assert _FIRST_ERROR in error assert "Automatic retry skipped" in error assert "runtime manager clock driver remains active" in error assert "Initial attempt" not in error @@ -231,7 +234,8 @@ def test_lab_start_json_reports_both_attempts_after_a_failed_retry(mocker, tmp_p assert result.exit_code == 1 assert payload["ok"] is False assert error.count(_START_FAILED) == 2 - assert _RETRY_ERROR in error and _FIRST_ERROR in error + assert _RETRY_ERROR in error + assert _FIRST_ERROR in error assert f"Initial attempt: RAES runtime handoff failed: {_START_FAILED}" in error assert backend.realize.call_count == 2 sleep.assert_called_once_with(60)