Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/reviews/962-lilrae-readiness/tracked-file-inventory.tsv
Original file line number Diff line number Diff line change
Expand Up @@ -870,6 +870,7 @@ src/aptl/backends/_raes_scenario_resolution.py core-candidate Migrate only after
src/aptl/backends/_raes_service_index_placement.py core-candidate Migrate only after public-import, history and scenario-independence verification
src/aptl/backends/_raes_shuffle_implementation_catalog.py backend-owned-runtime Retain Shuffle implementation profiles selected only under open authority
src/aptl/backends/_raes_start_failure.py core-candidate Keep redacted scenario-start failure translation with generic RAES admission
src/aptl/backends/_raes_start_retry.py core-candidate Keep the generic start-retry runtime-owner release with RAES orchestration
src/aptl/backends/_raes_stateful_observation.py core-candidate Migrate only after public-import, history and scenario-independence verification
src/aptl/backends/_raes_stateful_values.py backend-owned-runtime Retain authored value readers used by stateful realization
src/aptl/backends/_runtime_concern_disclosure.py core-candidate Migrate only after public-import, history and scenario-independence verification
Expand Down Expand Up @@ -1605,6 +1606,7 @@ tests/test_raes_runtime_observation.py tests-follow-owner Retain meaningful boun
tests/test_raes_runtime_orchestration.py tests-follow-owner Post-baseline regression coverage for the RAES orchestration boundary
tests/test_raes_service_index_schema.py tests-follow-owner Retain meaningful boundary and parity tests with migrated owner
tests/test_raes_service_index_schema_lowering.py tests-follow-owner Retain meaningful boundary and parity tests with migrated owner
tests/test_raes_start_retry.py tests-follow-owner Retain start-retry runtime-owner release and result regressions
tests/test_raes_stateful_realization.py tests-follow-owner Retain meaningful boundary and parity tests with migrated owner
tests/test_range_integration.py tests-follow-owner Retain meaningful boundary and parity tests with migrated owner
tests/test_realization_declaration_gate.py tests-follow-owner Retain meaningful boundary and parity tests with migrated owner
Expand Down
13 changes: 9 additions & 4 deletions docs/sdl/runtime-architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,15 @@ The supported runtime handoff is now:
5. Start the lab through `aptl.backends.raes.start_raes_scenario()`.

If the deployment backend returns its existing retryable start diagnostic and
the admitted plan selected the `soc` profile, the SOC recovery hook runs and
APTL applies that same admitted `ExecutionPlan` one more time. It does not parse
the SDL or invoke the planner again. The retry decision therefore follows the
realized profile set, not the global container configuration.
the scenario's startup adapter admits the `BEFORE_BACKEND_RETRY` hook, lab start
waits 60 seconds, runs that hook, and APTL applies that same admitted
`ExecutionPlan` one more time. It does not parse the SDL or invoke the planner
again. The retry decision therefore follows the admitted adapter, not the global
container configuration. RAES lets one runtime manager own the target at a time,
so APTL closes the first attempt's manager before the hook runs. If that manager
can't be closed, APTL skips the retry and reports the first attempt's error. If
the retry also fails, its error ends with `Initial attempt:` and the first
attempt's diagnostics.

Runtime bindings are an in-process admission input, not APTL configuration or
secrets. APTL does not source them from the environment, forward them to
Expand Down
79 changes: 79 additions & 0 deletions src/aptl/backends/_raes_start_retry.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
"""Lab start's single backend retry under RAES single runtime ownership."""

from __future__ import annotations

from collections.abc import Callable
from dataclasses import replace
from typing import cast

from aptl.backends.raes_start_model import AcesStartOutcome
from aptl.utils.logging import get_logger
from aptl.utils.redaction import redact

log = get_logger("raes-backend")

_RETRY_SKIPPED = (
"Automatic retry skipped: the first runtime owner could not be released"
)


def apply_with_single_retry(
run_attempt: Callable[[], AcesStartOutcome],
before_retry: Callable[[], None] | None,
) -> AcesStartOutcome:
"""Run one attempt, and one retry when it failed retryably and a hook exists.

raes 7 lets one ``RuntimeManager`` own a target's components until it is
closed, and each attempt builds its own manager, so the first attempt's
manager is closed before the hook runs. If it cannot be released, the retry
is skipped and the first diagnostic is kept. A failed retry also carries the
first attempt's diagnostic.
"""

first = run_attempt()
if not first.retryable or before_retry is None:
return first
release_failure = _release_runtime_owner(first)
if release_failure is not None:
log.warning("Automatic backend retry skipped: %s", release_failure)
return _with_error(
first, f"{first.lab_result.error}\n{_RETRY_SKIPPED} ({release_failure})."
)
return _retry_after_release(first, run_attempt, before_retry)


def _retry_after_release(
first: AcesStartOutcome,
run_attempt: Callable[[], AcesStartOutcome],
before_retry: Callable[[], None],
) -> AcesStartOutcome:
"""Run the hook and the retry; a failed retry also carries the first error."""

before_retry()
retry = run_attempt()
if retry.lab_result.success:
return retry
return _with_error(
retry, f"{retry.lab_result.error}\nInitial attempt: {first.lab_result.error}"
)


def _release_runtime_owner(outcome: AcesStartOutcome) -> str | None:
"""Close the attempt's manager; return why it stayed open, if it did."""

manager = outcome.runtime_manager
if manager is None:
return None
try:
manager.close()
except (RuntimeError, TimeoutError) as exc:
# RAES refuses while callbacks run or its clock driver does not stop.
return redact(f"{type(exc).__name__}: {exc}")
return None


def _with_error(outcome: AcesStartOutcome, error: str) -> AcesStartOutcome:
"""Return the outcome with its lab result's error replaced."""

lab_result = replace(outcome.lab_result, error=error)
return cast(AcesStartOutcome, replace(outcome, lab_result=lab_result))
18 changes: 6 additions & 12 deletions src/aptl/backends/raes.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
INSTANTIATION_FAILURE_MESSAGE as INSTANTIATION_FAILURE_MESSAGE,
start_failure_outcome as _start_failure_outcome,
)
from aptl.backends._raes_start_retry import apply_with_single_retry
from aptl.backends.raes_diagnostics import render_raes_diagnostics
from aptl.backends.raes_execution_helpers import (
evaluation_results as collect_evaluation_results,
Expand Down Expand Up @@ -357,23 +358,16 @@ def _apply_with_backend_retry(

run_store = run_target.run_store if run_target is not None else None
run_id = run_target.run_id if run_target is not None else None
outcome = _run_execution_plan(
target,
execution_plan,
scenario_path,
run_store=run_store,
run_id=run_id,
)
if outcome.retryable and before_backend_retry is not None:
before_backend_retry()
return _run_execution_plan(
return apply_with_single_retry(
lambda: _run_execution_plan(
target,
execution_plan,
scenario_path,
run_store=run_store,
run_id=run_id,
)
return outcome
),
before_backend_retry,
)


def _run_execution_plan(
Expand Down
3 changes: 3 additions & 0 deletions tests/test_raes_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -576,6 +576,9 @@ def apply(self, execution_plan):
self.target, execution_plan, self._initial_snapshot
)

def close(self):
"""Release nothing: unlike the real manager, the double claims no component."""


def test_create_aptl_manifest_is_canonical_backend_manifest_v2():
from raes_backend_protocols.capabilities import BackendManifest
Expand Down
Loading
Loading