diff --git a/charms/garm/src/charm.py b/charms/garm/src/charm.py index ecb72009..69fed873 100755 --- a/charms/garm/src/charm.py +++ b/charms/garm/src/charm.py @@ -27,7 +27,7 @@ resolve_entity, ) from entity_reconciler import EntityReconciler -from garm_api import GarmApiClient, GarmApiError, GarmAuthenticatedClient, GarmConnectionError +from garm_api import GarmApiClient, GarmApiError, GarmAuthenticatedClient from garm_template import CharmedTemplateError from garm_template import apply_charmed_template as _apply_garm_template from github_reconciler import ( @@ -161,6 +161,10 @@ def __init__(self, *args: typing.Any) -> None: """ _validate_paas_charm_hook_contract() super().__init__(*args) + # Multiple observers can deliver the same hook (notably update-status). Claim one + # teardown attempt per hook process; a new hook instance re-observes GARM state so + # a killed process can safely retry. + self._teardown_claimed = False for event in ( self.on.install, self.on.leader_elected, @@ -177,7 +181,9 @@ def __init__(self, *args: typing.Any) -> None: self.framework.observe(event, self._reconcile) self.framework.observe(self.on.get_credentials_action, self._on_get_credentials_action) - self.framework.observe(self.on.remove, self._on_remove) + self.framework.observe(self.on["postgresql"].relation_departed, self._teardown) + self.framework.observe(self.on.stop, self._teardown) + self.framework.observe(self.on.remove, self._teardown) def _is_tearing_down(self) -> bool: """Return whether Juju plans no remaining units for the local application.""" @@ -197,13 +203,6 @@ def _reconcile_with_migrations(self, event: ops.EventBase) -> None: return self._normal_reconcile_with_migrations(event) - def _teardown(self, event: ops.EventBase) -> None: - """Handle a local teardown event without normal reconciliation.""" - logger.info( - "Skipping normal GARM reconciliation for %s during local teardown", - event.handle.kind, - ) - @block_if_invalid_data def _normal_reconcile(self, _: ops.EventBase) -> None: """Reconcile active GARM charm state.""" @@ -239,7 +238,7 @@ def _route_reconcile_with_migrations(self, event: ops.EventBase) -> None: def _on_update_status(self, event: ops.HookEvent) -> None: """Run the framework update-status handler only while active.""" if self._is_tearing_down(): - logger.info("Skipping update-status handling during local teardown") + self._teardown(event) return super()._on_update_status(event) @@ -250,79 +249,114 @@ def _on_rotate_secret_key_action(self, event: ops.ActionEvent) -> None: return super()._on_rotate_secret_key_action(event) - def _on_remove(self, _: ops.RemoveEvent) -> None: - """Drain GARM resources before Juju removes the application.""" - if not self.unit.is_leader(): - logger.info("Skipping GARM removal cleanup on a non-leader unit") - return - if self.app.planned_units() > 0: - logger.info( - "Skipping GARM removal cleanup while the application still has planned units" - ) - return + def _teardown_failure( + self, + message: str, + *, + strict: bool, + cause: BaseException | None = None, + ) -> None: + """Raise an early cleanup error or record an unconfirmed late attempt.""" + if strict: + logger.error(message) + if cause is None: + raise GarmCleanupError(message) + raise GarmCleanupError(message) from cause + logger.warning("Late GARM teardown is unconfirmed: %s", message) - initialization_client = GarmApiClient(GARM_LOCAL_API_BASE_URL) + def _teardown_credentials(self, *, strict: bool) -> tuple[str, str] | None: + """Return local admin credentials without consulting any relation.""" try: - if not initialization_client.is_initialized(): - logger.info( - "GARM has not completed first-run; no resources require removal cleanup" - ) - return - except GarmConnectionError as exc: - if self._get_postgresql_config() is None: - logger.info( - "PostgreSQL is not configured and GARM is unreachable; " - "GARM cannot have been initialized, so no cleanup is required" - ) - return - message = ( - f"Cannot determine whether GARM was initialized: {exc}. " - "Operator action: restore GARM/API availability, then retry " - "Juju application removal." - ) - logger.error(message) - raise GarmCleanupError(message) from exc - except GarmApiError as exc: - message = ( - f"Cannot determine whether GARM was initialized: {exc}. " - "Operator action: restore GARM/API availability, then retry " - "Juju application removal." + admin_creds = self._get_admin_credentials() + except (ops.ModelError, ops.SecretNotFoundError) as exc: + self._teardown_failure( + "GARM admin credentials are unavailable; teardown cannot be confirmed. " + "Operator action: restore the labelled admin credentials secret, " + "then retry teardown.", + strict=strict, + cause=exc, ) - logger.error(message) - raise GarmCleanupError(message) from exc - - admin_creds = self._get_admin_credentials() + return None if not admin_creds: - message = ( - "GARM admin credentials are unavailable; refusing removal. " + self._teardown_failure( + "GARM admin credentials are unavailable; teardown cannot be confirmed. " "Operator action: restore the labelled admin credentials secret, " - "then retry Juju application removal." + "then retry teardown.", + strict=strict, ) - logger.error(message) - raise GarmCleanupError(message) + return None username = admin_creds.get("username") password = admin_creds.get("password") if not username or not password: - message = ( - "GARM admin credentials are incomplete; refusing removal. " + self._teardown_failure( + "GARM admin credentials are incomplete; teardown cannot be confirmed. " "Operator action: restore username and password in the labelled " - "admin credentials secret, then retry Juju application removal." + "admin credentials secret, then retry teardown.", + strict=strict, ) - logger.error(message) - raise GarmCleanupError(message) + return None + return username, password + + def _teardown(self, event: ops.EventBase) -> None: + """Idempotently drain GARM resources for a local teardown event. + + The operation intentionally re-observes GARM's current API state on every hook. + ``GarmResourceCleanup`` disables resources before deleting them, polls asynchronous + runner removal, and treats already-missing resources as success, so repeated calls + remain safe after retries or a hook process restart. No relation-backed data is read. + """ + if not self.unit.is_leader(): + logger.info("Skipping GARM teardown on a non-leader unit") + return + if not self._is_tearing_down(): + logger.info("Skipping GARM teardown while the application still has planned units") + return + if self._teardown_claimed: + logger.info("GARM teardown already claimed in this hook process") + return + self._teardown_claimed = True + + strict = isinstance(event, ops.RelationDepartedEvent) + logger.info("Handling GARM teardown for %s", event.handle.kind) + try: + initialized = GarmApiClient(GARM_LOCAL_API_BASE_URL).is_initialized() + except GarmApiError as exc: + self._teardown_failure( + f"Cannot determine whether GARM was initialized: {exc}. " + "Operator action: restore GARM/API availability, then retry teardown.", + strict=strict, + cause=exc, + ) + return + + if not initialized: + logger.info("GARM has not completed first-run; teardown is already complete") + return + + credentials = self._teardown_credentials(strict=strict) + if credentials is None: + return + username, password = credentials - base_url = GARM_LOCAL_API_BASE_URL try: - auth_client = GarmAuthenticatedClient.from_login(base_url, username, password) + auth_client = GarmAuthenticatedClient.from_login( + GARM_LOCAL_API_BASE_URL, username, password + ) GarmResourceCleanup(auth_client).run() + except GarmCleanupError as exc: + self._teardown_failure( + f"GARM teardown did not complete: {exc}", + strict=strict, + cause=exc, + ) except GarmApiError as exc: - logger.error( - "GARM removal cleanup failed: %s. Operator action: resolve the " - "reported GARM/API/runner issue, then retry Juju application removal.", - exc, + self._teardown_failure( + f"GARM teardown failed: {exc}. " + "Operator action: restore GARM/API/runner availability, then retry teardown.", + strict=strict, + cause=exc, ) - raise def _on_get_credentials_action(self, event: ops.ActionEvent) -> None: """Return the GARM admin credentials to the operator. diff --git a/charms/garm/tests/unit/test_charm.py b/charms/garm/tests/unit/test_charm.py index d15c7b26..fa6a0ee3 100644 --- a/charms/garm/tests/unit/test_charm.py +++ b/charms/garm/tests/unit/test_charm.py @@ -40,6 +40,7 @@ MANAGED_CREDENTIAL_DESCRIPTION, CredentialSpec, ) +from resource_cleanup import GarmCleanupError MODEL_NAME = "garm-model" CONTAINER_NAME = "app" @@ -571,58 +572,59 @@ def test_remove_unit_skips_global_cleanup_while_application_remains( garm_api.auth.from_login.assert_not_called() -def test_remove_blocks_when_garm_initialization_state_is_unknown( +def test_remove_tolerates_unknown_garm_initialization_state( ctx: Context, garm_api: _GarmApiMocks, caplog: pytest.LogCaptureFixture ): """ arrange: The unauthenticated GARM initialization probe cannot reach the API. act: Emit application removal. - assert: Removal remains blocked because resources cannot be proven absent. + assert: Late teardown logs an unconfirmed result and lets Juju finish removal. """ garm_api.client.return_value.is_initialized.side_effect = GarmConnectionError( "connection refused" ) - with pytest.raises(UncaughtCharmError, match="Cannot determine whether GARM was initialized"): - ctx.run(ctx.on.remove(), _state(planned_units=0)) + ctx.run(ctx.on.remove(), _state(planned_units=0)) garm_api.auth.from_login.assert_not_called() - assert "restore GARM/API availability" in caplog.text + assert "Late GARM teardown is unconfirmed" in caplog.text -def test_remove_allows_unreachable_garm_without_postgresql_setup( +def test_remove_allows_unreachable_garm_without_reading_postgresql( ctx: Context, garm_api: _GarmApiMocks, caplog: pytest.LogCaptureFixture ): """ arrange: PostgreSQL is not configured and GARM's API is unreachable. act: Emit application removal. - assert: Removal succeeds because GARM could not have been initialized without PostgreSQL. + assert: Late teardown succeeds without consulting the PostgreSQL relation. """ garm_api.client.return_value.is_initialized.side_effect = GarmConnectionError( "connection refused" ) - with patch("charm.GarmResourceCleanup") as cleanup_cls: + with patch.object( + GarmCharm, + "_get_postgresql_config", + side_effect=AssertionError("postgresql relation was read"), + ): ctx.run(ctx.on.remove(), _state(planned_units=0, postgresql_data={})) - cleanup_cls.assert_not_called() garm_api.auth.from_login.assert_not_called() - assert "PostgreSQL is not configured" in caplog.text + assert "Late GARM teardown is unconfirmed" in caplog.text -def test_remove_refuses_without_admin_credentials( +def test_remove_tolerates_missing_admin_credentials( ctx: Context, garm_api: _GarmApiMocks, caplog: pytest.LogCaptureFixture ): """ - arrange: A charm without GARM admin credentials. + arrange: A late removal hook cannot read the GARM admin credentials. act: Emit the application remove event. - assert: Removal fails and no authenticated client is created. + assert: Removal completes while recording that teardown is unconfirmed. """ - with pytest.raises(UncaughtCharmError, match="credentials are unavailable"): - ctx.run(ctx.on.remove(), _state(planned_units=0)) + ctx.run(ctx.on.remove(), _state(planned_units=0)) garm_api.auth.from_login.assert_not_called() - assert "Operator action: restore the labelled admin credentials secret" in caplog.text + assert "Late GARM teardown is unconfirmed" in caplog.text def test_unpopulated_configurator_relation_does_not_prune( @@ -661,6 +663,168 @@ def test_missing_configurator_relation_refreshes_stale_app_status( assert out.app_status == ops.WaitingStatus("Waiting for garm-configurator relation") +def test_remove_event_is_bound_to_the_teardown_orchestrator(ctx: Context, garm_api: _GarmApiMocks): + """ + arrange: The remove event is emitted for a local GARM teardown. + act: Observe which charm method receives the event. + assert: The remove observer uses the shared teardown orchestrator. + """ + state = _state(secrets=_owned_secrets(), planned_units=0) + with ctx(ctx.on.remove(), state) as manager: + with patch.object(manager.charm, "_teardown", wraps=manager.charm._teardown) as teardown: + manager.run() + + teardown.assert_called_once() + garm_api.auth.from_login.assert_called_once() + + +def test_update_status_teardown_is_single_flight(ctx: Context, garm_api: _GarmApiMocks): + """ + arrange: Both GARM and paas-charm observe update-status during local teardown. + act: Emit update-status. + assert: The shared teardown orchestrator performs one cleanup pass in the hook. + """ + state = _state(planned_units=0, secrets=_owned_secrets()) + with patch("charm.GarmResourceCleanup") as cleanup_cls: + ctx.run(ctx.on.update_status(), state) + + cleanup_cls.assert_called_once_with(garm_api.auth_client) + cleanup_cls.return_value.run.assert_called_once_with() + + +def test_late_teardown_failure_is_single_flight(ctx: Context, garm_api: _GarmApiMocks): + """ + arrange: GARM is unavailable during update-status teardown. + act: Both update-status observers invoke the teardown path. + assert: One failed late attempt is enough for this hook process; a later hook can retry. + """ + garm_api.client.return_value.is_initialized.side_effect = GarmConnectionError( + "connection refused" + ) + + ctx.run(ctx.on.update_status(), _state(planned_units=0)) + + assert garm_api.client.return_value.is_initialized.call_count == 1 + + +def test_postgresql_departure_does_not_teardown_a_planned_application( + ctx: Context, garm_api: _GarmApiMocks +): + """ + arrange: PostgreSQL departs while another GARM unit remains planned. + act: Emit the relation-departed event. + assert: The teardown coordinator preserves the application and does not drain GARM. + """ + state = _state(planned_units=1, secrets=_owned_secrets()) + with patch("charm.GarmResourceCleanup") as cleanup_cls: + ctx.run( + ctx.on.relation_departed(_relation(state, "postgresql"), remote_unit=0), + state, + ) + + cleanup_cls.assert_not_called() + garm_api.auth.from_login.assert_not_called() + + +def test_postgresql_departure_uses_the_teardown_orchestrator( + ctx: Context, garm_api: _GarmApiMocks +): + """ + arrange: PostgreSQL departs while the local GARM application is tearing down. + act: Emit the pre-break PostgreSQL relation event. + assert: The direct relation observer uses the shared GARM teardown operation. + """ + state = _state(planned_units=0, secrets=_owned_secrets()) + with patch("charm.GarmResourceCleanup") as cleanup_cls: + ctx.run( + ctx.on.relation_departed(_relation(state, "postgresql"), remote_unit=0), + state, + ) + + cleanup_cls.assert_called_once_with(garm_api.auth_client) + cleanup_cls.return_value.run.assert_called_once_with() + + +def test_early_postgresql_teardown_wraps_api_failure(ctx: Context, garm_api: _GarmApiMocks): + """ + arrange: PostgreSQL is departing and the GARM API cannot be reached. + act: Emit the pre-break PostgreSQL relation event. + assert: Strict teardown fails with the domain cleanup error so Juju can retry. + """ + garm_api.client.return_value.is_initialized.side_effect = GarmConnectionError( + "connection refused" + ) + state = _state(planned_units=0) + + with pytest.raises(UncaughtCharmError) as exc_info: + ctx.run(ctx.on.relation_departed(_relation(state, "postgresql"), remote_unit=0), state) + + assert isinstance(exc_info.value.__cause__, GarmCleanupError) + + +def test_relation_teardown_uses_the_same_idempotent_orchestrator( + ctx: Context, garm_api: _GarmApiMocks +): + """ + arrange: A configurator relation departs after no GARM units remain planned. + act: Emit the relation-departed event. + assert: The shared teardown path runs the existing GARM cleanup operation. + """ + state = _state(planned_units=0, secrets=_owned_secrets()) + with patch("charm.GarmResourceCleanup") as cleanup_cls: + ctx.run( + ctx.on.relation_departed( + _relation(state, GARM_CONFIGURATOR_RELATION_NAME), remote_unit=0 + ), + state, + ) + + cleanup_cls.assert_called_once_with(garm_api.auth_client) + cleanup_cls.return_value.run.assert_called_once_with() + + +def test_late_teardown_tolerates_missing_admin_secret( + ctx: Context, garm_api: _GarmApiMocks, caplog: pytest.LogCaptureFixture +): + """ + arrange: GARM is initialized but the admin secret cannot be read after teardown starts. + act: Emit application removal. + assert: Teardown completes without relation fallback or an uncaught secret error. + """ + garm_api.client.return_value.is_initialized.return_value = True + with patch.object( + GarmCharm, + "_get_admin_credentials", + side_effect=ops.ModelError("secret unavailable"), + ): + ctx.run(ctx.on.remove(), _state(planned_units=0)) + + garm_api.auth.from_login.assert_not_called() + assert "Late GARM teardown is unconfirmed" in caplog.text + + +def test_late_teardown_does_not_read_postgresql_when_garm_is_unavailable( + ctx: Context, garm_api: _GarmApiMocks +): + """ + arrange: GARM is unavailable during a late remove hook and PostgreSQL data is forbidden. + act: Emit application removal. + assert: The teardown orchestrator returns without consulting relation data. + """ + garm_api.client.return_value.is_initialized.side_effect = GarmConnectionError( + "connection refused" + ) + + with patch.object( + GarmCharm, + "_get_postgresql_config", + side_effect=AssertionError("postgresql relation was read"), + ): + ctx.run(ctx.on.remove(), _state(planned_units=0)) + + garm_api.auth.from_login.assert_not_called() + + # --- First-run initialisation ------------------------------------------------------------- @@ -1324,35 +1488,36 @@ def _assert_teardown_event_is_gated( """Assert that one event cannot enter normal state or GARM reconciliation.""" state = event_case.state_factory() - with ( - patch.object( - GarmCharm, - "_create_charm_state", - side_effect=AssertionError("charm state was created during teardown"), - ), - patch( - "charm_state.CharmState.from_charm", - side_effect=AssertionError("GARM charm state was reconstructed"), - ), - patch.object( - GarmCharm, "_ensure_secrets", side_effect=AssertionError("secrets were read") - ), - patch.object( - GarmCharm, - "_get_postgresql_config", - side_effect=AssertionError("postgresql relation was read"), - ), - patch.object( - GarmCharm, - "_get_configurator_provider_configs", - side_effect=AssertionError("configurator relation was read"), - ), - patch.object(GarmCharm, "_teardown", autospec=True) as teardown, - ): - out = ctx.run(event_case.emit(ctx, state), state) + with ctx(event_case.emit(ctx, state), state) as manager: + with ( + patch.object(manager.charm, "_teardown", return_value=None) as teardown, + patch.object( + GarmCharm, + "_create_charm_state", + side_effect=AssertionError("charm state was created during teardown"), + ), + patch( + "charm_state.CharmState.from_charm", + side_effect=AssertionError("GARM charm state was reconstructed"), + ), + patch.object( + GarmCharm, "_ensure_secrets", side_effect=AssertionError("secrets were read") + ), + patch.object( + GarmCharm, + "_get_postgresql_config", + side_effect=AssertionError("postgresql relation was read"), + ), + patch.object( + GarmCharm, + "_get_configurator_provider_configs", + side_effect=AssertionError("configurator relation was read"), + ), + ): + out = manager.run() assert out is not None - teardown.assert_called_once() + teardown.assert_called() garm_api.client.assert_not_called() garm_api.auth.from_login.assert_not_called() garm_api.github.assert_not_called() diff --git a/charms/tests/e2e/conftest.py b/charms/tests/e2e/conftest.py index 42a6e7ec..f0fb54c0 100644 --- a/charms/tests/e2e/conftest.py +++ b/charms/tests/e2e/conftest.py @@ -329,7 +329,13 @@ def deploy_e2e_scaleset_fixture( # Best effort only: the workflow's own sweep is what guarantees no VM is left # behind, since a fixture cannot run if the model or the runner dies mid-test. try: - _drain_and_delete_scaleset(juju, garm_app, label) + if garm_app not in juju.status().apps: + logger.info( + "GARM application %s was removed by the test; skipping API scale-set drain", + garm_app, + ) + else: + _drain_and_delete_scaleset(juju, garm_app, label) except (requests.RequestException, ValueError, KeyError) as exc: logger.warning("Best-effort scale set teardown did not complete: %s", exc) diff --git a/charms/tests/e2e/test_garm_teardown.py b/charms/tests/e2e/test_garm_teardown.py new file mode 100644 index 00000000..8b273e1a --- /dev/null +++ b/charms/tests/e2e/test_garm_teardown.py @@ -0,0 +1,179 @@ +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. + +"""GARM charm teardown integration tests on ProdStack.""" + +from __future__ import annotations + +import logging +import time +from typing import Any + +import jubilant +import pytest +import requests +from tests.e2e.conftest import GARM_API_PORT, _garm_login, _get_garm_address + +logger = logging.getLogger(__name__) + + +def _wait_for_provider_running_instance( + juju: jubilant.Juju, garm_app: str, scaleset_name: str, timeout: int = 25 * 60 +) -> dict[str, Any]: + """Wait for the GARM provider to report one real runner instance as running.""" + deadline = time.monotonic() + timeout + last_instances: list[dict[str, Any]] = [] + + while time.monotonic() < deadline: + address = _get_garm_address(juju, garm_app) + base_url = f"http://{address}:{GARM_API_PORT}/api/v1" + token = _garm_login(juju, address) + headers = {"Authorization": f"Bearer {token}"} + try: + scalesets_response = requests.get( + f"{base_url}/scalesets", headers=headers, timeout=30 + ) + scalesets_response.raise_for_status() + scaleset = next( + ( + item + for item in scalesets_response.json() + if item.get("name") == scaleset_name + ), + None, + ) + if scaleset is not None: + instances_response = requests.get( + f"{base_url}/scalesets/{scaleset['id']}/instances", + headers=headers, + timeout=30, + ) + instances_response.raise_for_status() + last_instances = instances_response.json() or [] + for instance in last_instances: + if instance.get("status") == "running": + logger.info( + "Observed provider-running GARM instance %s (runner_status=%s)", + instance.get("name"), + instance.get("runner_status"), + ) + return instance + except (requests.RequestException, ValueError, KeyError) as exc: + logger.info("Waiting for a provider-running instance: %s", exc) + time.sleep(15) + + pytest.fail( + f"No provider-running instance appeared in {scaleset_name!r}; " + f"last observed instances: {last_instances!r}" + ) + + +def _openstack_endpoint_and_token(credentials: dict[str, str]) -> tuple[str, str]: + """Authenticate to Keystone and return the compute endpoint and token.""" + auth_url = credentials["auth_url"].rstrip("/") + payload = { + "auth": { + "identity": { + "methods": ["password"], + "password": { + "user": { + "name": credentials["username"], + "domain": {"name": credentials["user_domain_name"]}, + } + }, + }, + "scope": { + "project": { + "name": credentials["project_name"], + "domain": {"name": credentials["project_domain_name"]}, + } + }, + } + } + response = requests.post(f"{auth_url}/auth/tokens", json=payload, timeout=30) + response.raise_for_status() + token = response.headers.get("X-Subject-Token") + assert token, "Keystone did not return a subject token" + + catalog = response.json()["token"]["catalog"] + compute = next(service for service in catalog if service.get("type") == "compute") + endpoints = compute.get("endpoints", []) + region = credentials["region_name"] + endpoint = next( + ( + item["url"] + for item in endpoints + if item.get("region") == region and item.get("interface") == "public" + ), + None, + ) + if endpoint is None: + endpoint = next( + (item["url"] for item in endpoints if item.get("region") == region), + None, + ) + assert endpoint, f"No compute endpoint was returned for region {region!r}" + return endpoint.rstrip("/"), token + + +def _wait_for_openstack_server_state( + credentials: dict[str, str], server_name: str, present: bool, timeout: int +) -> None: + """Wait until the exact GARM server is present or absent in Nova.""" + endpoint, token = _openstack_endpoint_and_token(credentials) + deadline = time.monotonic() + timeout + last_servers: list[dict[str, Any]] = [] + + while time.monotonic() < deadline: + response = requests.get( + f"{endpoint}/servers/detail", + params={"name": server_name}, + headers={"X-Auth-Token": token}, + timeout=30, + ) + response.raise_for_status() + last_servers = response.json().get("servers", []) + exists = any(item.get("name") == server_name for item in last_servers) + if exists == present: + logger.info( + "Nova server %s is %s", + server_name, + "present" if present else "absent", + ) + return + time.sleep(10) + + pytest.fail( + f"Nova server {server_name!r} did not become " + f"{'present' if present else 'absent'}; last response contained " + f"{len(last_servers)} matching server(s)" + ) + + +def test_garm_charm_removal_drains_provider_runner( + juju: jubilant.Juju, + garm_with_ingress: str, + e2e_scaleset: str, + openstack_credentials: dict[str, str], +) -> None: + """Remove GARM normally and verify its live runner is removed from Nova.""" + instance = _wait_for_provider_running_instance( + juju, garm_with_ingress, e2e_scaleset + ) + server_name = instance.get("name") + assert server_name, f"GARM instance did not include a provider name: {instance!r}" + _wait_for_openstack_server_state( + openstack_credentials, server_name, present=True, timeout=120 + ) + + logger.info("Removing disposable GARM application through the normal Juju path") + juju.remove_application(garm_with_ingress) + juju.wait( + lambda status: garm_with_ingress not in status.apps, + timeout=15 * 60, + delay=10, + ) + + _wait_for_openstack_server_state( + openstack_credentials, server_name, present=False, timeout=10 * 60 + )