ci(backend): deflake Keycloak and Temporal test fixtures

.github/workflows/ci.yaml

@@ -12,6 +12,12 @@ jobs:
         defaults:
             run:
                 working-directory: backend
+        env:
+            # Temporal dev server for tests/fixtures/temporal_fixtures.py. Installed here
+            # rather than downloaded by the SDK at test time, which intermittently failed
+            # ("Failed starting Temporal dev server: error decoding response body").
+            TEMPORAL_CLI_VERSION: 1.9.1
+            TEMPORAL_CLI_SHA256: 09a0326a51db84d02735e53542b9ebd8c4758daf47482a9ab0abce15844e60d5
         steps:
         - name: checkout repo
           uses: actions/checkout@v4
@@ -22,5 +28,23 @@ jobs:
             enable-cache: true
         - name: Install dependencies
           run: uv sync --frozen --all-groups
+        - name: Restore Temporal CLI
+          uses: actions/cache@v4
+          with:
+            path: ~/.temporal-cli
+            key: temporal-cli-${{ env.TEMPORAL_CLI_VERSION }}-linux-amd64
+        - name: Install Temporal CLI
+          run: |
+            cli="$HOME/.temporal-cli/temporal"
+            if [ ! -x "$cli" ]; then
+              archive="$RUNNER_TEMP/temporal_cli.tar.gz"
+              curl -fsSL --retry 5 --retry-all-errors -o "$archive" \
+                "https://github.com/temporalio/cli/releases/download/v${TEMPORAL_CLI_VERSION}/temporal_cli_${TEMPORAL_CLI_VERSION}_linux_amd64.tar.gz"
+              echo "${TEMPORAL_CLI_SHA256}  $archive" | sha256sum -c -
+              mkdir -p "$HOME/.temporal-cli"
+              tar -xzf "$archive" -C "$HOME/.temporal-cli" temporal
+            fi
+            "$cli" --version
+            echo "TEMPORAL_CLI_PATH=$cli" >> "$GITHUB_ENV"
         - name: Run tests
           run: uv run python -m pytest

backend/tests/fixtures/keycloak/container.py

@@ -8,13 +8,25 @@
 container instead of Auth0.
 """
 
+import time
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Iterator
 
 import pytest
+import requests
 from testcontainers.keycloak import KeycloakContainer  # type: ignore[import-untyped]
 
+# Pinned rather than testcontainers' default ":latest", so the server CI starts
+# cannot change underneath it. Must stay >= 25: the readiness probe below relies
+# on /health/* being served on the management port.
+KEYCLOAK_IMAGE = "quay.io/keycloak/keycloak:26.7.5"
+# Covers booting Keycloak and importing realm.json on a slow CI runner. The image
+# pull happens earlier, inside DockerContainer.start(), and is not counted here.
+READINESS_TIMEOUT_SECONDS = 180.0
+# How much of Keycloak's output a failed startup carries, so a CI log shows why.
+_LOG_TAIL_LINES = 50
+
 REALM_NAME = "biosim-test"
 CLIENT_ID = "biosim-test-client"
 # Same audience/roles mappers as CLIENT_ID, but its own email-claim shape --
@@ -44,6 +56,50 @@ class KeycloakTestRealm:
     roles_claim: str = ROLES_CLAIM
 
 
+class _ManagementPortKeycloakContainer(KeycloakContainer):  # type: ignore[misc]
+    """KeycloakContainer whose readiness probe only polls the management port.
+
+    The upstream probe (testcontainers 4.14) falls back to the HTTP port's
+    /health/ready whenever the management port refuses a connection. On
+    Keycloak >= 25 that path is a 404, and the resulting HTTPError is not one of
+    the errors its retry decorator catches -- so a container that was merely
+    still booting failed the whole session (backend-ci run 38454571680: 13 setup
+    errors, all "404 ... /health/ready").
+
+    Unlike upstream, it does not also wait for the "started in" and "Created
+    temporary admin user" log lines. Tests only fetch realm users' tokens
+    (tokens.py) and never use the master-realm admin client, and Keycloak
+    serves /health/ready only after the realm import has finished. Restore the
+    admin-user wait before calling get_client().
+    """
+
+    def _readiness_probe(self) -> None:
+        url = f"{self.get_management_url()}/health/ready"
+        deadline = time.monotonic() + READINESS_TIMEOUT_SECONDS
+        while True:
+            try:
+                requests.get(url, timeout=2).raise_for_status()
+                return
+            except requests.RequestException as exc:
+                # A container that has exited will never become ready, so fail now
+                # rather than at the deadline.
+                self.reload()
+                if self.status in ("exited", "dead"):
+                    raise RuntimeError(
+                        f"Keycloak container {self.status} before it was ready.\n{self._log_tail()}"
+                    ) from exc
+                if time.monotonic() >= deadline:
+                    raise TimeoutError(
+                        f"Keycloak was not ready within {READINESS_TIMEOUT_SECONDS:.0f}s: {exc}\n{self._log_tail()}"
+                    ) from exc
+                time.sleep(1)
+
+    def _log_tail(self) -> str:
+        stdout, stderr = self.get_logs()
+        lines = (stdout + stderr).decode(errors="replace").splitlines()
+        return "Last Keycloak output:\n" + "\n".join(lines[-_LOG_TAIL_LINES:])
+
+
 @pytest.fixture(scope="session")
 def keycloak_container() -> Iterator[KeycloakContainer]:
     """Starts a single Keycloak dev container for the whole test session, pre-loaded with realm.json.
@@ -52,7 +108,7 @@ def keycloak_container() -> Iterator[KeycloakContainer]:
     needs a real token shares this one running container instead of paying
     the startup cost per test.
     """
-    with KeycloakContainer().with_realm_import_file(REALM_IMPORT_FILE) as container:
+    with _ManagementPortKeycloakContainer(KEYCLOAK_IMAGE).with_realm_import_file(REALM_IMPORT_FILE) as container:
         yield container
 
 

backend/tests/fixtures/temporal_fixtures.py

@@ -1,3 +1,4 @@
+import os
 from typing import AsyncGenerator
 
 import pytest
@@ -21,7 +22,15 @@
 async def temporal_env(request: pytest.FixtureRequest) -> AsyncGenerator[WorkflowEnvironment, None]:
     env_type = request.config.getoption("--workflow-environment")
     if env_type == "local":
-        env = await WorkflowEnvironment.start_local(ui=True, data_converter=pydantic_data_converter)
+        # CI pre-installs a pinned, checksum-verified Temporal CLI and points
+        # TEMPORAL_CLI_PATH at it (.github/workflows/ci.yaml): the SDK's on-demand
+        # download from temporal.download intermittently failed there with
+        # "error decoding response body". Unset locally, the SDK downloads as before.
+        env = await WorkflowEnvironment.start_local(
+            ui=True,
+            data_converter=pydantic_data_converter,
+            dev_server_existing_path=os.environ.get("TEMPORAL_CLI_PATH"),
+        )
     elif env_type == "time-skipping":
         env = await WorkflowEnvironment.start_time_skipping(data_converter=pydantic_data_converter)
     else: