From 06097081848831ffdc474250c075e65953c564b7 Mon Sep 17 00:00:00 2001 From: Novrus Shehaj <78491450+NovrusShehaj@users.noreply.github.com> Date: Wed, 21 Oct 2026 13:15:33 -0400 Subject: [PATCH 1/2] ci(backend): deflake Keycloak and Temporal test fixtures Keycloak: testcontainers 4.14's readiness probe falls back to the HTTP port's /health/ready when the management port refuses a connection. That path is a 404 on Keycloak >= 25, and HTTPError is not among the errors its retry decorator catches, so a container that was still booting failed the whole session (run 38454571680: 13 setup errors). Poll only the management port with a bounded retry, and pin the image to 26.7.5 instead of :latest. Temporal: WorkflowEnvironment.start_local downloaded the dev server from temporal.download on every run and intermittently failed with "error decoding response body" (run 39444224018). CI now installs a pinned, checksum-verified CLI from the GitHub release (cached per version) and the fixture uses it via TEMPORAL_CLI_PATH; locally the SDK still downloads as before. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yaml | 24 +++++++++++++ backend/tests/fixtures/keycloak/container.py | 37 +++++++++++++++++++- backend/tests/fixtures/temporal_fixtures.py | 11 +++++- 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 45332394..49653ed0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -12,6 +12,12 @@ jobs: defaults: run: working-directory: backend + env: + # Temporal dev server for tests/fixtures/temporal_fixtures.py. Installed here + # rather than downloaded by the SDK at test time, which intermittently failed + # ("Failed starting Temporal dev server: error decoding response body"). + TEMPORAL_CLI_VERSION: 1.9.1 + TEMPORAL_CLI_SHA256: 09a0326a51db84d02735e53542b9ebd8c4758daf47482a9ab0abce15844e60d5 steps: - name: checkout repo uses: actions/checkout@v4 @@ -22,5 +28,23 @@ jobs: enable-cache: true - name: Install dependencies run: uv sync --frozen --all-groups + - name: Restore Temporal CLI + uses: actions/cache@v4 + with: + path: ~/.temporal-cli + key: temporal-cli-${{ env.TEMPORAL_CLI_VERSION }}-linux-amd64 + - name: Install Temporal CLI + run: | + cli="$HOME/.temporal-cli/temporal" + if [ ! -x "$cli" ]; then + archive="$RUNNER_TEMP/temporal_cli.tar.gz" + curl -fsSL --retry 5 --retry-all-errors -o "$archive" \ + "https://github.com/temporalio/cli/releases/download/v${TEMPORAL_CLI_VERSION}/temporal_cli_${TEMPORAL_CLI_VERSION}_linux_amd64.tar.gz" + echo "${TEMPORAL_CLI_SHA256} $archive" | sha256sum -c - + mkdir -p "$HOME/.temporal-cli" + tar -xzf "$archive" -C "$HOME/.temporal-cli" temporal + fi + "$cli" --version + echo "TEMPORAL_CLI_PATH=$cli" >> "$GITHUB_ENV" - name: Run tests run: uv run python -m pytest diff --git a/backend/tests/fixtures/keycloak/container.py b/backend/tests/fixtures/keycloak/container.py index 76aea3d3..d57cc191 100644 --- a/backend/tests/fixtures/keycloak/container.py +++ b/backend/tests/fixtures/keycloak/container.py @@ -8,13 +8,22 @@ container instead of Auth0. """ +import time from dataclasses import dataclass from pathlib import Path from typing import Iterator import pytest +import requests from testcontainers.keycloak import KeycloakContainer # type: ignore[import-untyped] +# Pinned rather than testcontainers' default ":latest", so the server CI starts +# cannot change underneath it. Must stay >= 25: the readiness probe below relies +# on /health/* being served on the management port. +KEYCLOAK_IMAGE = "quay.io/keycloak/keycloak:26.7.5" +# A cold CI runner pulls the image and imports realm.json before reporting ready. +READINESS_TIMEOUT_SECONDS = 180.0 + REALM_NAME = "biosim-test" CLIENT_ID = "biosim-test-client" # Same audience/roles mappers as CLIENT_ID, but its own email-claim shape -- @@ -44,6 +53,32 @@ class KeycloakTestRealm: roles_claim: str = ROLES_CLAIM +class _ManagementPortKeycloakContainer(KeycloakContainer): # type: ignore[misc] + """KeycloakContainer whose readiness probe only polls the management port. + + The upstream probe (testcontainers 4.14) falls back to the HTTP port's + /health/ready whenever the management port refuses a connection. On + Keycloak >= 25 that path is a 404, and the resulting HTTPError is not one of + the errors its retry decorator catches -- so a container that was merely + still booting failed the whole session (backend-ci run 38454571680: 13 setup + errors, all "404 ... /health/ready"). + """ + + def _readiness_probe(self) -> None: + url = f"{self.get_management_url()}/health/ready" + deadline = time.monotonic() + READINESS_TIMEOUT_SECONDS + while True: + try: + requests.get(url, timeout=2).raise_for_status() + return + except requests.RequestException as exc: + if time.monotonic() >= deadline: + raise TimeoutError( + f"Keycloak was not ready within {READINESS_TIMEOUT_SECONDS:.0f}s: {exc}" + ) from exc + time.sleep(1) + + @pytest.fixture(scope="session") def keycloak_container() -> Iterator[KeycloakContainer]: """Starts a single Keycloak dev container for the whole test session, pre-loaded with realm.json. @@ -52,7 +87,7 @@ def keycloak_container() -> Iterator[KeycloakContainer]: needs a real token shares this one running container instead of paying the startup cost per test. """ - with KeycloakContainer().with_realm_import_file(REALM_IMPORT_FILE) as container: + with _ManagementPortKeycloakContainer(KEYCLOAK_IMAGE).with_realm_import_file(REALM_IMPORT_FILE) as container: yield container diff --git a/backend/tests/fixtures/temporal_fixtures.py b/backend/tests/fixtures/temporal_fixtures.py index 5f0ff3d1..85d686b2 100644 --- a/backend/tests/fixtures/temporal_fixtures.py +++ b/backend/tests/fixtures/temporal_fixtures.py @@ -1,3 +1,4 @@ +import os from typing import AsyncGenerator import pytest @@ -21,7 +22,15 @@ async def temporal_env(request: pytest.FixtureRequest) -> AsyncGenerator[WorkflowEnvironment, None]: env_type = request.config.getoption("--workflow-environment") if env_type == "local": - env = await WorkflowEnvironment.start_local(ui=True, data_converter=pydantic_data_converter) + # CI pre-installs a pinned, checksum-verified Temporal CLI and points + # TEMPORAL_CLI_PATH at it (.github/workflows/ci.yaml): the SDK's on-demand + # download from temporal.download intermittently failed there with + # "error decoding response body". Unset locally, the SDK downloads as before. + env = await WorkflowEnvironment.start_local( + ui=True, + data_converter=pydantic_data_converter, + dev_server_existing_path=os.environ.get("TEMPORAL_CLI_PATH"), + ) elif env_type == "time-skipping": env = await WorkflowEnvironment.start_time_skipping(data_converter=pydantic_data_converter) else: From 2f710f172ecd8e28936cd5c87928598fbf8a3de9 Mon Sep 17 00:00:00 2001 From: Novrus Shehaj <78491450+NovrusShehaj@users.noreply.github.com> Date: Wed, 21 Oct 2026 14:57:20 -0400 Subject: [PATCH 2/2] ci(backend): fail fast and surface logs when Keycloak startup fails Follow-up to review on #151: - The readiness probe now stops at once with a RuntimeError when the container has exited, instead of polling until the 180 s deadline. That error and the timeout both carry the last 50 lines of Keycloak's output, so a failed CI run shows why. - Document that the probe skips upstream's "started in" / admin-user log waits, why that is safe for these tests, and what to restore before using the admin client. - Correct the timeout comment: the image pull happens in DockerContainer.start(), before the probe, and is not counted. Co-Authored-By: Claude Opus 5.5 --- backend/tests/fixtures/keycloak/container.py | 25 ++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/backend/tests/fixtures/keycloak/container.py b/backend/tests/fixtures/keycloak/container.py index d57cc191..60213343 100644 --- a/backend/tests/fixtures/keycloak/container.py +++ b/backend/tests/fixtures/keycloak/container.py @@ -21,8 +21,11 @@ # cannot change underneath it. Must stay >= 25: the readiness probe below relies # on /health/* being served on the management port. KEYCLOAK_IMAGE = "quay.io/keycloak/keycloak:26.7.5" -# A cold CI runner pulls the image and imports realm.json before reporting ready. +# Covers booting Keycloak and importing realm.json on a slow CI runner. The image +# pull happens earlier, inside DockerContainer.start(), and is not counted here. READINESS_TIMEOUT_SECONDS = 180.0 +# How much of Keycloak's output a failed startup carries, so a CI log shows why. +_LOG_TAIL_LINES = 50 REALM_NAME = "biosim-test" CLIENT_ID = "biosim-test-client" @@ -62,6 +65,12 @@ class _ManagementPortKeycloakContainer(KeycloakContainer): # type: ignore[misc] the errors its retry decorator catches -- so a container that was merely still booting failed the whole session (backend-ci run 38454571680: 13 setup errors, all "404 ... /health/ready"). + + Unlike upstream, it does not also wait for the "started in" and "Created + temporary admin user" log lines. Tests only fetch realm users' tokens + (tokens.py) and never use the master-realm admin client, and Keycloak + serves /health/ready only after the realm import has finished. Restore the + admin-user wait before calling get_client(). """ def _readiness_probe(self) -> None: @@ -72,12 +81,24 @@ def _readiness_probe(self) -> None: requests.get(url, timeout=2).raise_for_status() return except requests.RequestException as exc: + # A container that has exited will never become ready, so fail now + # rather than at the deadline. + self.reload() + if self.status in ("exited", "dead"): + raise RuntimeError( + f"Keycloak container {self.status} before it was ready.\n{self._log_tail()}" + ) from exc if time.monotonic() >= deadline: raise TimeoutError( - f"Keycloak was not ready within {READINESS_TIMEOUT_SECONDS:.0f}s: {exc}" + f"Keycloak was not ready within {READINESS_TIMEOUT_SECONDS:.0f}s: {exc}\n{self._log_tail()}" ) from exc time.sleep(1) + def _log_tail(self) -> str: + stdout, stderr = self.get_logs() + lines = (stdout + stderr).decode(errors="replace").splitlines() + return "Last Keycloak output:\n" + "\n".join(lines[-_LOG_TAIL_LINES:]) + @pytest.fixture(scope="session") def keycloak_container() -> Iterator[KeycloakContainer]: