From 5f8212f29676dc09254872344b1dbc5f7b35057b Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Tue, 13 Oct 2026 13:20:13 +0000 Subject: [PATCH] ci: select python lanes by marker and give every integration test a lane The general Python lanes used -k "not integration", which also dropped the unmarked tests in test_api_integration.py and test_cortex_integration_config.py, and 19 files of integration-marked tests ran in no lane. - Select the coverage, fuzz, Ubuntu and macOS lanes by -m. - Add a python-integration job that runs the 18 unlaned integration files on the runner's Docker behind Docker, Compose and cgroup v2 probes. - Add tests/test_integration_lane_coverage.py: every integration-marked test must be selected by a checks.yml lane or listed as live-lab-only with a reason; -k lanes and unknown -m forms fail. - Fix what the new job exposed: the mixed-realization test's image node needed environment a root compose file no longer supplies (#1269); the DNS test built the wrong base image and removed the wrong container name; the substrate and Samba capability tests skipped when an image was absent instead of building it. Closes #1321 --- .github/workflows/checks.yml | 65 ++- CONTRIBUTING.md | 6 + .../tracked-file-inventory.tsv | 1 + tests/test_compose_capability_necessity.py | 17 +- tests/test_integration_lane_coverage.py | 382 ++++++++++++++++++ tests/test_mixed_realization_integration.py | 42 +- tests/test_substrate_posture_real_docker.py | 29 +- tests/test_techvault_dns_integration.py | 37 +- 8 files changed, 512 insertions(+), 67 deletions(-) create mode 100644 tests/test_integration_lane_coverage.py diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 1a6d5e50..466b06d8 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -65,14 +65,19 @@ jobs: - run: | pip install --require-hashes -r requirements/dev.txt pip install -e . --no-deps --no-build-isolation + # Lanes select by marker, never by `-k`: `-k "not integration"` also + # dropped unmarked tests whose file name contains "integration" (#1321). + # A command-line `-m` replaces the `addopts` one, so `not fuzz` is spelled + # out. tests/test_integration_lane_coverage.py fails if an + # integration-marked test is selected by no lane in this file. - run: | - coverage run -m pytest tests/ -q -k "not integration" + coverage run -m pytest tests/ -q -m "not integration and not fuzz" coverage xml -o coverage.xml - # Property-based tests carry the `fuzz` marker, which `addopts` excludes - # from the default run above. Without this step no gate ran them, and two - # went stale behind contract changes (#1141, #1152) unnoticed (#1193). + # Property-based tests carry the `fuzz` marker, which the run above + # excludes. Without this step no gate ran them, and two went stale + # behind contract changes (#1141, #1152) unnoticed (#1193). - name: Property-based (fuzz) tests - run: python -m pytest tests/ -q -m fuzz -k "not integration" + run: python -m pytest tests/ -q -m "fuzz and not integration" # The env-file writers must deliver each value exactly (#966). Only the # real Docker and Compose readers can prove that. The readback tests are # integration-marked, so the run above deselects them. The runner has @@ -128,6 +133,52 @@ jobs: python -m pytest tests/test_env_pack_bundle.py tests/test_plugin_pack_compatibility.py -q -m integration + python-integration: + name: Python integration tests on the runner's Docker + runs-on: ubuntu-latest + # Issue #1321: these integration-marked tests ran in no lane, and two of + # them had gone stale unnoticed. They need the runner's Docker, Compose or a + # wheel build, not a lab. They run in their own job so the required + # python-tests job, which SonarCloud waits on, does not grow. The probes + # stop the step if Docker, Compose or cgroup v2 is missing, so the tests + # cannot pass by a skip; -rs lists any skip in the log. A new + # integration-marked test needs a lane here or an entry in the live-lab + # list in tests/test_integration_lane_coverage.py. + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - run: | + pip install --require-hashes -r requirements/dev.txt + pip install -e . --no-deps --no-build-isolation + - name: Integration tests on Docker, Compose and wheel builds + run: | + docker info > /dev/null + docker compose version + test "$(docker info --format '{{.CgroupVersion}}')" = 2 + python -m pytest -q -rs -m integration \ + tests/test_appliance_boundary_integration.py \ + tests/test_base_container_admission_integration.py \ + tests/test_compose_capability_necessity.py \ + tests/test_env_pack_realization.py \ + tests/test_hatch_build_hook.py \ + tests/test_kali_capture_compose.py \ + tests/test_listener_observation_integration.py \ + tests/test_materializer_integration.py \ + tests/test_misp_redis_realization_contract.py \ + tests/test_mixed_realization_integration.py \ + tests/test_node_materialization_integration.py \ + tests/test_out_of_tree_bundle_integration.py \ + tests/test_pack_adapter_install_artifact.py \ + tests/test_participant_workbench_adapter.py \ + tests/test_ssh_key_bundle.py \ + tests/test_stage_hosted_seat_payload.py \ + tests/test_substrate_posture_real_docker.py \ + tests/test_techvault_dns_integration.py + python-cross-platform: name: Python cross-platform smoke (${{ matrix.os }}) runs-on: ${{ matrix.os }} @@ -136,9 +187,9 @@ jobs: matrix: include: - os: ubuntu-latest - pytest_args: tests/ -q -k "not integration" + pytest_args: tests/ -q -m "not integration and not fuzz" - os: macos-latest - pytest_args: tests/ -q -k "not integration" + pytest_args: tests/ -q -m "not integration and not fuzz" - os: windows-latest pytest_args: >- tests/test_cli.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 67319a61..3a3cc4d6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -113,6 +113,12 @@ Docker Desktop/nested-virtualization environment needed to prove full lab boot on macOS or Windows, so `aptl lab start` remains a manual platform-validation gate. +CI lanes select tests by marker (`-m`), never by keyword (`-k`). Every test +marked `integration` runs in a `checks.yml` job that names its file. A test +that needs a live lab instead of a hosted runner goes in `LIVE_LAB_ONLY` in +`tests/test_integration_lane_coverage.py` with its reason. That test fails when +a new integration test has neither. + When changing `docker-compose.yml`, container Dockerfiles, or files under `config/`, run only focused contract tests locally. CI/CD owns the clean lab stop/start validation so local verification does not expand into a full system diff --git a/docs/reviews/962-lilrae-readiness/tracked-file-inventory.tsv b/docs/reviews/962-lilrae-readiness/tracked-file-inventory.tsv index 324a06e1..b8e13367 100644 --- a/docs/reviews/962-lilrae-readiness/tracked-file-inventory.tsv +++ b/docs/reviews/962-lilrae-readiness/tracked-file-inventory.tsv @@ -1491,6 +1491,7 @@ tests/test_hostenv.py tests-follow-owner Retain meaningful boundary and parity t tests/test_base_container_admission_integration.py tests-follow-owner Retain meaningful boundary and parity tests with migrated owner tests/test_realization_declaration_gate_contract.py tests-follow-owner Retain meaningful boundary and parity tests with migrated owner tests/test_in_process_agents.py tests-follow-owner Retain meaningful boundary and parity tests with migrated owner +tests/test_integration_lane_coverage.py tests-follow-owner Retain meaningful boundary and parity tests with migrated owner tests/test_kali_capture_broker.py tests-follow-owner Retain meaningful boundary and parity tests with migrated owner tests/test_kali_capture_compose.py tests-follow-owner Retain meaningful boundary and parity tests with migrated owner tests/test_kill.py tests-follow-owner Retain meaningful boundary and parity tests with migrated owner diff --git a/tests/test_compose_capability_necessity.py b/tests/test_compose_capability_necessity.py index cc463b81..d60e0b09 100644 --- a/tests/test_compose_capability_necessity.py +++ b/tests/test_compose_capability_necessity.py @@ -26,9 +26,11 @@ Issue #976 evaluates replacing both product choices so neither grant is needed. The behavioural test is marked `integration` (per-test, not module-level, so the -default `-k "not integration"` run does not zero this module's coverage) because -it needs a real Docker daemon and the built Samba provider image. The contract -tests run everywhere and fail if a grant or its rationale is quietly dropped. +default `-m "not integration and not fuzz"` run does not zero this module's +coverage) because it needs a real Docker daemon. It builds the Samba provider +image through the backend's own `ensure_generic_base_image` (#1006) rather than +skipping when the tag is absent (#1321). The contract tests run everywhere and +fail if a grant or its rationale is quietly dropped. """ from __future__ import annotations @@ -165,12 +167,9 @@ def test_security_xattr_write_requires_sys_admin_in_the_samba_provider_image(): failure mode this test exists to prevent. """ - if subprocess.run( - ["docker", "image", "inspect", SAMBA_AD_BASE_IMAGE], - capture_output=True, - check=False, - ).returncode != 0: - pytest.skip(f"{SAMBA_AD_BASE_IMAGE} is not built on this host") + builder = DockerComposeBackend(project_dir=PROJECT_ROOT, project_name="aptl955") + failures = builder.ensure_generic_base_image(SAMBA_AD_BASE_IMAGE) + assert failures == [], failures without = _write_security_xattr() assert without.returncode != 0 diff --git a/tests/test_integration_lane_coverage.py b/tests/test_integration_lane_coverage.py new file mode 100644 index 00000000..71363597 --- /dev/null +++ b/tests/test_integration_lane_coverage.py @@ -0,0 +1,382 @@ +"""Every integration-marked test runs in a CI lane or says why it cannot (#1321). + +The general lanes in `.github/workflows/checks.yml` used `-k "not integration"`. +`-k` matches file and class names as well as markers, so it also dropped the +unmarked tests in `test_api_integration.py` and `test_cortex_integration_config.py`, +and nothing checked that the integration-marked tests it excluded ran anywhere +else. Nineteen files of them ran in no lane, and two had gone stale. + +This module reads every `pytest` command in `checks.yml`, works out which tests +each one selects by its `-m` expression and path arguments, and compares that +with what pytest itself collects as integration-marked. Collection is the +authority: decorators, `pytestmark`, class marks and `pytest.param` marks all +count. A test that needs a live lab instead of a hosted runner goes in +`LIVE_LAB_ONLY` with its reason and the command that runs it. +""" + +from __future__ import annotations + +import re +import shlex +import subprocess +import sys +import tomllib +from dataclasses import dataclass +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = ROOT / ".github" / "workflows" / "checks.yml" + +# Integration-marked tests no hosted lane can run, each with its reason and how +# to run it. An entry that a lane selects, or that is no longer collected as +# integration-marked, fails the check, so the list cannot go stale. +LIVE_LAB_ONLY: dict[str, str] = { + "tests/test_techvault_live_gate.py::test_live_gate_passes_on_techvault": ( + "Deploys the full TechVault range and tears it down, which a hosted " + "runner cannot hold; gated by APTL_LIVE_GATE=1. On a lab host run " + "`APTL_LIVE_GATE=1 pytest tests/test_techvault_live_gate.py -m integration` " + "(docs/raes/techvault-live-validation-gate.md)." + ), +} + +# What each `-m` form a lane uses selects: (integration-marked tests, tests +# with neither the integration nor the fuzz marker). A closed table rather than +# an expression evaluator: an unknown form fails until someone adds it here. +MARKER_FORMS: dict[str, tuple[bool, bool]] = { + "integration": (True, False), + "not integration and not fuzz": (False, True), + "fuzz and not integration": (False, False), + "not fuzz": (True, True), +} + +_VALUE_OPTIONS = frozenset({"-m", "-k", "-n", "-p"}) +_MATRIX_REF = re.compile(r"\$\{\{\s*matrix\.([A-Za-z0-9_-]+)\s*\}\}") +_API_AND_CORTEX = ( + "tests/test_api_integration.py", + "tests/test_cortex_integration_config.py", +) + + +@dataclass(frozen=True) +class PytestConfig: + marker: str + testpaths: tuple[str, ...] + + +@dataclass(frozen=True) +class Lane: + job: str + command: str + marker: str + keyword: str | None + paths: tuple[str, ...] + + def selects(self, node_or_path: str) -> bool: + return any( + node_or_path == path + or node_or_path.startswith(f"{path}::") + or node_or_path.startswith(f"{path.rstrip('/')}/") + for path in self.paths + ) + + +def _pytest_config() -> PytestConfig: + options = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8")) + ini = options["tool"]["pytest"]["ini_options"] + addopts = shlex.split(ini.get("addopts", "")) + marker = addopts[addopts.index("-m") + 1] if "-m" in addopts else "" + return PytestConfig(marker=marker, testpaths=tuple(ini.get("testpaths", ()))) + + +def _job_commands(job: dict) -> list[str]: + """Every shell line a job runs, once per matrix row, continuations joined.""" + + rows = ((job.get("strategy") or {}).get("matrix") or {}).get("include") or [{}] + lines: list[str] = [] + for step in job.get("steps") or []: + run = step.get("run") + if not run: + continue + for row in rows: + text = _MATRIX_REF.sub( + lambda ref, row=row: str(row.get(ref.group(1), ref.group(0))), run + ) + lines.extend(text.replace("\\\n", " ").splitlines()) + return lines + + +def _pytest_arguments(line: str) -> list[str] | None: + """The arguments of a `pytest` / `-m pytest` command, or None for others.""" + + if "pytest" not in line: + return None + try: + tokens = shlex.split(line) + except ValueError: + return None + for index, token in enumerate(tokens): + if token == "pytest" and (index == 0 or tokens[index - 1] == "-m"): + return tokens[index + 1 :] + return None + + +def _lane(job: str, line: str, arguments: list[str], config: PytestConfig) -> Lane: + marker, keyword, paths = config.marker, None, [] + tokens = iter(arguments) + for token in tokens: + if token in _VALUE_OPTIONS: + value = next(tokens, "") + marker = value if token == "-m" else marker + keyword = value if token == "-k" else keyword + elif not token.startswith("-"): + paths.append(token.removeprefix("./")) + return Lane( + job=job, + command=line.strip(), + marker=marker, + keyword=keyword, + paths=tuple(paths) or config.testpaths, + ) + + +def workflow_lanes(workflow: dict, config: PytestConfig) -> list[Lane]: + lanes: list[Lane] = [] + for job_id, job in (workflow.get("jobs") or {}).items(): + for line in _job_commands(job): + arguments = _pytest_arguments(line) + if arguments is not None: + lanes.append(_lane(job_id, line, arguments, config)) + return lanes + + +def _lane_problems(lanes: list[Lane]) -> list[str]: + problems: list[str] = [] + for lane in lanes: + if lane.keyword is not None: + problems.append( + f"{lane.job}: selects by -k {lane.keyword!r}, which also matches " + f"file and class names; select by -m instead: {lane.command}" + ) + if lane.marker not in MARKER_FORMS: + problems.append( + f"{lane.job}: unknown -m form {lane.marker!r}; add it to " + f"MARKER_FORMS in {Path(__file__).name}: {lane.command}" + ) + return problems + + +def _live_lab_problems( + integration_ids: set[str], selected: set[str], live_lab: dict[str, str] +) -> list[str]: + problems: list[str] = [] + for node, reason in live_lab.items(): + if node not in integration_ids: + problems.append( + f"{node}: listed live-lab test is not collected as integration-marked" + ) + elif node in selected: + problems.append( + f"{node}: listed live-lab test is selected by a lane; remove its entry" + ) + if not reason.strip(): + problems.append(f"{node}: listed live-lab test has no reason") + return problems + + +def audit( + workflow: dict, + integration_ids: set[str], + live_lab: dict[str, str], + config: PytestConfig, +) -> list[str]: + """Return every way the lanes and the live-lab list fail to cover the tests.""" + + lanes = workflow_lanes(workflow, config) + # A `-k` expression cannot be read from the workflow alone, so a lane that + # uses one is counted as selecting nothing (and reported above). + integration_lanes = [ + lane + for lane in lanes + if lane.keyword is None and MARKER_FORMS.get(lane.marker, (False, False))[0] + ] + selected = { + node + for node in integration_ids + if any(lane.selects(node) for lane in integration_lanes) + } + orphans = sorted(integration_ids - selected - set(live_lab)) + return [ + *_lane_problems(lanes), + *( + f"{node}: integration-marked but selected by no checks.yml lane and " + f"not in LIVE_LAB_ONLY (file {node.split('::', 1)[0]})" + for node in orphans + ), + *_live_lab_problems(integration_ids, selected, live_lab), + ] + + +def _collect(marker: str, *paths: str, keyword: str | None = None) -> set[str]: + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "--collect-only", + "-q", + "-p", + "no:cacheprovider", + "-m", + marker, + *(["-k", keyword] if keyword is not None else []), + *paths, + ], + cwd=ROOT, + capture_output=True, + text=True, + timeout=600, + check=False, + ) + # Exit code 5 means nothing was collected: an empty selection, not an error. + assert result.returncode in (0, 5), result.stdout[-2000:] + result.stderr[-2000:] + return {line for line in result.stdout.splitlines() if "::" in line} + + +@pytest.fixture(scope="module") +def checks_workflow() -> dict: + return yaml.safe_load(WORKFLOW.read_text(encoding="utf-8")) + + +def test_every_integration_test_has_a_lane_or_a_live_lab_reason(checks_workflow): + integration_ids = _collect("integration", "tests/") + assert integration_ids, "collection found no integration-marked tests" + + problems = audit(checks_workflow, integration_ids, LIVE_LAB_ONLY, _pytest_config()) + + assert problems == [], "\n".join(problems) + + +def test_the_coverage_lane_runs_the_unmarked_api_and_cortex_tests(checks_workflow): + (lane,) = [ + lane + for lane in workflow_lanes(checks_workflow, _pytest_config()) + if lane.job == "python-tests" and lane.command.startswith("coverage run") + ] + + assert all(lane.selects(path) for path in _API_AND_CORTEX), lane + collected = { + node.split("::", 1)[0] + for node in _collect(lane.marker, *_API_AND_CORTEX, keyword=lane.keyword) + } + assert collected == set(_API_AND_CORTEX) + + +_CONFIG = PytestConfig(marker="not fuzz", testpaths=("tests",)) +_LIVE = "tests/test_live.py::test_boot" +_DOCKER = "tests/test_docker.py::test_up" + + +def _workflow(*runs: str, matrix: list[dict] | None = None) -> dict: + job: dict = {"steps": [{"run": run} for run in runs]} + if matrix is not None: + job["strategy"] = {"matrix": {"include": matrix}} + return {"jobs": {"lane": job}} + + +_GENERAL = 'python -m pytest tests/ -q -m "not integration and not fuzz"' + + +@pytest.mark.parametrize( + ("workflow", "live_lab", "expected"), + [ + pytest.param( + _workflow( + _GENERAL, + "python -m pytest -q -m integration \\\n tests/test_docker.py", + ), + {_LIVE: "needs a lab"}, + [], + id="continued-file-lane-and-live-lab-entry-cover-everything", + ), + pytest.param( + _workflow(_GENERAL), + {_LIVE: "needs a lab"}, + [f"{_DOCKER}: integration-marked but selected by no checks.yml lane"], + id="orphan-is-reported-with-its-file", + ), + pytest.param( + _workflow('python -m pytest tests/ -q -k "not integration"'), + {_LIVE: "needs a lab"}, + [ + "lane: selects by -k 'not integration'", + f"{_DOCKER}: integration-marked but selected by no checks.yml lane", + ], + id="keyword-selection-is-rejected", + ), + pytest.param( + _workflow(f"python -m pytest -m 'docker or integration' {_DOCKER}"), + {_LIVE: "needs a lab"}, + [ + "lane: unknown -m form 'docker or integration'", + f"{_DOCKER}: integration", + ], + id="unknown-marker-form-fails-closed", + ), + pytest.param( + _workflow( + f"python -m pytest -q -m integration tests/test_docker.py {_LIVE}" + ), + {_LIVE: "needs a lab"}, + [f"{_LIVE}: listed live-lab test is selected by a lane"], + id="stale-live-lab-entry-is-rejected", + ), + pytest.param( + _workflow(f"python -m pytest -m integration {_DOCKER}"), + {_LIVE: " ", "tests/test_gone.py::test_x": "removed"}, + [ + f"{_LIVE}: listed live-lab test has no reason", + "tests/test_gone.py::test_x: listed live-lab test is not collected", + ], + id="live-lab-entry-needs-a-reason-and-a-collected-test", + ), + pytest.param( + _workflow("python -m pytest"), + {}, + [], + id="addopts-marker-and-testpaths-apply-without-arguments", + ), + pytest.param( + _workflow(f"coverage run -m pytest -m integration {_DOCKER}"), + {_LIVE: "needs a lab"}, + [], + id="node-id-argument-selects-that-test", + ), + pytest.param( + _workflow("python -m pytest -m integration tests/test_docker.py::test_u"), + {_LIVE: "needs a lab"}, + [f"{_DOCKER}: integration-marked but selected by no checks.yml lane"], + id="node-id-prefix-selects-nothing-else", + ), + pytest.param( + _workflow( + "python -m pytest ${{ matrix.args }}", + matrix=[ + {"args": _GENERAL.split("pytest ", 1)[1]}, + {"args": f"-m integration {_DOCKER}"}, + ], + ), + {_LIVE: "needs a lab"}, + [], + id="matrix-values-are-expanded-per-row", + ), + ], +) +def test_audit_rules(workflow, live_lab, expected): + problems = audit(workflow, {_LIVE, _DOCKER}, live_lab, _CONFIG) + + assert len(problems) == len(expected), problems + for prefix, problem in zip(expected, problems, strict=True): + assert problem.startswith(prefix), problems diff --git a/tests/test_mixed_realization_integration.py b/tests/test_mixed_realization_integration.py index 7ce710dd..6468036b 100644 --- a/tests/test_mixed_realization_integration.py +++ b/tests/test_mixed_realization_integration.py @@ -59,19 +59,13 @@ def _materialized_project(tmp_path): return tmp_path -_COMPOSE = """\ -services: - image-box: - image: postgres:16-alpine - container_name: aptl-image-box - environment: - POSTGRES_PASSWORD: test - POSTGRES_HOST_AUTH_METHOD: trust - free-box: - image: alpine:3.19 - container_name: aptl-free-box - command: ["sleep", "infinity"] -""" +# The image node's source must serve with no environment: a root +# `docker-compose.yml` is never a scenario realization input (#1269), so an +# image that needs a password from one exits before it gets an address. It must +# also declare no `VOLUME`, because removing the container leaves an anonymous +# volume behind (#1321). +_IMAGE_SOURCE = ("nginx", "1.27-alpine") +_IMAGE_REF = ":".join(_IMAGE_SOURCE) def _docker_available() -> bool: @@ -85,7 +79,6 @@ def _docker_available() -> bool: @pytest.mark.skipif(not _docker_available(), reason="docker daemon not available") def test_realize_materializes_runtime_node_and_starts_image_node_together(tmp_path): - (tmp_path / "docker-compose.yml").write_text(_COMPOSE) free_container = "aptl-free-box" image_container = "aptl-image-box" @@ -136,9 +129,9 @@ def test_realize_materializes_runtime_node_and_starts_image_node_together(tmp_pa DeploymentImageRealization( address="provision.node.image-box", service_name="image-box", - source_name="postgres", - source_version="16-alpine", - image_ref="postgres:16-alpine", + source_name=_IMAGE_SOURCE[0], + source_version=_IMAGE_SOURCE[1], + image_ref=_IMAGE_REF, mode="pull", policy_rule="allowed-source", ), @@ -194,15 +187,18 @@ def test_realize_materializes_runtime_node_and_starts_image_node_together(tmp_pa ).stdout ) - # The source: node was started via Compose as usual. - pg_ready = None + # The source: node was started via Compose as usual and serves. + served = None for _ in range(20): - pg_ready = backend.container_exec(realized_image, ["pg_isready"]) - if pg_ready.returncode == 0: + served = backend.container_exec( + realized_image, ["wget", "-qO-", "http://127.0.0.1/"] + ) + if served.returncode == 0: break time.sleep(1) - assert pg_ready is not None, pg_ready.stdout - assert pg_ready.returncode == 0, pg_ready.stdout + assert served is not None + assert served.returncode == 0, served.stdout + assert "nginx" in served.stdout # The generic materializer's directly-run container is connected to # the declared scenario network with its declared static IP, exactly diff --git a/tests/test_substrate_posture_real_docker.py b/tests/test_substrate_posture_real_docker.py index 2eb7a354..83bb1130 100644 --- a/tests/test_substrate_posture_real_docker.py +++ b/tests/test_substrate_posture_real_docker.py @@ -19,8 +19,11 @@ default seccomp profile boots it `running` with zero failed units. Marked `integration` per-test (not module-level) so the default -`-k "not integration"` run does not zero this module's coverage, and skipped -when the daemon, the images, or the required engine version are unavailable. +`-m "not integration and not fuzz"` run does not zero this module's coverage. +Each test builds its image through the backend's own +`ensure_generic_base_image`, as `aptl lab start` does (#1006), and fails if the +build fails; it skips only when the daemon or the required engine version is +unavailable (#1321). """ from __future__ import annotations @@ -49,6 +52,7 @@ "node22": "aptl/generic-systemd-node22-base:latest", } _ALL_SYSTEMD_IMAGES = list(_IMAGES) +_REPO_ROOT = Path(__file__).resolve().parents[1] def _docker(*args: str, timeout: int = 120) -> subprocess.CompletedProcess: @@ -73,6 +77,19 @@ def _daemon_supports_the_posture() -> bool: return False +def _build_image(image: str) -> str: + """Build ``image`` from this checkout's Dockerfile, as a lab start does. + + Skipping when the tag was absent let these tests pass only after some + earlier test happened to build it (#1321). + """ + + builder = DockerComposeBackend(project_dir=_REPO_ROOT, project_name="aptl955") + failures = builder.ensure_generic_base_image(image) + assert failures == [], failures + return image + + requires_daemon = pytest.mark.skipif( not _daemon_supports_the_posture(), reason="needs a cgroup v2 Docker daemon at Engine " @@ -87,9 +104,7 @@ def started_node(tmp_path): started: list[str] = [] def start(image_key: str) -> tuple[str, dict]: - image = _IMAGES[image_key] - if _docker("image", "inspect", image).returncode != 0: - pytest.skip(f"{image} is not built on this host") + image = _build_image(_IMAGES[image_key]) name = f"aptl-955-{image_key}-{int(time.time() * 1000)}" spec = BaseContainerSpec( node_address=f"provision.node.{name}", @@ -236,9 +251,7 @@ def test_a_retired_recipe_container_is_not_reused(started_node, tmp_path): accepted it and the hardening silently never applied. """ - image = _IMAGES["debian"] - if _docker("image", "inspect", image).returncode != 0: - pytest.skip(f"{image} is not built on this host") + image = _build_image(_IMAGES["debian"]) name = f"aptl-955-stale-{int(time.time() * 1000)}" created = _docker( diff --git a/tests/test_techvault_dns_integration.py b/tests/test_techvault_dns_integration.py index 10441e94..e5e4d4e5 100644 --- a/tests/test_techvault_dns_integration.py +++ b/tests/test_techvault_dns_integration.py @@ -26,8 +26,11 @@ from aptl.backends.raes_node_materialization import realize_node from aptl.backends.raes_realization import interpret_provisioning_plan from aptl.core.config import AptlConfig +from aptl.core.deployment._compose_base_container_realization import ( + _ensure_generic_base_images, +) from aptl.core.deployment.docker_compose import DockerComposeBackend -from tests.helpers import techvault_scenario_bundle +from tests.helpers import realized_container_name, techvault_scenario_bundle pytestmark = pytest.mark.integration @@ -47,25 +50,13 @@ def _docker_available() -> bool: ) +def _remove_container(name: str) -> None: + subprocess.run(["docker", "rm", "-f", name], capture_output=True, text=True) + + @pytest.mark.skipif(not _docker_available(), reason="docker daemon not available") def test_dns_node_boots_base_container_materialized_and_resolves(tmp_path): repo = Path(__file__).resolve().parent.parent - subprocess.run( - [ - "docker", - "build", - "-t", - "aptl/generic-systemd-base-debian:latest", - "-f", - str(repo / "containers/generic-systemd-base-debian/Dockerfile"), - str(repo), - ], - capture_output=True, - text=True, - timeout=600, - check=True, - ) - subprocess.run(["docker", "rm", "-f", "aptl-dns"], capture_output=True, text=True) cfg = AptlConfig(lab={"name": "x"}, containers={}) be = DockerComposeBackend(project_dir=repo, project_name="aptl") @@ -85,6 +76,12 @@ def test_dns_node_boots_base_container_materialized_and_resolves(tmp_path): spec = real.deployment_spec([]) dns_node = next(n for n in spec.nodes if n.name == "dns") + # Build the base image the plan selected for this node, the way a lab start + # does before it materializes anything. Building one fixed image by hand + # went stale when the plan moved the node onto a derived base (#1321). + assert _ensure_generic_base_images(be, (dns_node,)) == [] + dns_container = realized_container_name(be, dns_node.container_name) + _remove_container(dns_container) dns_content = [c for c in spec.content if c.target_address == dns_node.address] ops = [] for c in dns_content: @@ -136,6 +133,6 @@ def test_dns_node_boots_base_container_materialized_and_resolves(tmp_path): ) assert "172.20.1.20" in dig.stdout finally: - subprocess.run( - ["docker", "rm", "-f", "aptl-dns"], capture_output=True, text=True - ) + # The realized name is workspace-scoped (#1054): removing the declared + # name removed nothing and leaked the container on every run. + _remove_container(dns_container)