diff --git a/app/constants.py b/app/constants.py index 091872f1..bd6c6de8 100644 --- a/app/constants.py +++ b/app/constants.py @@ -53,6 +53,12 @@ CASHPILOT_DEPLOYED = frozenset({"worker", "direct"}) +#: Linux's default ephemeral range (net.ipv4.ip_local_port_range): any program on +#: the host can hold a port in it for a moment, and Docker publishing a port 1:1 +#: then fails the whole start. +EPHEMERAL_PORTS = (32768, 60999) + + def drift_warnings_enabled() -> bool: """CASHPILOT_DRIFT_WARNINGS: off stops the worker reporting container settings and the UI judging them (app/drift.py). On unless set to 0, off, false or no.""" diff --git a/app/drift.py b/app/drift.py index b866233b..518cd64f 100644 --- a/app/drift.py +++ b/app/drift.py @@ -24,7 +24,8 @@ import shlex from typing import Any -from app import arch, catalog +from app import arch, catalog, lan_fence +from app.constants import EPHEMERAL_PORTS #: Repositories CashPilot builds itself (images//). The catalog pins their tag. OWN_IMAGE_PREFIX = "drumsergio/cashpilot-" @@ -127,6 +128,53 @@ def _names(values: set[str]) -> str: return ", ".join(sorted(values)) +def move_advice( + svc: dict[str, Any] | None, + shape: dict[str, Any] | None, + recorded: dict[str, Any] | None, + isolation_mode: str | None, +) -> dict[str, Any] | None: + """For a fence-gated container still on the host's network: the one action, and what to fix first. + + ``action`` depends on the worker's isolation mode (lan_fence.move_advice); + ``redeploy`` says whether the worker reports a mode where a redeploy alone does it; ``warning`` is set + when the recorded UDP range, which the redeploy keeps, overlaps Linux's + ephemeral range, where one port a host program holds fails the start. + None for any other container. + """ + if not svc or not isinstance(shape, dict) or not catalog.fence_gated(svc): + return None + if shape.get("network_mode") != "host": + return None + docker_conf = svc.get("docker") or {} + name = svc.get("name") or "it" + advice: dict[str, Any] = { + "action": lan_fence.move_advice(isolation_mode, name), + # Only where the worker says a redeploy alone works: one that reports no + # mode yet may be in legacy, off or on Podman, where it is refused. + "redeploy": isolation_mode in (lan_fence.MODE_ON, lan_fence.MODE_OPERATOR), + "warning": None, + } + rng = docker_conf.get("udp_port_range") + key = str(rng.get("env") or "") if isinstance(rng, dict) else "" + value = str(((recorded or {}).get("env") or {}).get(key) or "").strip() if key else "" + m = re.fullmatch(r"(\d{1,5}):(\d{1,5})", value) + if m and int(m[1]) <= EPHEMERAL_PORTS[1] and int(m[2]) >= EPHEMERAL_PORTS[0]: + low = EPHEMERAL_PORTS[0] + default = next((str(v.get("default") or "") for v in docker_conf.get("env") or [] if v.get("key") == key), "") + example = "" + d = re.fullmatch(r"(\d{1,5}):(\d{1,5})", default) + if d and int(d[2]) + 100 < low: + example = f", such as {int(d[1]) + 100}:{int(d[2]) + 100}" + advice["warning"] = ( + f"Its recorded {key}, {value}, overlaps Linux's ephemeral range ({low}-{EPHEMERAL_PORTS[1]}), which the " + "redeploy keeps: on its own network every port in it is published, and one a host program holds fails " + f"the start. Before you redeploy, set {key} to a free range below {low} other than the default{example} " + "(a value equal to the default keeps the recorded one), and change the router forward to match." + ) + return advice + + def container_drift( svc: dict[str, Any] | None, shape: dict[str, Any] | None, @@ -192,13 +240,7 @@ def container_drift( found.append( "uses its own network, but should share the host's" if want_host - else "shares the host's network" - + ( - f", where {reach}. Switch isolation on in Fleet, then redeploy {svc.get('name') or 'it'} once to " - "move it onto the fenced bridge" - if gated - else "" - ) + else "shares the host's network" + (f", where {reach}" if gated else "") ) elif not want_host: recorded_ports = None if gated else rec.get("ports") diff --git a/app/isolation_check.py b/app/isolation_check.py index 6cba2cfd..b1bd788d 100644 --- a/app/isolation_check.py +++ b/app/isolation_check.py @@ -405,7 +405,7 @@ def _name(container: Any) -> str: NETNS_CAPABILITIES = frozenset({"NET_ADMIN", "SYS_ADMIN"}) -def host_netns_findings(containers: list[Any]) -> list[dict[str, Any]]: +def host_netns_findings(containers: list[Any], mode: str | None = None) -> list[dict[str, Any]]: """Managed containers holding NET_ADMIN or SYS_ADMIN in the host's network namespace. Such a container can rewrite the host's firewall, the LAN fence included, @@ -431,8 +431,7 @@ def host_netns_findings(containers: list[Any]) -> list[dict[str, Any]]: "detail": ( "These containers hold firewall capabilities in this host's own network namespace, so they can " "rewrite the host's firewall, the LAN fence included, and reach everything the host reaches: " - f"{', '.join(sorted(held))}. Switch isolation on in Fleet, then redeploy them once from the dashboard to " - "move them onto the fenced bridge." + f"{', '.join(sorted(held))}. " + lan_fence.move_advice(mode, "them", "them") ), "docs": ISOLATION_PAGE, } @@ -770,7 +769,7 @@ def _run(client: Any) -> dict[str, Any]: "not_answering_from_host": [], "addresses_left_out": [], "findings": ( - host_netns_findings(every) + host_netns_findings(every, mode) + placement_findings(every, network, mode) + ipv6_findings(client, every, network) ), diff --git a/app/lan_fence.py b/app/lan_fence.py index f27074fa..39c24cd5 100644 --- a/app/lan_fence.py +++ b/app/lan_fence.py @@ -1075,6 +1075,36 @@ def network_for_deploy() -> str | None: return NETWORK_NAME +def move_advice(mode: str | None, service: str = "it", it: str = "it") -> str: + """The one thing to do to move a fence-gated node off the host's network, in this isolation mode. + + ``on`` and ``operator`` take a redeploy alone (switching isolation on is + refused in ``operator`` mode); ``legacy`` needs isolation on first, ``off`` + the setting changed, and Podman cannot be fenced at all. ``None`` is a + worker that reports no mode. + """ + if mode == MODE_ON: + return f"Redeploy {service} once to move {it} onto the fenced bridge." + if mode == MODE_OPERATOR: + return f"Redeploy {service} once to move {it} onto your own network (CASHPILOT_CONTAINER_NETWORK)." + if mode == MODE_LEGACY: + return f"Switch isolation on in Fleet first, then redeploy {service} once to move {it} onto the fenced bridge." + if mode == MODE_OFF: + return ( + f"Set CASHPILOT_ISOLATION=auto on this worker and restart it, then redeploy {service} once to move " + f"{it} onto the fenced bridge." + ) + if mode == MODE_UNSUPPORTED: + return ( + f"{PODMAN_UNSUPPORTED}, so {service} cannot be fenced on this worker. Run {it} on Docker, or on your " + "own network (CASHPILOT_CONTAINER_NETWORK)." + ) + return ( + f"Redeploy {service} once to move {it} onto the fenced bridge; if isolation is not on for this worker, " + "switch it on in Fleet first." + ) + + def mode_problem(mode: str, service: str = "this service") -> str | None: """Why a fence-gated service cannot run in this isolation mode at all, or None. @@ -1086,12 +1116,7 @@ def mode_problem(mode: str, service: str = "this service") -> str | None: """ if mode in (MODE_ON, MODE_OPERATOR): return None - what = { - MODE_LEGACY: f"Switch isolation on in Fleet, then redeploy {service}.", - MODE_OFF: f"Set CASHPILOT_ISOLATION=auto on this worker and restart it, then redeploy {service}.", - MODE_UNSUPPORTED: "Run it on Docker with isolation on, or on your own network (CASHPILOT_CONTAINER_NETWORK).", - }.get(mode, f"Switch isolation on in Fleet, then redeploy {service}.") - return f"isolation is {mode} on this worker, and {service} runs only behind the LAN fence. {what}" + return f"isolation is {mode} on this worker, and {service} runs only behind the LAN fence. {move_advice(mode, service)}" def fence_problem(client: Any, service: str = "this service") -> str | None: diff --git a/app/main.py b/app/main.py index 5d8a8cab..386e6fff 100644 --- a/app/main.py +++ b/app/main.py @@ -279,6 +279,11 @@ async def _get_all_worker_containers(workers: list[dict[str, Any]] | None = None # check (app/drift.py). None from a worker too old to send it. "shape": c.get("shape"), "_arch": sys_info.get("arch"), + # The worker's isolation mode, for the drift advice on + # a fence-gated node still on the host's network. + "_isolation_mode": ( + (sys_info.get("isolation") or {}) if isinstance(sys_info.get("isolation"), dict) else {} + ).get("mode"), "_node": worker_name, "_worker_id": w.get("id"), "_has_docker": worker_has_docker, @@ -2618,8 +2623,8 @@ async def _instance_drift( inst: dict[str, Any], custom_image: str | None, specs: dict[tuple[str, int], dict[str, Any] | None], -) -> tuple[list[str], bool]: - """How this running copy differs from what a redeploy would build, and whether it has a record. +) -> tuple[list[str], bool, dict[str, Any] | None]: + """How this running copy differs from what a redeploy would build, whether it has a record, and the move advice. Never raises: a record that cannot be read leaves the copy judged on the catalog alone, and a check that fails says nothing rather than blanking @@ -2637,10 +2642,11 @@ async def _instance_drift( found = drift.container_drift( svc, inst.get("shape"), recorded, custom_image=custom_image, worker_arch=inst.get("_arch") ) + move = drift.move_advice(svc, inst.get("shape"), recorded, inst.get("_isolation_mode")) if found else None except Exception as exc: logger.warning("Drift check failed for %s on worker %s: %s", slug, wid, exc) - return [], False - return found, recorded is not None + return [], False, None + return found, recorded is not None, move @app.get("/api/services/deployed") @@ -2786,12 +2792,14 @@ async def api_services_deployed(request: Request) -> list[dict[str, Any]]: and inst.get("deployed_by") in constants.CASHPILOT_DEPLOYED and drift_ignored.get(f"{slug}|{inst.get('_worker_id')}") != drift.catalog_fingerprint(svc) ): - drift_found, has_record = await _instance_drift( + drift_found, has_record, move = await _instance_drift( svc, slug, inst, custom_images.get(inst.get("_worker_id")), specs ) if drift_found: detail["drift"] = drift_found detail["drift_has_record"] = has_record + if move: + detail["drift_move"] = move if inst.get("_is_android"): detail["net_tx_24h"] = inst.get("_net_tx_24h", 0) detail["net_rx_24h"] = inst.get("_net_rx_24h", 0) diff --git a/app/orchestrator.py b/app/orchestrator.py index 67a604cd..a58fa2d8 100644 --- a/app/orchestrator.py +++ b/app/orchestrator.py @@ -1244,8 +1244,7 @@ def _deploy_failure(exc: BaseException, restored: str, slug: str = "") -> Contai return ContainerRuntimeError(f"The container could not be created: {exc}. {restored}", 500) -#: Linux's default ephemeral range (net.ipv4.ip_local_port_range). -EPHEMERAL_PORTS = (32768, 60999) +from app.constants import EPHEMERAL_PORTS # noqa: E402 - Linux's default ephemeral range def _ephemeral_hint(reason: str, slug: str = "") -> str: diff --git a/app/static/js/app.js b/app/static/js/app.js index 592b432d..f3ea9646 100644 --- a/app/static/js/app.js +++ b/app/static/js/app.js @@ -1585,16 +1585,23 @@ const CP = (() => { const fix = inst.drift_has_record ? 'Redeploy it here: CashPilot keeps the settings it was deployed with.' : 'CashPilot has no record of how it was deployed, so it cannot rebuild it without losing its settings. Recreate it with the setting above, or remove it and deploy it again from the service page.'; + // A fence-gated node still on the host's network (app/drift.py move_advice): + // the one action for this worker's isolation mode first, then what to fix + // before it, then the reasons. + const move = inst.drift_move; + const why = move + ? `${escapeHtml(move.action)}${move.warning ? ` ${escapeHtml(move.warning)}` : ''}${inst.drift_has_record ? '' : ` ${fix}`}${inst.drift.map(d => `
ยท ${escapeHtml(d)}`).join('')}` + : `container does not match the catalog: ${what}. ${fix}`; let actions = ''; if (_isOwner && !inst.unmanaged) { - actions = inst.drift_has_record && inst.worker_id != null + actions = inst.drift_has_record && inst.worker_id != null && !(move && move.redeploy === false) ? `` : ``; if (inst.worker_id != null) { actions += ``; } } - items.push({ level: 'fail', tag: 'Needs fixing', what: svc.name, where, why: `container does not match the catalog: ${what}. ${fix}`, actions }); + items.push({ level: 'fail', tag: 'Needs fixing', what: svc.name, where, why, actions }); } } diff --git a/docs/isolation.md b/docs/isolation.md index 5cd9bd10..d1b2ea47 100644 --- a/docs/isolation.md +++ b/docs/isolation.md @@ -217,9 +217,11 @@ rules, and keeping other containers off its WebUI is up to those rules too. In `off` and `legacy` mode, and on Podman, the worker does not run the node at all: on Docker's default bridge every other earner could reach its WebUI and change its payout wallet, and nothing would keep it off your LAN. A deploy is -refused before the running node is touched, with the order to follow: switch -isolation on in Fleet, then redeploy Mysterium. A fenced node already running -in such a mode is stopped. +refused before the running node is touched, with what to do in that mode: +in `legacy`, switch isolation on in Fleet first, then redeploy Mysterium; in +`off`, set `CASHPILOT_ISOLATION=auto` and restart the worker first; on Podman +it cannot be fenced. In `on` and operator mode a redeploy alone moves it. A +fenced node already running in such a mode is stopped. **A node from 2.0.6 or earlier keeps running as it is.** A Mysterium container on the host's network, which is what every node deployed through 2.0.6 is, is diff --git a/tests/test_drift.py b/tests/test_drift.py index 4e19a9e8..c2bf17d1 100644 --- a/tests/test_drift.py +++ b/tests/test_drift.py @@ -201,7 +201,7 @@ def test_a_device_the_catalog_dropped_is_drift(self): def test_an_unreadable_record_still_judges_the_catalog(self): """A failed record read must not hide what the catalog alone shows.""" with patch("app.main.database.get_deployment_spec", AsyncMock(side_effect=RuntimeError("db"))): - found, has_record = asyncio.run( + found, has_record, _move = asyncio.run( main._instance_drift( BITPING, "bitping", {"_worker_id": 1, "shape": _shape(_bitping_spec(cap_add=None))}, None, {} ) @@ -256,7 +256,7 @@ def test_the_row_has_a_redeploy_handler(self): assert "tag: 'Needs fixing'" in js -def _deployed_row(svc, shape, recorded, deployed_by="worker", config=None): +def _deployed_row(svc, shape, recorded, deployed_by="worker", config=None, isolation_mode=None): from fastapi.testclient import TestClient container = { @@ -272,7 +272,13 @@ def _deployed_row(svc, shape, recorded, deployed_by="worker", config=None): "id": 1, "name": "w1", "status": "online", - "system_info": json.dumps({"docker_available": True, "arch": "x86_64"}), + "system_info": json.dumps( + { + "docker_available": True, + "arch": "x86_64", + **({"isolation": {"mode": isolation_mode}} if isolation_mode else {}), + } + ), "containers": json.dumps([container]), "apps": "[]", } diff --git a/tests/test_mysterium_fenced.py b/tests/test_mysterium_fenced.py index 78728deb..3c2a5a85 100644 --- a/tests/test_mysterium_fenced.py +++ b/tests/test_mysterium_fenced.py @@ -16,6 +16,7 @@ from __future__ import annotations +import contextlib import json import os from contextlib import asynccontextmanager @@ -445,7 +446,7 @@ def test_net_admin_in_the_host_netns_is_an_exposure(self): found = isolation_check.host_netns_findings([_c("cashpilot-mysterium", "host", ["NET_ADMIN", "SETUID"])]) assert found and found[0]["exposure"] is True assert "cashpilot-mysterium (NET_ADMIN)" in found[0]["detail"] - assert "Switch isolation on in Fleet, then redeploy them once" in found[0]["detail"] + assert "Redeploy them once to move them onto the fenced bridge" in found[0]["detail"] def test_control_net_admin_in_its_own_namespace_is_not(self): assert ( @@ -791,13 +792,15 @@ def test_a_redeploy_in_legacy_mode_is_refused_and_says_what_to_do(self, markers) with pytest.raises(orchestrator.ContainerNetworkError) as refused: _raw_deploy(old=old, mode=lan_fence.MODE_LEGACY) # The catalog's name, the one on the dashboard card the owner redeploys from. - assert f"Switch isolation on in Fleet, then redeploy {MYST['name']}." in str(refused.value) + assert f"Switch isolation on in Fleet first, then redeploy {MYST['name']} once" in str(refused.value) old.stop.assert_not_called() old.remove.assert_not_called() def test_drift_still_flags_it_and_gives_the_order(self): found = " ".join(drift.container_drift(MYST, _shape(network_mode="host"), HOST_RECORD)) - assert f"Switch isolation on in Fleet, then redeploy {MYST['name']} once" in found + assert "shares the host's network" in found + move = drift.move_advice(MYST, _shape(network_mode="host"), HOST_RECORD, lan_fence.MODE_LEGACY) + assert move["action"].startswith(f"Switch isolation on in Fleet first, then redeploy {MYST['name']} once") def test_control_a_fence_gated_one_in_legacy_mode_is_stopped(self, markers): node = _node(network_mode="bridge") @@ -981,15 +984,17 @@ class TestTheMessagesNameTheService: def test_the_mode_refusal_names_the_service_it_is_given(self): problem = lan_fence.mode_problem(lan_fence.MODE_LEGACY, "Other") assert "Other runs only behind the LAN fence" in problem - assert problem.endswith("Switch isolation on in Fleet, then redeploy Other.") + assert problem.endswith( + "Switch isolation on in Fleet first, then redeploy Other once to move it onto the fenced bridge." + ) def test_drift_names_the_service_and_claims_no_capability_it_lacks(self): other = {**MYST, "name": "Other", "docker": {**MYST["docker"], "cap_add": []}} shape = _shape(network_mode="host") shape["cap_add"] = [] found = " ".join(drift.container_drift(other, shape, None)) - assert "redeploy Other once" in found assert "NET_ADMIN" not in found + assert "Redeploy Other once" in drift.move_advice(other, shape, None, lan_fence.MODE_ON)["action"] def test_drift_flags_a_node_that_publishes_its_webui_but_not_its_range(self): found = drift.container_drift(MYST, _shape(udp=False), None) @@ -1622,3 +1627,91 @@ def test_control_a_taken_port_below_it_gets_no_hint(self): def test_control_another_refusal_gets_no_hint(self): assert orchestrator._ephemeral_hint("no such device /dev/net/tun") == "" + + +# --------------------------------------------------------------------------- +# The move off the host's network: one action for the worker's mode, then why +# --------------------------------------------------------------------------- + +NEW_RECORD = {**HOST_RECORD, "env": {"UI_ADDRESS": "127.0.0.1", "UDP_PORTS": "23000:23029"}} + + +class TestTheMoveAdvice: + @pytest.mark.parametrize( + ("mode", "says", "redeploy"), + [ + (lan_fence.MODE_ON, f"Redeploy {MYST['name']} once to move it onto the fenced bridge.", True), + (lan_fence.MODE_OPERATOR, f"Redeploy {MYST['name']} once to move it onto your own network", True), + (lan_fence.MODE_LEGACY, "Switch isolation on in Fleet first, then redeploy", False), + (lan_fence.MODE_OFF, "Set CASHPILOT_ISOLATION=auto on this worker and restart it, then redeploy", False), + (lan_fence.MODE_UNSUPPORTED, "cannot be fenced on this worker", False), + (None, "if isolation is not on for this worker, switch it on in Fleet first", False), + ], + ) + def test_the_action_fits_the_workers_mode(self, mode, says, redeploy): + move = drift.move_advice(MYST, _shape(network_mode="host"), NEW_RECORD, mode) + assert move["action"].startswith(says) or says in move["action"] + assert move["redeploy"] is redeploy + if mode in (lan_fence.MODE_ON, lan_fence.MODE_OPERATOR): + # Switching isolation on is refused in operator mode: a redeploy alone is the way. + assert "Switch isolation" not in move["action"] and "CASHPILOT_ISOLATION" not in move["action"] + + def test_the_drift_reason_carries_no_action_of_its_own(self): + found = drift.container_drift(MYST, _shape(network_mode="host"), HOST_RECORD) + host = next(f for f in found if "host's network" in f) + assert host == "shares the host's network, where its NET_ADMIN reaches the host's firewall" + + def test_a_recorded_range_in_the_ephemeral_range_is_flagged_before_the_redeploy(self): + move = drift.move_advice(MYST, _shape(network_mode="host"), HOST_RECORD, lan_fence.MODE_ON) + warning = move["warning"] + assert "56000:56100" in warning and "32768-60999" in warning + assert "Before you redeploy" in warning and "below 32768 other than the default" in warning + assert "23100:23129" in warning + + def test_control_a_range_below_it_or_none_on_record_is_not(self): + for record in (NEW_RECORD, None, {**HOST_RECORD, "env": {}}): + assert drift.move_advice(MYST, _shape(network_mode="host"), record, lan_fence.MODE_ON)["warning"] is None + + def test_control_a_node_already_off_the_host_network_gets_no_advice(self): + assert drift.move_advice(MYST, _shape(), NEW_RECORD, lan_fence.MODE_ON) is None + + @pytest.mark.parametrize( + ("mode", "switch"), + [(lan_fence.MODE_ON, False), (lan_fence.MODE_OPERATOR, False), (lan_fence.MODE_LEGACY, True)], + ) + def test_the_isolation_check_gives_the_same_action(self, mode, switch): + found = isolation_check.host_netns_findings([_c("cashpilot-mysterium", "host", ["NET_ADMIN"])], mode) + assert ("Switch isolation on in Fleet first" in found[0]["detail"]) is switch + assert "Redeploy them once" in found[0]["detail"] or "redeploy them once" in found[0]["detail"] + + def test_the_mode_refusal_uses_the_same_advice(self): + assert lan_fence.move_advice(lan_fence.MODE_OFF, "Other") in lan_fence.mode_problem(lan_fence.MODE_OFF, "Other") + unknown = lan_fence.mode_problem("weird", "Other") + assert "Redeploy Other once" in unknown and "switch it on in Fleet first" in unknown + + def test_the_dashboard_gets_the_workers_mode_and_the_warning(self): + from tests.test_drift import _deployed_row + + row = _deployed_row(MYST, _shape(network_mode="host"), recorded=HOST_RECORD, isolation_mode="operator") + move = row["instance_details"][0]["drift_move"] + assert move["action"].startswith(f"Redeploy {MYST['name']} once to move it onto your own network") + assert "56000:56100" in move["warning"] + + def test_the_dashboard_leads_with_the_action_and_offers_redeploy_only_where_it_works(self): + js = (ROOT / "app" / "static" / "js" / "app.js").read_text(encoding="utf-8") + assert "${escapeHtml(move.action)}${move.warning ? ` ${escapeHtml(move.warning)}` : ''}" in js + assert "!(move && move.redeploy === false)" in js + + +def test_the_isolation_check_passes_the_workers_mode_to_the_finding(): + client = MagicMock() + client.containers.list.return_value = [] + with ( + patch.object(lan_fence, "effective_network", return_value=(lan_fence.MODE_OPERATOR, "x")), + patch.object(isolation_check, "host_netns_findings", return_value=[]) as spy, + patch.object(isolation_check, "placement_findings", return_value=[]), + patch.object(isolation_check, "ipv6_findings", return_value=[]), + contextlib.suppress(Exception), + ): + isolation_check._run(client) + spy.assert_called_once_with([], lan_fence.MODE_OPERATOR)