From fe1166edfeafa571c87e395b11b6cab38a2a3873 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:48:15 +0100 Subject: [PATCH 1/3] fix(mysterium): tell each isolation mode the one step that moves a host-network node The drift finding, the isolation check and the mode refusal all told an owner to switch isolation on before redeploying a Mysterium node still on the host's network. In operator mode switching isolation on is refused, and in on mode it is already on, so the advice sent owners to a step that fails. Each now names the step for the worker's mode: a redeploy alone in on and operator mode, isolation on first in legacy, the setting in off, and no fence on Podman. The dashboard item leads with that step, lists the reasons after it, and offers Redeploy only where a redeploy can work. When the recorded UDP range, which the redeploy keeps, overlaps Linux's ephemeral range, it says so before the owner presses Redeploy. --- app/constants.py | 6 ++ app/drift.py | 56 +++++++++++++++--- app/isolation_check.py | 7 +-- app/lan_fence.py | 37 ++++++++++-- app/main.py | 18 ++++-- app/orchestrator.py | 3 +- app/static/js/app.js | 11 +++- docs/isolation.md | 8 ++- tests/test_drift.py | 12 +++- tests/test_mysterium_fenced.py | 102 +++++++++++++++++++++++++++++++-- 10 files changed, 222 insertions(+), 38 deletions(-) diff --git a/app/constants.py b/app/constants.py index 091872f1..bd6c6de8 100644 --- a/app/constants.py +++ b/app/constants.py @@ -53,6 +53,12 @@ CASHPILOT_DEPLOYED = frozenset({"worker", "direct"}) +#: Linux's default ephemeral range (net.ipv4.ip_local_port_range): any program on +#: the host can hold a port in it for a moment, and Docker publishing a port 1:1 +#: then fails the whole start. +EPHEMERAL_PORTS = (32768, 60999) + + def drift_warnings_enabled() -> bool: """CASHPILOT_DRIFT_WARNINGS: off stops the worker reporting container settings and the UI judging them (app/drift.py). On unless set to 0, off, false or no.""" diff --git a/app/drift.py b/app/drift.py index b866233b..830c479c 100644 --- a/app/drift.py +++ b/app/drift.py @@ -24,7 +24,8 @@ import shlex from typing import Any -from app import arch, catalog +from app import arch, catalog, lan_fence +from app.constants import EPHEMERAL_PORTS #: Repositories CashPilot builds itself (images//). The catalog pins their tag. OWN_IMAGE_PREFIX = "drumsergio/cashpilot-" @@ -127,6 +128,51 @@ def _names(values: set[str]) -> str: return ", ".join(sorted(values)) +def move_advice( + svc: dict[str, Any] | None, + shape: dict[str, Any] | None, + recorded: dict[str, Any] | None, + isolation_mode: str | None, +) -> dict[str, Any] | None: + """For a fence-gated container still on the host's network: the one action, and what to fix first. + + ``action`` depends on the worker's isolation mode (lan_fence.move_advice); + ``redeploy`` says whether a redeploy alone does it there; ``warning`` is set + when the recorded UDP range, which the redeploy keeps, overlaps Linux's + ephemeral range, where one port a host program holds fails the start. + None for any other container. + """ + if not svc or not isinstance(shape, dict) or not catalog.fence_gated(svc): + return None + if shape.get("network_mode") != "host": + return None + docker_conf = svc.get("docker") or {} + name = svc.get("name") or "it" + advice: dict[str, Any] = { + "action": lan_fence.move_advice(isolation_mode, name), + "redeploy": isolation_mode in (lan_fence.MODE_ON, lan_fence.MODE_OPERATOR, None), + "warning": None, + } + rng = docker_conf.get("udp_port_range") + key = str(rng.get("env") or "") if isinstance(rng, dict) else "" + value = str(((recorded or {}).get("env") or {}).get(key) or "").strip() if key else "" + m = re.fullmatch(r"(\d{1,5}):(\d{1,5})", value) + if m and int(m[1]) <= EPHEMERAL_PORTS[1] and int(m[2]) >= EPHEMERAL_PORTS[0]: + low = EPHEMERAL_PORTS[0] + default = next((str(v.get("default") or "") for v in docker_conf.get("env") or [] if v.get("key") == key), "") + example = "" + d = re.fullmatch(r"(\d{1,5}):(\d{1,5})", default) + if d and int(d[2]) + 100 < low: + example = f", such as {int(d[1]) + 100}:{int(d[2]) + 100}" + advice["warning"] = ( + f"Its recorded {key}, {value}, overlaps Linux's ephemeral range ({low}-{EPHEMERAL_PORTS[1]}), which the " + "redeploy keeps: on its own network every port in it is published, and one a host program holds fails " + f"the start. Before you redeploy, set {key} to a free range below {low} other than the default{example} " + "(a value equal to the default keeps the recorded one), and change the router forward to match." + ) + return advice + + def container_drift( svc: dict[str, Any] | None, shape: dict[str, Any] | None, @@ -192,13 +238,7 @@ def container_drift( found.append( "uses its own network, but should share the host's" if want_host - else "shares the host's network" - + ( - f", where {reach}. Switch isolation on in Fleet, then redeploy {svc.get('name') or 'it'} once to " - "move it onto the fenced bridge" - if gated - else "" - ) + else "shares the host's network" + (f", where {reach}" if gated else "") ) elif not want_host: recorded_ports = None if gated else rec.get("ports") diff --git a/app/isolation_check.py b/app/isolation_check.py index 6cba2cfd..b1bd788d 100644 --- a/app/isolation_check.py +++ b/app/isolation_check.py @@ -405,7 +405,7 @@ def _name(container: Any) -> str: NETNS_CAPABILITIES = frozenset({"NET_ADMIN", "SYS_ADMIN"}) -def host_netns_findings(containers: list[Any]) -> list[dict[str, Any]]: +def host_netns_findings(containers: list[Any], mode: str | None = None) -> list[dict[str, Any]]: """Managed containers holding NET_ADMIN or SYS_ADMIN in the host's network namespace. Such a container can rewrite the host's firewall, the LAN fence included, @@ -431,8 +431,7 @@ def host_netns_findings(containers: list[Any]) -> list[dict[str, Any]]: "detail": ( "These containers hold firewall capabilities in this host's own network namespace, so they can " "rewrite the host's firewall, the LAN fence included, and reach everything the host reaches: " - f"{', '.join(sorted(held))}. Switch isolation on in Fleet, then redeploy them once from the dashboard to " - "move them onto the fenced bridge." + f"{', '.join(sorted(held))}. " + lan_fence.move_advice(mode, "them", "them") ), "docs": ISOLATION_PAGE, } @@ -770,7 +769,7 @@ def _run(client: Any) -> dict[str, Any]: "not_answering_from_host": [], "addresses_left_out": [], "findings": ( - host_netns_findings(every) + host_netns_findings(every, mode) + placement_findings(every, network, mode) + ipv6_findings(client, every, network) ), diff --git a/app/lan_fence.py b/app/lan_fence.py index f27074fa..39c24cd5 100644 --- a/app/lan_fence.py +++ b/app/lan_fence.py @@ -1075,6 +1075,36 @@ def network_for_deploy() -> str | None: return NETWORK_NAME +def move_advice(mode: str | None, service: str = "it", it: str = "it") -> str: + """The one thing to do to move a fence-gated node off the host's network, in this isolation mode. + + ``on`` and ``operator`` take a redeploy alone (switching isolation on is + refused in ``operator`` mode); ``legacy`` needs isolation on first, ``off`` + the setting changed, and Podman cannot be fenced at all. ``None`` is a + worker that reports no mode. + """ + if mode == MODE_ON: + return f"Redeploy {service} once to move {it} onto the fenced bridge." + if mode == MODE_OPERATOR: + return f"Redeploy {service} once to move {it} onto your own network (CASHPILOT_CONTAINER_NETWORK)." + if mode == MODE_LEGACY: + return f"Switch isolation on in Fleet first, then redeploy {service} once to move {it} onto the fenced bridge." + if mode == MODE_OFF: + return ( + f"Set CASHPILOT_ISOLATION=auto on this worker and restart it, then redeploy {service} once to move " + f"{it} onto the fenced bridge." + ) + if mode == MODE_UNSUPPORTED: + return ( + f"{PODMAN_UNSUPPORTED}, so {service} cannot be fenced on this worker. Run {it} on Docker, or on your " + "own network (CASHPILOT_CONTAINER_NETWORK)." + ) + return ( + f"Redeploy {service} once to move {it} onto the fenced bridge; if isolation is not on for this worker, " + "switch it on in Fleet first." + ) + + def mode_problem(mode: str, service: str = "this service") -> str | None: """Why a fence-gated service cannot run in this isolation mode at all, or None. @@ -1086,12 +1116,7 @@ def mode_problem(mode: str, service: str = "this service") -> str | None: """ if mode in (MODE_ON, MODE_OPERATOR): return None - what = { - MODE_LEGACY: f"Switch isolation on in Fleet, then redeploy {service}.", - MODE_OFF: f"Set CASHPILOT_ISOLATION=auto on this worker and restart it, then redeploy {service}.", - MODE_UNSUPPORTED: "Run it on Docker with isolation on, or on your own network (CASHPILOT_CONTAINER_NETWORK).", - }.get(mode, f"Switch isolation on in Fleet, then redeploy {service}.") - return f"isolation is {mode} on this worker, and {service} runs only behind the LAN fence. {what}" + return f"isolation is {mode} on this worker, and {service} runs only behind the LAN fence. {move_advice(mode, service)}" def fence_problem(client: Any, service: str = "this service") -> str | None: diff --git a/app/main.py b/app/main.py index 5d8a8cab..386e6fff 100644 --- a/app/main.py +++ b/app/main.py @@ -279,6 +279,11 @@ async def _get_all_worker_containers(workers: list[dict[str, Any]] | None = None # check (app/drift.py). None from a worker too old to send it. "shape": c.get("shape"), "_arch": sys_info.get("arch"), + # The worker's isolation mode, for the drift advice on + # a fence-gated node still on the host's network. + "_isolation_mode": ( + (sys_info.get("isolation") or {}) if isinstance(sys_info.get("isolation"), dict) else {} + ).get("mode"), "_node": worker_name, "_worker_id": w.get("id"), "_has_docker": worker_has_docker, @@ -2618,8 +2623,8 @@ async def _instance_drift( inst: dict[str, Any], custom_image: str | None, specs: dict[tuple[str, int], dict[str, Any] | None], -) -> tuple[list[str], bool]: - """How this running copy differs from what a redeploy would build, and whether it has a record. +) -> tuple[list[str], bool, dict[str, Any] | None]: + """How this running copy differs from what a redeploy would build, whether it has a record, and the move advice. Never raises: a record that cannot be read leaves the copy judged on the catalog alone, and a check that fails says nothing rather than blanking @@ -2637,10 +2642,11 @@ async def _instance_drift( found = drift.container_drift( svc, inst.get("shape"), recorded, custom_image=custom_image, worker_arch=inst.get("_arch") ) + move = drift.move_advice(svc, inst.get("shape"), recorded, inst.get("_isolation_mode")) if found else None except Exception as exc: logger.warning("Drift check failed for %s on worker %s: %s", slug, wid, exc) - return [], False - return found, recorded is not None + return [], False, None + return found, recorded is not None, move @app.get("/api/services/deployed") @@ -2786,12 +2792,14 @@ async def api_services_deployed(request: Request) -> list[dict[str, Any]]: and inst.get("deployed_by") in constants.CASHPILOT_DEPLOYED and drift_ignored.get(f"{slug}|{inst.get('_worker_id')}") != drift.catalog_fingerprint(svc) ): - drift_found, has_record = await _instance_drift( + drift_found, has_record, move = await _instance_drift( svc, slug, inst, custom_images.get(inst.get("_worker_id")), specs ) if drift_found: detail["drift"] = drift_found detail["drift_has_record"] = has_record + if move: + detail["drift_move"] = move if inst.get("_is_android"): detail["net_tx_24h"] = inst.get("_net_tx_24h", 0) detail["net_rx_24h"] = inst.get("_net_rx_24h", 0) diff --git a/app/orchestrator.py b/app/orchestrator.py index 67a604cd..a58fa2d8 100644 --- a/app/orchestrator.py +++ b/app/orchestrator.py @@ -1244,8 +1244,7 @@ def _deploy_failure(exc: BaseException, restored: str, slug: str = "") -> Contai return ContainerRuntimeError(f"The container could not be created: {exc}. {restored}", 500) -#: Linux's default ephemeral range (net.ipv4.ip_local_port_range). -EPHEMERAL_PORTS = (32768, 60999) +from app.constants import EPHEMERAL_PORTS # noqa: E402 - Linux's default ephemeral range def _ephemeral_hint(reason: str, slug: str = "") -> str: diff --git a/app/static/js/app.js b/app/static/js/app.js index 592b432d..f3ea9646 100644 --- a/app/static/js/app.js +++ b/app/static/js/app.js @@ -1585,16 +1585,23 @@ const CP = (() => { const fix = inst.drift_has_record ? 'Redeploy it here: CashPilot keeps the settings it was deployed with.' : 'CashPilot has no record of how it was deployed, so it cannot rebuild it without losing its settings. Recreate it with the setting above, or remove it and deploy it again from the service page.'; + // A fence-gated node still on the host's network (app/drift.py move_advice): + // the one action for this worker's isolation mode first, then what to fix + // before it, then the reasons. + const move = inst.drift_move; + const why = move + ? `${escapeHtml(move.action)}${move.warning ? ` ${escapeHtml(move.warning)}` : ''}${inst.drift_has_record ? '' : ` ${fix}`}${inst.drift.map(d => `
ยท ${escapeHtml(d)}`).join('')}` + : `container does not match the catalog: ${what}. ${fix}`; let actions = ''; if (_isOwner && !inst.unmanaged) { - actions = inst.drift_has_record && inst.worker_id != null + actions = inst.drift_has_record && inst.worker_id != null && !(move && move.redeploy === false) ? `` : ``; if (inst.worker_id != null) { actions += ``; } } - items.push({ level: 'fail', tag: 'Needs fixing', what: svc.name, where, why: `container does not match the catalog: ${what}. ${fix}`, actions }); + items.push({ level: 'fail', tag: 'Needs fixing', what: svc.name, where, why, actions }); } } diff --git a/docs/isolation.md b/docs/isolation.md index 5cd9bd10..d1b2ea47 100644 --- a/docs/isolation.md +++ b/docs/isolation.md @@ -217,9 +217,11 @@ rules, and keeping other containers off its WebUI is up to those rules too. In `off` and `legacy` mode, and on Podman, the worker does not run the node at all: on Docker's default bridge every other earner could reach its WebUI and change its payout wallet, and nothing would keep it off your LAN. A deploy is -refused before the running node is touched, with the order to follow: switch -isolation on in Fleet, then redeploy Mysterium. A fenced node already running -in such a mode is stopped. +refused before the running node is touched, with what to do in that mode: +in `legacy`, switch isolation on in Fleet first, then redeploy Mysterium; in +`off`, set `CASHPILOT_ISOLATION=auto` and restart the worker first; on Podman +it cannot be fenced. In `on` and operator mode a redeploy alone moves it. A +fenced node already running in such a mode is stopped. **A node from 2.0.6 or earlier keeps running as it is.** A Mysterium container on the host's network, which is what every node deployed through 2.0.6 is, is diff --git a/tests/test_drift.py b/tests/test_drift.py index 4e19a9e8..c2bf17d1 100644 --- a/tests/test_drift.py +++ b/tests/test_drift.py @@ -201,7 +201,7 @@ def test_a_device_the_catalog_dropped_is_drift(self): def test_an_unreadable_record_still_judges_the_catalog(self): """A failed record read must not hide what the catalog alone shows.""" with patch("app.main.database.get_deployment_spec", AsyncMock(side_effect=RuntimeError("db"))): - found, has_record = asyncio.run( + found, has_record, _move = asyncio.run( main._instance_drift( BITPING, "bitping", {"_worker_id": 1, "shape": _shape(_bitping_spec(cap_add=None))}, None, {} ) @@ -256,7 +256,7 @@ def test_the_row_has_a_redeploy_handler(self): assert "tag: 'Needs fixing'" in js -def _deployed_row(svc, shape, recorded, deployed_by="worker", config=None): +def _deployed_row(svc, shape, recorded, deployed_by="worker", config=None, isolation_mode=None): from fastapi.testclient import TestClient container = { @@ -272,7 +272,13 @@ def _deployed_row(svc, shape, recorded, deployed_by="worker", config=None): "id": 1, "name": "w1", "status": "online", - "system_info": json.dumps({"docker_available": True, "arch": "x86_64"}), + "system_info": json.dumps( + { + "docker_available": True, + "arch": "x86_64", + **({"isolation": {"mode": isolation_mode}} if isolation_mode else {}), + } + ), "containers": json.dumps([container]), "apps": "[]", } diff --git a/tests/test_mysterium_fenced.py b/tests/test_mysterium_fenced.py index 78728deb..293427a3 100644 --- a/tests/test_mysterium_fenced.py +++ b/tests/test_mysterium_fenced.py @@ -16,6 +16,7 @@ from __future__ import annotations +import contextlib import json import os from contextlib import asynccontextmanager @@ -445,7 +446,7 @@ def test_net_admin_in_the_host_netns_is_an_exposure(self): found = isolation_check.host_netns_findings([_c("cashpilot-mysterium", "host", ["NET_ADMIN", "SETUID"])]) assert found and found[0]["exposure"] is True assert "cashpilot-mysterium (NET_ADMIN)" in found[0]["detail"] - assert "Switch isolation on in Fleet, then redeploy them once" in found[0]["detail"] + assert "Redeploy them once to move them onto the fenced bridge" in found[0]["detail"] def test_control_net_admin_in_its_own_namespace_is_not(self): assert ( @@ -791,13 +792,15 @@ def test_a_redeploy_in_legacy_mode_is_refused_and_says_what_to_do(self, markers) with pytest.raises(orchestrator.ContainerNetworkError) as refused: _raw_deploy(old=old, mode=lan_fence.MODE_LEGACY) # The catalog's name, the one on the dashboard card the owner redeploys from. - assert f"Switch isolation on in Fleet, then redeploy {MYST['name']}." in str(refused.value) + assert f"Switch isolation on in Fleet first, then redeploy {MYST['name']} once" in str(refused.value) old.stop.assert_not_called() old.remove.assert_not_called() def test_drift_still_flags_it_and_gives_the_order(self): found = " ".join(drift.container_drift(MYST, _shape(network_mode="host"), HOST_RECORD)) - assert f"Switch isolation on in Fleet, then redeploy {MYST['name']} once" in found + assert "shares the host's network" in found + move = drift.move_advice(MYST, _shape(network_mode="host"), HOST_RECORD, lan_fence.MODE_LEGACY) + assert move["action"].startswith(f"Switch isolation on in Fleet first, then redeploy {MYST['name']} once") def test_control_a_fence_gated_one_in_legacy_mode_is_stopped(self, markers): node = _node(network_mode="bridge") @@ -981,15 +984,17 @@ class TestTheMessagesNameTheService: def test_the_mode_refusal_names_the_service_it_is_given(self): problem = lan_fence.mode_problem(lan_fence.MODE_LEGACY, "Other") assert "Other runs only behind the LAN fence" in problem - assert problem.endswith("Switch isolation on in Fleet, then redeploy Other.") + assert problem.endswith( + "Switch isolation on in Fleet first, then redeploy Other once to move it onto the fenced bridge." + ) def test_drift_names_the_service_and_claims_no_capability_it_lacks(self): other = {**MYST, "name": "Other", "docker": {**MYST["docker"], "cap_add": []}} shape = _shape(network_mode="host") shape["cap_add"] = [] found = " ".join(drift.container_drift(other, shape, None)) - assert "redeploy Other once" in found assert "NET_ADMIN" not in found + assert "Redeploy Other once" in drift.move_advice(other, shape, None, lan_fence.MODE_ON)["action"] def test_drift_flags_a_node_that_publishes_its_webui_but_not_its_range(self): found = drift.container_drift(MYST, _shape(udp=False), None) @@ -1622,3 +1627,90 @@ def test_control_a_taken_port_below_it_gets_no_hint(self): def test_control_another_refusal_gets_no_hint(self): assert orchestrator._ephemeral_hint("no such device /dev/net/tun") == "" + + +# --------------------------------------------------------------------------- +# The move off the host's network: one action for the worker's mode, then why +# --------------------------------------------------------------------------- + +NEW_RECORD = {**HOST_RECORD, "env": {"UI_ADDRESS": "127.0.0.1", "UDP_PORTS": "23000:23029"}} + + +class TestTheMoveAdvice: + @pytest.mark.parametrize( + ("mode", "says", "redeploy"), + [ + (lan_fence.MODE_ON, f"Redeploy {MYST['name']} once to move it onto the fenced bridge.", True), + (lan_fence.MODE_OPERATOR, f"Redeploy {MYST['name']} once to move it onto your own network", True), + (lan_fence.MODE_LEGACY, "Switch isolation on in Fleet first, then redeploy", False), + (lan_fence.MODE_OFF, "Set CASHPILOT_ISOLATION=auto on this worker and restart it, then redeploy", False), + (lan_fence.MODE_UNSUPPORTED, "cannot be fenced on this worker", False), + ], + ) + def test_the_action_fits_the_workers_mode(self, mode, says, redeploy): + move = drift.move_advice(MYST, _shape(network_mode="host"), NEW_RECORD, mode) + assert move["action"].startswith(says) or says in move["action"] + assert move["redeploy"] is redeploy + if mode in (lan_fence.MODE_ON, lan_fence.MODE_OPERATOR): + # Switching isolation on is refused in operator mode: a redeploy alone is the way. + assert "Switch isolation" not in move["action"] and "CASHPILOT_ISOLATION" not in move["action"] + + def test_the_drift_reason_carries_no_action_of_its_own(self): + found = drift.container_drift(MYST, _shape(network_mode="host"), HOST_RECORD) + host = next(f for f in found if "host's network" in f) + assert host == "shares the host's network, where its NET_ADMIN reaches the host's firewall" + + def test_a_recorded_range_in_the_ephemeral_range_is_flagged_before_the_redeploy(self): + move = drift.move_advice(MYST, _shape(network_mode="host"), HOST_RECORD, lan_fence.MODE_ON) + warning = move["warning"] + assert "56000:56100" in warning and "32768-60999" in warning + assert "Before you redeploy" in warning and "below 32768 other than the default" in warning + assert "23100:23129" in warning + + def test_control_a_range_below_it_or_none_on_record_is_not(self): + for record in (NEW_RECORD, None, {**HOST_RECORD, "env": {}}): + assert drift.move_advice(MYST, _shape(network_mode="host"), record, lan_fence.MODE_ON)["warning"] is None + + def test_control_a_node_already_off_the_host_network_gets_no_advice(self): + assert drift.move_advice(MYST, _shape(), NEW_RECORD, lan_fence.MODE_ON) is None + + @pytest.mark.parametrize( + ("mode", "switch"), + [(lan_fence.MODE_ON, False), (lan_fence.MODE_OPERATOR, False), (lan_fence.MODE_LEGACY, True)], + ) + def test_the_isolation_check_gives_the_same_action(self, mode, switch): + found = isolation_check.host_netns_findings([_c("cashpilot-mysterium", "host", ["NET_ADMIN"])], mode) + assert ("Switch isolation on in Fleet first" in found[0]["detail"]) is switch + assert "Redeploy them once" in found[0]["detail"] or "redeploy them once" in found[0]["detail"] + + def test_the_mode_refusal_uses_the_same_advice(self): + assert lan_fence.move_advice(lan_fence.MODE_OFF, "Other") in lan_fence.mode_problem(lan_fence.MODE_OFF, "Other") + unknown = lan_fence.mode_problem("weird", "Other") + assert "Redeploy Other once" in unknown and "switch it on in Fleet first" in unknown + + def test_the_dashboard_gets_the_workers_mode_and_the_warning(self): + from tests.test_drift import _deployed_row + + row = _deployed_row(MYST, _shape(network_mode="host"), recorded=HOST_RECORD, isolation_mode="operator") + move = row["instance_details"][0]["drift_move"] + assert move["action"].startswith(f"Redeploy {MYST['name']} once to move it onto your own network") + assert "56000:56100" in move["warning"] + + def test_the_dashboard_leads_with_the_action_and_offers_redeploy_only_where_it_works(self): + js = (ROOT / "app" / "static" / "js" / "app.js").read_text(encoding="utf-8") + assert "${escapeHtml(move.action)}${move.warning ? ` ${escapeHtml(move.warning)}` : ''}" in js + assert "!(move && move.redeploy === false)" in js + + +def test_the_isolation_check_passes_the_workers_mode_to_the_finding(): + client = MagicMock() + client.containers.list.return_value = [] + with ( + patch.object(lan_fence, "effective_network", return_value=(lan_fence.MODE_OPERATOR, "x")), + patch.object(isolation_check, "host_netns_findings", return_value=[]) as spy, + patch.object(isolation_check, "placement_findings", return_value=[]), + patch.object(isolation_check, "ipv6_findings", return_value=[]), + contextlib.suppress(Exception), + ): + isolation_check._run(client) + spy.assert_called_once_with([], lan_fence.MODE_OPERATOR) From 6213d8a58d1364db89becb609d3afb7dd930cf1e Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 8 Nov 2026 18:02:07 +0100 Subject: [PATCH 2/3] fix(drift): offer Redeploy only where the worker reports a mode it works in --- app/drift.py | 6 ++++-- tests/test_mysterium_fenced.py | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/app/drift.py b/app/drift.py index 830c479c..518cd64f 100644 --- a/app/drift.py +++ b/app/drift.py @@ -137,7 +137,7 @@ def move_advice( """For a fence-gated container still on the host's network: the one action, and what to fix first. ``action`` depends on the worker's isolation mode (lan_fence.move_advice); - ``redeploy`` says whether a redeploy alone does it there; ``warning`` is set + ``redeploy`` says whether the worker reports a mode where a redeploy alone does it; ``warning`` is set when the recorded UDP range, which the redeploy keeps, overlaps Linux's ephemeral range, where one port a host program holds fails the start. None for any other container. @@ -150,7 +150,9 @@ def move_advice( name = svc.get("name") or "it" advice: dict[str, Any] = { "action": lan_fence.move_advice(isolation_mode, name), - "redeploy": isolation_mode in (lan_fence.MODE_ON, lan_fence.MODE_OPERATOR, None), + # Only where the worker says a redeploy alone works: one that reports no + # mode yet may be in legacy, off or on Podman, where it is refused. + "redeploy": isolation_mode in (lan_fence.MODE_ON, lan_fence.MODE_OPERATOR), "warning": None, } rng = docker_conf.get("udp_port_range") diff --git a/tests/test_mysterium_fenced.py b/tests/test_mysterium_fenced.py index 293427a3..3c2a5a85 100644 --- a/tests/test_mysterium_fenced.py +++ b/tests/test_mysterium_fenced.py @@ -1645,6 +1645,7 @@ class TestTheMoveAdvice: (lan_fence.MODE_LEGACY, "Switch isolation on in Fleet first, then redeploy", False), (lan_fence.MODE_OFF, "Set CASHPILOT_ISOLATION=auto on this worker and restart it, then redeploy", False), (lan_fence.MODE_UNSUPPORTED, "cannot be fenced on this worker", False), + (None, "if isolation is not on for this worker, switch it on in Fleet first", False), ], ) def test_the_action_fits_the_workers_mode(self, mode, says, redeploy): From 4e6c8b517fe24f26c31a2cd258d3628b8e7e88e5 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 8 Nov 2026 18:30:53 +0100 Subject: [PATCH 3/3] fix(mysterium): offer reserving the UDP range on the host, keep a hold a refused deploy popped, and record one node per IP The ephemeral-range warning now gives the other route: keep the range and reserve it with net.ipv4.ip_local_reserved_ports, persisted in sysctl.d. The worker reads its own network namespace, not the host's, so it cannot see the reservation; the warning says a box that reserved it can redeploy. A deploy refused because the running node's identity could not be read dropped a mismatch hold kept only in memory. It is restored now, like the stop and rename refusals already did. MystNodes allows one node per public IP; the catalog said unlimited. --- UPGRADING.md | 7 ++++++ app/drift.py | 11 +++++++- app/orchestrator.py | 2 ++ docs/guides/mysterium.md | 19 ++++++++++++-- docs/research/per-ip-device-limits.md | 1 + services/bandwidth/mysterium.yml | 4 ++- tests/test_mysterium_fenced.py | 36 +++++++++++++++++++++++++++ tests/test_preflight.py | 10 +++++++- 8 files changed, 85 insertions(+), 5 deletions(-) diff --git a/UPGRADING.md b/UPGRADING.md index 44edcec5..28df6c2b 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -52,6 +52,13 @@ drift, and **Check isolation** fails on it. (`23000:23029`) reads as a field left as it was, so the range on record would stay; to take the default itself, remove the service from that machine (its data volume stays) and deploy again. + Or keep the range and reserve it on the host instead, with no router + change: `sudo sysctl -w net.ipv4.ip_local_reserved_ports=56000-56100` (use + your range, with a dash, added to any ranges already listed there), and the + line `net.ipv4.ip_local_reserved_ports = 56000-56100` in a file under + `/etc/sysctl.d/` so it survives a reboot. The worker cannot see the host's + reservation, so Fleet keeps warning about the range; once it is reserved, + redeploy anyway. The worker publishes the range 1:1 through Docker, and a node's range on record (`56000:56100` through 2.0.6, `56000:56029` in 2.0.7 and 2.1.0) sits in Linux's ephemeral range, 32768-60999, where any program on the host can diff --git a/app/drift.py b/app/drift.py index 518cd64f..d9b70ad3 100644 --- a/app/drift.py +++ b/app/drift.py @@ -128,6 +128,11 @@ def _names(values: set[str]) -> str: return ", ".join(sorted(values)) +#: Per network namespace: the worker, on its own bridge, reads its own value and +#: never the host's, so the warning names the reservation instead of checking it. +RESERVED_PORTS_SYSCTL = "net.ipv4.ip_local_reserved_ports" + + def move_advice( svc: dict[str, Any] | None, shape: dict[str, Any] | None, @@ -170,7 +175,11 @@ def move_advice( f"Its recorded {key}, {value}, overlaps Linux's ephemeral range ({low}-{EPHEMERAL_PORTS[1]}), which the " "redeploy keeps: on its own network every port in it is published, and one a host program holds fails " f"the start. Before you redeploy, set {key} to a free range below {low} other than the default{example} " - "(a value equal to the default keeps the recorded one), and change the router forward to match." + "(a value equal to the default keeps the recorded one), and change the router forward to match. " + f"Or keep this range and reserve it on the host, so no program there takes one of its ports: run " + f"sysctl -w {RESERVED_PORTS_SYSCTL}={m[1]}-{m[2]} (add it to any ranges already listed there, " + f"comma-separated), and put the line {RESERVED_PORTS_SYSCTL} = {m[1]}-{m[2]} in a file under " + "/etc/sysctl.d/ so it survives a reboot. If you've reserved this range on the host, you can redeploy." ) return advice diff --git a/app/orchestrator.py b/app/orchestrator.py index a58fa2d8..9f98819c 100644 --- a/app/orchestrator.py +++ b/app/orchestrator.py @@ -917,6 +917,8 @@ def deploy_raw( identity = read_node_identity(old, slug) if running else None if running and identity is None and _reads_identity(slug): # Without it the new node could run on another keystore unchecked. + # Nothing is replaced, so a hold popped above comes back. + _restore_identity(name, identity_before, held_before) raise ContainerRuntimeError( f"{name} was not replaced: its identity could not be read from the running node, so the new " "one could not be checked against it. The running container was not touched. Try again in a " diff --git a/docs/guides/mysterium.md b/docs/guides/mysterium.md index c695a796..2ba47f75 100644 --- a/docs/guides/mysterium.md +++ b/docs/guides/mysterium.md @@ -20,9 +20,9 @@ MystNodes (Mysterium Network) is a decentralized VPN and proxy network built on > Earnings in MYST tokens. Residential IPs earn significantly more. Node WebUI at port 4449 for management, published on the machine it runs on only (127.0.0.1); from another machine open an SSH tunnel first (`ssh -L 4449:127.0.0.1:4449 `, then browse `http://127.0.0.1:4449`), or put a reverse proxy in front (see **WebUI address** below). VPS accepted. Important: after first run, set your beneficiary (settlement) wallet via the node WebUI or CLI to match your mystnodes.com account -- this links on-chain earnings to your cloud dashboard. -> **One node per public IP.** Mysterium strictly enforces one active node per public IP address. Additional nodes on the same IP show as offline and earn nothing. Do not run on a phone if a Docker node is already running on the same network. Use separate public IPs (e.g. dual WAN, different locations) for additional nodes. +> **One node per public IP.** Mysterium strictly enforces one active node per public IP address. Additional nodes on the same IP show as offline and earn nothing. Do not run on a phone if a Docker node is already running on the same network. Use separate public IPs (e.g. dual WAN, different locations) for additional nodes. Fleet warns before a second node is deployed behind an address that already has one. -> **Port forwarding recommended.** Forward the node's UDP range to maximize earnings; see **Port forwarding** below. Without it many VPN and proxy sessions fail to connect, which cuts income. The Docker image runs with `--net host` and the `NET_ADMIN`, `SETUID` and `SETGID` capabilities. +> **Port forwarding recommended.** Forward the node's UDP range to maximize earnings; see **Port forwarding** below. Without it many VPN and proxy sessions fail to connect, which cuts income. The node runs behind CashPilot's LAN fence, on its fenced bridge or your own network (see [Isolation](../isolation.md)), with the `NET_ADMIN`, `SETUID` and `SETGID` capabilities. ## Cashing out @@ -104,6 +104,21 @@ as a field left as it was, so the recorded range would stay. To take the default itself, remove the service from that machine (its data volume stays) and deploy again. +Or keep the range on record and reserve it on the host, so no program there +can take one of its ports, then redeploy without touching the router. For +`56000:56100`, on the machine running the node: + +```bash +sudo sysctl -w net.ipv4.ip_local_reserved_ports=56000-56100 +echo 'net.ipv4.ip_local_reserved_ports = 56000-56100' | sudo tee /etc/sysctl.d/90-mysterium-ports.conf +``` + +If `sysctl net.ipv4.ip_local_reserved_ports` already lists ranges, add yours +to them, comma-separated, in both places. The file under `/etc/sysctl.d/` +keeps it after a reboot. The worker cannot read the host's setting (it runs +on its own network, which has its own copy), so the Fleet warning about the +range stays after you reserve it; once the range is reserved, redeploy anyway. + **Check it.** After a few sessions, on the machine running the node: ```bash diff --git a/docs/research/per-ip-device-limits.md b/docs/research/per-ip-device-limits.md index df50d560..1aa30c06 100644 --- a/docs/research/per-ip-device-limits.md +++ b/docs/research/per-ip-device-limits.md @@ -26,6 +26,7 @@ reviewer with a link opens the page and sees it; without one, the error survives | iproyal (Pawns.app) | **1** | hard limit, in binding Terms | *"Pawns.app limits the number of devices per single IP address to one (1)."* | | spide | **1** | hard limit, in binding Terms | *"Spide Network limits the number of devices per single IP address to one."* Restrictions also forbid *"connect[ing] more than the allowed number of devices"*. The same paragraph adds that traffic through one IP *"will be distributed between devices utilizing the same IP address"*. | | earnfm | **1** | help-centre guidance, not a Terms clause | *"One Device Per Network: Each device **should** be connected to a unique IP or network."* Phrased as guidance; no device-per-IP clause found in the Terms, so treat enforcement as unproven. | +| mysterium (MystNodes) | **1** | hard limit, enforced | *"You can only run one MystNodes per public IP address"*; *"Each node needs its own unique public IP."* Extra nodes on one IP show offline. (earlier: *"We do not support multiple devices under a single IP address"*, ) | | ebesucher | **1** | hard limit | Multi-device is allowed only if *"each device ... has to be connected to the Internet through a unique IP address."* | ## Documented as sharing, with no number diff --git a/services/bandwidth/mysterium.yml b/services/bandwidth/mysterium.yml index 8030cf6b..5ffded44 100644 --- a/services/bandwidth/mysterium.yml +++ b/services/bandwidth/mysterium.yml @@ -163,7 +163,9 @@ docker: requirements: residential_ip: false - devices_per_ip: 0 + # MystNodes help centre: one node per public IP; the others show offline + # (docs/research/per-ip-device-limits.md). + devices_per_ip: 1 min_bandwidth: "" gpu: false min_storage: "" diff --git a/tests/test_mysterium_fenced.py b/tests/test_mysterium_fenced.py index 3c2a5a85..a16e0b8c 100644 --- a/tests/test_mysterium_fenced.py +++ b/tests/test_mysterium_fenced.py @@ -1190,6 +1190,17 @@ def test_a_running_node_whose_identity_cannot_be_read_is_not_replaced(self, mark old.stop.assert_not_called() old.remove.assert_not_called() + def test_the_refusal_keeps_a_hold_kept_only_in_memory(self, markers): + # No record on disk: the hold could not be written, so memory is all there is. + orchestrator._MISMATCH_HELD["cashpilot-mysterium"] = "its identity is 0xb, not 0xa" + old = _node() + old.exec_run.return_value = (1, b"") + old.attrs["Mounts"] = [] + with pytest.raises(orchestrator.ContainerRuntimeError, match="not touched"): + _raw_deploy(old=old) + assert orchestrator._MISMATCH_HELD.get("cashpilot-mysterium") == "its identity is 0xb, not 0xa" + assert "0xb" in orchestrator.identity_problem(_node(status="exited")) + def test_control_a_stopped_node_is_replaced_without_the_check(self, markers): old = _node(status="exited") old.exec_run.return_value = (1, b"") @@ -1668,6 +1679,31 @@ def test_a_recorded_range_in_the_ephemeral_range_is_flagged_before_the_redeploy( assert "Before you redeploy" in warning and "below 32768 other than the default" in warning assert "23100:23129" in warning + def test_the_warning_offers_reserving_the_range_on_the_host_instead(self): + warning = drift.move_advice(MYST, _shape(network_mode="host"), HOST_RECORD, lan_fence.MODE_ON)["warning"] + assert "keep this range and reserve it on the host" in warning + assert "sysctl -w net.ipv4.ip_local_reserved_ports=56000-56100" in warning + assert "net.ipv4.ip_local_reserved_ports = 56000-56100 in a file under /etc/sysctl.d/" in warning + assert "add it to any ranges already listed there" in warning + # The worker cannot read the host's reservation, so it says when to ignore the warning. + assert warning.endswith("If you've reserved this range on the host, you can redeploy.") + + def test_the_reservation_names_the_recorded_range_not_a_fixed_one(self): + record = {**HOST_RECORD, "env": {"UDP_PORTS": "40000:40020"}} + warning = drift.move_advice(MYST, _shape(network_mode="host"), record, lan_fence.MODE_ON)["warning"] + assert "ip_local_reserved_ports=40000-40020" in warning and "56000" not in warning + + def test_the_guide_and_the_upgrade_notes_give_the_same_reservation(self): + for doc in ("docs/guides/mysterium.md", "UPGRADING.md"): + text = (ROOT / doc).read_text(encoding="utf-8") + assert "sysctl -w net.ipv4.ip_local_reserved_ports=56000-56100" in text, doc + assert "net.ipv4.ip_local_reserved_ports = 56000-56100" in text, doc + assert "/etc/sysctl.d/" in text, doc + + def test_the_guide_no_longer_says_the_node_runs_on_the_host_network(self): + guide = (ROOT / "docs" / "guides" / "mysterium.md").read_text(encoding="utf-8") + assert "--net host" not in guide + def test_control_a_range_below_it_or_none_on_record_is_not(self): for record in (NEW_RECORD, None, {**HOST_RECORD, "env": {}}): assert drift.move_advice(MYST, _shape(network_mode="host"), record, lan_fence.MODE_ON)["warning"] is None diff --git a/tests/test_preflight.py b/tests/test_preflight.py index 3f64572c..6f58d989 100644 --- a/tests/test_preflight.py +++ b/tests/test_preflight.py @@ -221,7 +221,7 @@ def test_the_flag_is_not_set_on_services_merely_wanting_a_residential_ip(self): class TestSourcedPerIpLimits: """Values must trace to a provider statement; see docs/research/per-ip-device-limits.md.""" - @pytest.mark.parametrize("slug", ["honeygain", "iproyal", "spide", "earnfm", "ebesucher"]) + @pytest.mark.parametrize("slug", ["honeygain", "iproyal", "spide", "earnfm", "ebesucher", "mysterium"]) def test_the_documented_services_declare_one_per_ip(self, slug): """spide was MISSING from this list, and that is how the bug shipped. @@ -269,6 +269,14 @@ def test_services_with_no_provider_source_stay_absent(self, slug): assert svc is not None, f"{slug} is not in the catalog" assert "devices_per_ip" not in (svc.get("requirements") or {}) + def test_a_second_mystnodes_on_one_machine_earns_nothing(self): + """MystNodes shows every node but one per public IP offline; it read as unlimited.""" + from app import catalog + + out = preflight.assess(catalog.get_service("mysterium"), already_deployed_slugs={"mysterium"}) + assert out["verdict"] == preflight.EARNS_NOTHING + assert "one device per IP" in " ".join(f["message"] for f in out["findings"]) + def test_a_second_device_behind_one_ip_now_gets_the_strongest_verdict(self): """The point of the bead: the verdict was weak exactly where the mistake is most likely, because the limit was never recorded."""