From e0046974f6d58d30ca555dca819a892b3b14d3df Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 8 Nov 2026 18:30:53 +0100 Subject: [PATCH 1/4] fix(mysterium): offer reserving the UDP range on the host, keep a hold a refused deploy popped, and record one node per IP The ephemeral-range warning now gives the other route: keep the range and reserve it with net.ipv4.ip_local_reserved_ports, persisted in sysctl.d. The worker reads its own network namespace, not the host's, so it cannot see the reservation; the warning says a box that reserved it can redeploy. A deploy refused because the running node's identity could not be read dropped a mismatch hold kept only in memory. It is restored now, like the stop and rename refusals already did. MystNodes allows one node per public IP; the catalog said unlimited. --- UPGRADING.md | 7 ++++++ app/drift.py | 11 +++++++- app/orchestrator.py | 2 ++ docs/guides/mysterium.md | 19 ++++++++++++-- docs/research/per-ip-device-limits.md | 1 + services/bandwidth/mysterium.yml | 4 ++- tests/test_mysterium_fenced.py | 36 +++++++++++++++++++++++++++ tests/test_preflight.py | 10 +++++++- 8 files changed, 85 insertions(+), 5 deletions(-) diff --git a/UPGRADING.md b/UPGRADING.md index 44edcec5..28df6c2b 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -52,6 +52,13 @@ drift, and **Check isolation** fails on it. (`23000:23029`) reads as a field left as it was, so the range on record would stay; to take the default itself, remove the service from that machine (its data volume stays) and deploy again. + Or keep the range and reserve it on the host instead, with no router + change: `sudo sysctl -w net.ipv4.ip_local_reserved_ports=56000-56100` (use + your range, with a dash, added to any ranges already listed there), and the + line `net.ipv4.ip_local_reserved_ports = 56000-56100` in a file under + `/etc/sysctl.d/` so it survives a reboot. The worker cannot see the host's + reservation, so Fleet keeps warning about the range; once it is reserved, + redeploy anyway. The worker publishes the range 1:1 through Docker, and a node's range on record (`56000:56100` through 2.0.6, `56000:56029` in 2.0.7 and 2.1.0) sits in Linux's ephemeral range, 32768-60999, where any program on the host can diff --git a/app/drift.py b/app/drift.py index 518cd64f..d9b70ad3 100644 --- a/app/drift.py +++ b/app/drift.py @@ -128,6 +128,11 @@ def _names(values: set[str]) -> str: return ", ".join(sorted(values)) +#: Per network namespace: the worker, on its own bridge, reads its own value and +#: never the host's, so the warning names the reservation instead of checking it. +RESERVED_PORTS_SYSCTL = "net.ipv4.ip_local_reserved_ports" + + def move_advice( svc: dict[str, Any] | None, shape: dict[str, Any] | None, @@ -170,7 +175,11 @@ def move_advice( f"Its recorded {key}, {value}, overlaps Linux's ephemeral range ({low}-{EPHEMERAL_PORTS[1]}), which the " "redeploy keeps: on its own network every port in it is published, and one a host program holds fails " f"the start. Before you redeploy, set {key} to a free range below {low} other than the default{example} " - "(a value equal to the default keeps the recorded one), and change the router forward to match." + "(a value equal to the default keeps the recorded one), and change the router forward to match. " + f"Or keep this range and reserve it on the host, so no program there takes one of its ports: run " + f"sysctl -w {RESERVED_PORTS_SYSCTL}={m[1]}-{m[2]} (add it to any ranges already listed there, " + f"comma-separated), and put the line {RESERVED_PORTS_SYSCTL} = {m[1]}-{m[2]} in a file under " + "/etc/sysctl.d/ so it survives a reboot. If you've reserved this range on the host, you can redeploy." ) return advice diff --git a/app/orchestrator.py b/app/orchestrator.py index a58fa2d8..9f98819c 100644 --- a/app/orchestrator.py +++ b/app/orchestrator.py @@ -917,6 +917,8 @@ def deploy_raw( identity = read_node_identity(old, slug) if running else None if running and identity is None and _reads_identity(slug): # Without it the new node could run on another keystore unchecked. + # Nothing is replaced, so a hold popped above comes back. + _restore_identity(name, identity_before, held_before) raise ContainerRuntimeError( f"{name} was not replaced: its identity could not be read from the running node, so the new " "one could not be checked against it. The running container was not touched. Try again in a " diff --git a/docs/guides/mysterium.md b/docs/guides/mysterium.md index c695a796..2ba47f75 100644 --- a/docs/guides/mysterium.md +++ b/docs/guides/mysterium.md @@ -20,9 +20,9 @@ MystNodes (Mysterium Network) is a decentralized VPN and proxy network built on > Earnings in MYST tokens. Residential IPs earn significantly more. Node WebUI at port 4449 for management, published on the machine it runs on only (127.0.0.1); from another machine open an SSH tunnel first (`ssh -L 4449:127.0.0.1:4449 `, then browse `http://127.0.0.1:4449`), or put a reverse proxy in front (see **WebUI address** below). VPS accepted. Important: after first run, set your beneficiary (settlement) wallet via the node WebUI or CLI to match your mystnodes.com account -- this links on-chain earnings to your cloud dashboard. -> **One node per public IP.** Mysterium strictly enforces one active node per public IP address. Additional nodes on the same IP show as offline and earn nothing. Do not run on a phone if a Docker node is already running on the same network. Use separate public IPs (e.g. dual WAN, different locations) for additional nodes. +> **One node per public IP.** Mysterium strictly enforces one active node per public IP address. Additional nodes on the same IP show as offline and earn nothing. Do not run on a phone if a Docker node is already running on the same network. Use separate public IPs (e.g. dual WAN, different locations) for additional nodes. Fleet warns before a second node is deployed behind an address that already has one. -> **Port forwarding recommended.** Forward the node's UDP range to maximize earnings; see **Port forwarding** below. Without it many VPN and proxy sessions fail to connect, which cuts income. The Docker image runs with `--net host` and the `NET_ADMIN`, `SETUID` and `SETGID` capabilities. +> **Port forwarding recommended.** Forward the node's UDP range to maximize earnings; see **Port forwarding** below. Without it many VPN and proxy sessions fail to connect, which cuts income. The node runs behind CashPilot's LAN fence, on its fenced bridge or your own network (see [Isolation](../isolation.md)), with the `NET_ADMIN`, `SETUID` and `SETGID` capabilities. ## Cashing out @@ -104,6 +104,21 @@ as a field left as it was, so the recorded range would stay. To take the default itself, remove the service from that machine (its data volume stays) and deploy again. +Or keep the range on record and reserve it on the host, so no program there +can take one of its ports, then redeploy without touching the router. For +`56000:56100`, on the machine running the node: + +```bash +sudo sysctl -w net.ipv4.ip_local_reserved_ports=56000-56100 +echo 'net.ipv4.ip_local_reserved_ports = 56000-56100' | sudo tee /etc/sysctl.d/90-mysterium-ports.conf +``` + +If `sysctl net.ipv4.ip_local_reserved_ports` already lists ranges, add yours +to them, comma-separated, in both places. The file under `/etc/sysctl.d/` +keeps it after a reboot. The worker cannot read the host's setting (it runs +on its own network, which has its own copy), so the Fleet warning about the +range stays after you reserve it; once the range is reserved, redeploy anyway. + **Check it.** After a few sessions, on the machine running the node: ```bash diff --git a/docs/research/per-ip-device-limits.md b/docs/research/per-ip-device-limits.md index df50d560..1aa30c06 100644 --- a/docs/research/per-ip-device-limits.md +++ b/docs/research/per-ip-device-limits.md @@ -26,6 +26,7 @@ reviewer with a link opens the page and sees it; without one, the error survives | iproyal (Pawns.app) | **1** | hard limit, in binding Terms | *"Pawns.app limits the number of devices per single IP address to one (1)."* | | spide | **1** | hard limit, in binding Terms | *"Spide Network limits the number of devices per single IP address to one."* Restrictions also forbid *"connect[ing] more than the allowed number of devices"*. The same paragraph adds that traffic through one IP *"will be distributed between devices utilizing the same IP address"*. | | earnfm | **1** | help-centre guidance, not a Terms clause | *"One Device Per Network: Each device **should** be connected to a unique IP or network."* Phrased as guidance; no device-per-IP clause found in the Terms, so treat enforcement as unproven. | +| mysterium (MystNodes) | **1** | hard limit, enforced | *"You can only run one MystNodes per public IP address"*; *"Each node needs its own unique public IP."* Extra nodes on one IP show offline. (earlier: *"We do not support multiple devices under a single IP address"*, ) | | ebesucher | **1** | hard limit | Multi-device is allowed only if *"each device ... has to be connected to the Internet through a unique IP address."* | ## Documented as sharing, with no number diff --git a/services/bandwidth/mysterium.yml b/services/bandwidth/mysterium.yml index 8030cf6b..5ffded44 100644 --- a/services/bandwidth/mysterium.yml +++ b/services/bandwidth/mysterium.yml @@ -163,7 +163,9 @@ docker: requirements: residential_ip: false - devices_per_ip: 0 + # MystNodes help centre: one node per public IP; the others show offline + # (docs/research/per-ip-device-limits.md). + devices_per_ip: 1 min_bandwidth: "" gpu: false min_storage: "" diff --git a/tests/test_mysterium_fenced.py b/tests/test_mysterium_fenced.py index 3c2a5a85..a16e0b8c 100644 --- a/tests/test_mysterium_fenced.py +++ b/tests/test_mysterium_fenced.py @@ -1190,6 +1190,17 @@ def test_a_running_node_whose_identity_cannot_be_read_is_not_replaced(self, mark old.stop.assert_not_called() old.remove.assert_not_called() + def test_the_refusal_keeps_a_hold_kept_only_in_memory(self, markers): + # No record on disk: the hold could not be written, so memory is all there is. + orchestrator._MISMATCH_HELD["cashpilot-mysterium"] = "its identity is 0xb, not 0xa" + old = _node() + old.exec_run.return_value = (1, b"") + old.attrs["Mounts"] = [] + with pytest.raises(orchestrator.ContainerRuntimeError, match="not touched"): + _raw_deploy(old=old) + assert orchestrator._MISMATCH_HELD.get("cashpilot-mysterium") == "its identity is 0xb, not 0xa" + assert "0xb" in orchestrator.identity_problem(_node(status="exited")) + def test_control_a_stopped_node_is_replaced_without_the_check(self, markers): old = _node(status="exited") old.exec_run.return_value = (1, b"") @@ -1668,6 +1679,31 @@ def test_a_recorded_range_in_the_ephemeral_range_is_flagged_before_the_redeploy( assert "Before you redeploy" in warning and "below 32768 other than the default" in warning assert "23100:23129" in warning + def test_the_warning_offers_reserving_the_range_on_the_host_instead(self): + warning = drift.move_advice(MYST, _shape(network_mode="host"), HOST_RECORD, lan_fence.MODE_ON)["warning"] + assert "keep this range and reserve it on the host" in warning + assert "sysctl -w net.ipv4.ip_local_reserved_ports=56000-56100" in warning + assert "net.ipv4.ip_local_reserved_ports = 56000-56100 in a file under /etc/sysctl.d/" in warning + assert "add it to any ranges already listed there" in warning + # The worker cannot read the host's reservation, so it says when to ignore the warning. + assert warning.endswith("If you've reserved this range on the host, you can redeploy.") + + def test_the_reservation_names_the_recorded_range_not_a_fixed_one(self): + record = {**HOST_RECORD, "env": {"UDP_PORTS": "40000:40020"}} + warning = drift.move_advice(MYST, _shape(network_mode="host"), record, lan_fence.MODE_ON)["warning"] + assert "ip_local_reserved_ports=40000-40020" in warning and "56000" not in warning + + def test_the_guide_and_the_upgrade_notes_give_the_same_reservation(self): + for doc in ("docs/guides/mysterium.md", "UPGRADING.md"): + text = (ROOT / doc).read_text(encoding="utf-8") + assert "sysctl -w net.ipv4.ip_local_reserved_ports=56000-56100" in text, doc + assert "net.ipv4.ip_local_reserved_ports = 56000-56100" in text, doc + assert "/etc/sysctl.d/" in text, doc + + def test_the_guide_no_longer_says_the_node_runs_on_the_host_network(self): + guide = (ROOT / "docs" / "guides" / "mysterium.md").read_text(encoding="utf-8") + assert "--net host" not in guide + def test_control_a_range_below_it_or_none_on_record_is_not(self): for record in (NEW_RECORD, None, {**HOST_RECORD, "env": {}}): assert drift.move_advice(MYST, _shape(network_mode="host"), record, lan_fence.MODE_ON)["warning"] is None diff --git a/tests/test_preflight.py b/tests/test_preflight.py index 3f64572c..6f58d989 100644 --- a/tests/test_preflight.py +++ b/tests/test_preflight.py @@ -221,7 +221,7 @@ def test_the_flag_is_not_set_on_services_merely_wanting_a_residential_ip(self): class TestSourcedPerIpLimits: """Values must trace to a provider statement; see docs/research/per-ip-device-limits.md.""" - @pytest.mark.parametrize("slug", ["honeygain", "iproyal", "spide", "earnfm", "ebesucher"]) + @pytest.mark.parametrize("slug", ["honeygain", "iproyal", "spide", "earnfm", "ebesucher", "mysterium"]) def test_the_documented_services_declare_one_per_ip(self, slug): """spide was MISSING from this list, and that is how the bug shipped. @@ -269,6 +269,14 @@ def test_services_with_no_provider_source_stay_absent(self, slug): assert svc is not None, f"{slug} is not in the catalog" assert "devices_per_ip" not in (svc.get("requirements") or {}) + def test_a_second_mystnodes_on_one_machine_earns_nothing(self): + """MystNodes shows every node but one per public IP offline; it read as unlimited.""" + from app import catalog + + out = preflight.assess(catalog.get_service("mysterium"), already_deployed_slugs={"mysterium"}) + assert out["verdict"] == preflight.EARNS_NOTHING + assert "one device per IP" in " ".join(f["message"] for f in out["findings"]) + def test_a_second_device_behind_one_ip_now_gets_the_strongest_verdict(self): """The point of the bead: the verdict was weak exactly where the mistake is most likely, because the limit was never recorded.""" From 0d887c49f533a34b8c212d7bf695b54ea93096d1 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 8 Nov 2026 18:44:25 +0100 Subject: [PATCH 2/4] fix(mysterium): say a reservation stops automatic picks only, and undo only the in-memory pop on the refusal A reserved range keeps the kernel from handing its ports out, but a program that binds one by number still takes it. The refusal now puts back only the hold it popped and leaves the record on disk as it is, so a record written while the deploy ran is not overwritten. --- UPGRADING.md | 3 ++- app/drift.py | 3 ++- app/orchestrator.py | 6 ++++-- docs/guides/mysterium.md | 4 +++- tests/test_mysterium_fenced.py | 20 ++++++++++++++++++++ 5 files changed, 31 insertions(+), 5 deletions(-) diff --git a/UPGRADING.md b/UPGRADING.md index 28df6c2b..3d42fcd8 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -56,7 +56,8 @@ drift, and **Check isolation** fails on it. change: `sudo sysctl -w net.ipv4.ip_local_reserved_ports=56000-56100` (use your range, with a dash, added to any ranges already listed there), and the line `net.ipv4.ip_local_reserved_ports = 56000-56100` in a file under - `/etc/sysctl.d/` so it survives a reboot. The worker cannot see the host's + `/etc/sysctl.d/` so it survives a reboot. It stops automatic picks only; a + program that binds one of the ports by number still fails the start. The worker cannot see the host's reservation, so Fleet keeps warning about the range; once it is reserved, redeploy anyway. The worker publishes the range 1:1 through Docker, and a node's range on diff --git a/app/drift.py b/app/drift.py index d9b70ad3..15dd9968 100644 --- a/app/drift.py +++ b/app/drift.py @@ -179,7 +179,8 @@ def move_advice( f"Or keep this range and reserve it on the host, so no program there takes one of its ports: run " f"sysctl -w {RESERVED_PORTS_SYSCTL}={m[1]}-{m[2]} (add it to any ranges already listed there, " f"comma-separated), and put the line {RESERVED_PORTS_SYSCTL} = {m[1]}-{m[2]} in a file under " - "/etc/sysctl.d/ so it survives a reboot. If you've reserved this range on the host, you can redeploy." + "/etc/sysctl.d/ so it survives a reboot. That stops automatic picks only: a program that binds one of " + "these ports by number still fails the start. If you've reserved this range on the host, you can redeploy." ) return advice diff --git a/app/orchestrator.py b/app/orchestrator.py index 9f98819c..14ef975c 100644 --- a/app/orchestrator.py +++ b/app/orchestrator.py @@ -917,8 +917,10 @@ def deploy_raw( identity = read_node_identity(old, slug) if running else None if running and identity is None and _reads_identity(slug): # Without it the new node could run on another keystore unchecked. - # Nothing is replaced, so a hold popped above comes back. - _restore_identity(name, identity_before, held_before) + # Nothing is replaced, so the hold popped above comes back. Only + # the memory moved: the record on disk is left as it is. + if held_before is not None: + _MISMATCH_HELD[name] = held_before raise ContainerRuntimeError( f"{name} was not replaced: its identity could not be read from the running node, so the new " "one could not be checked against it. The running container was not touched. Try again in a " diff --git a/docs/guides/mysterium.md b/docs/guides/mysterium.md index 2ba47f75..10d20ee0 100644 --- a/docs/guides/mysterium.md +++ b/docs/guides/mysterium.md @@ -115,7 +115,9 @@ echo 'net.ipv4.ip_local_reserved_ports = 56000-56100' | sudo tee /etc/sysctl.d/9 If `sysctl net.ipv4.ip_local_reserved_ports` already lists ranges, add yours to them, comma-separated, in both places. The file under `/etc/sysctl.d/` -keeps it after a reboot. The worker cannot read the host's setting (it runs +keeps it after a reboot. A reservation only stops the kernel handing those +ports out on its own: a program on the machine that binds one of them by +number still takes it, and the node's start fails as before. The worker cannot read the host's setting (it runs on its own network, which has its own copy), so the Fleet warning about the range stays after you reserve it; once the range is reserved, redeploy anyway. diff --git a/tests/test_mysterium_fenced.py b/tests/test_mysterium_fenced.py index a16e0b8c..01157c30 100644 --- a/tests/test_mysterium_fenced.py +++ b/tests/test_mysterium_fenced.py @@ -1201,6 +1201,24 @@ def test_the_refusal_keeps_a_hold_kept_only_in_memory(self, markers): assert orchestrator._MISMATCH_HELD.get("cashpilot-mysterium") == "its identity is 0xb, not 0xa" assert "0xb" in orchestrator.identity_problem(_node(status="exited")) + def test_the_refusal_leaves_the_record_on_disk_alone(self, markers): + # Written by someone else while the deploy ran: the refusal must not undo it. + old = _node() + old.exec_run.return_value = (1, b"") + old.attrs["Mounts"] = [] + real = orchestrator.read_node_identity + + def read_and_race(container, slug): + orchestrator._remember_identity("cashpilot-mysterium", "MISMATCH written meanwhile") + return real(container, slug) + + with ( + patch.object(orchestrator, "read_node_identity", side_effect=read_and_race), + pytest.raises(orchestrator.ContainerRuntimeError, match="not touched"), + ): + _raw_deploy(old=old) + assert orchestrator._identity_snapshot("cashpilot-mysterium") == "MISMATCH written meanwhile" + def test_control_a_stopped_node_is_replaced_without_the_check(self, markers): old = _node(status="exited") old.exec_run.return_value = (1, b"") @@ -1685,6 +1703,8 @@ def test_the_warning_offers_reserving_the_range_on_the_host_instead(self): assert "sysctl -w net.ipv4.ip_local_reserved_ports=56000-56100" in warning assert "net.ipv4.ip_local_reserved_ports = 56000-56100 in a file under /etc/sysctl.d/" in warning assert "add it to any ranges already listed there" in warning + # The kernel keeps it from automatic picks only; an explicit bind still takes the port. + assert "stops automatic picks only" in warning and "binds one of these ports by number" in warning # The worker cannot read the host's reservation, so it says when to ignore the warning. assert warning.endswith("If you've reserved this range on the host, you can redeploy.") From b34a3601b57cd579e4b54a4f50a08b42e57f76c0 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 8 Nov 2026 18:58:37 +0100 Subject: [PATCH 3/4] docs(mysterium): read the reservations already set before the sysctl write replaces them --- app/drift.py | 2 +- docs/guides/mysterium.md | 23 +++++++++++++---------- tests/test_mysterium_fenced.py | 3 +++ 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/app/drift.py b/app/drift.py index 15dd9968..f5363ce2 100644 --- a/app/drift.py +++ b/app/drift.py @@ -176,7 +176,7 @@ def move_advice( "redeploy keeps: on its own network every port in it is published, and one a host program holds fails " f"the start. Before you redeploy, set {key} to a free range below {low} other than the default{example} " "(a value equal to the default keeps the recorded one), and change the router forward to match. " - f"Or keep this range and reserve it on the host, so no program there takes one of its ports: run " + f"Or keep this range and reserve it on the host, so the kernel stops handing its ports out: run " f"sysctl -w {RESERVED_PORTS_SYSCTL}={m[1]}-{m[2]} (add it to any ranges already listed there, " f"comma-separated), and put the line {RESERVED_PORTS_SYSCTL} = {m[1]}-{m[2]} in a file under " "/etc/sysctl.d/ so it survives a reboot. That stops automatic picks only: a program that binds one of " diff --git a/docs/guides/mysterium.md b/docs/guides/mysterium.md index 10d20ee0..9d987bd0 100644 --- a/docs/guides/mysterium.md +++ b/docs/guides/mysterium.md @@ -104,22 +104,25 @@ as a field left as it was, so the recorded range would stay. To take the default itself, remove the service from that machine (its data volume stays) and deploy again. -Or keep the range on record and reserve it on the host, so no program there -can take one of its ports, then redeploy without touching the router. For -`56000:56100`, on the machine running the node: +Or keep the range on record and reserve it on the host, so the kernel stops +handing its ports out to other programs, then redeploy without touching the +router. For +`56000:56100`, on the machine running the node, first read +`sysctl net.ipv4.ip_local_reserved_ports`: if it already lists ranges, add +`56000-56100` to them, comma-separated, and use that whole list in both +commands below, or the first one drops the reservations already there. ```bash sudo sysctl -w net.ipv4.ip_local_reserved_ports=56000-56100 echo 'net.ipv4.ip_local_reserved_ports = 56000-56100' | sudo tee /etc/sysctl.d/90-mysterium-ports.conf ``` -If `sysctl net.ipv4.ip_local_reserved_ports` already lists ranges, add yours -to them, comma-separated, in both places. The file under `/etc/sysctl.d/` -keeps it after a reboot. A reservation only stops the kernel handing those -ports out on its own: a program on the machine that binds one of them by -number still takes it, and the node's start fails as before. The worker cannot read the host's setting (it runs -on its own network, which has its own copy), so the Fleet warning about the -range stays after you reserve it; once the range is reserved, redeploy anyway. +The file under `/etc/sysctl.d/` keeps it after a reboot. A reservation only +stops the kernel handing those ports out on its own: a program on the machine +that binds one of them by number still takes it, and the node's start fails as +before. The worker cannot read the host's setting (it runs on its own network, +which has its own copy), so the Fleet warning about the range stays after you +reserve it; once the range is reserved, redeploy anyway. **Check it.** After a few sessions, on the machine running the node: diff --git a/tests/test_mysterium_fenced.py b/tests/test_mysterium_fenced.py index 01157c30..7e3909ae 100644 --- a/tests/test_mysterium_fenced.py +++ b/tests/test_mysterium_fenced.py @@ -1719,6 +1719,9 @@ def test_the_guide_and_the_upgrade_notes_give_the_same_reservation(self): assert "sysctl -w net.ipv4.ip_local_reserved_ports=56000-56100" in text, doc assert "net.ipv4.ip_local_reserved_ports = 56000-56100" in text, doc assert "/etc/sysctl.d/" in text, doc + guide = (ROOT / "docs" / "guides" / "mysterium.md").read_text(encoding="utf-8") + # Read before the write: the sysctl -w replaces the whole list. + assert guide.index("read\n`sysctl net.ipv4.ip_local_reserved_ports`") < guide.index("sudo sysctl -w") def test_the_guide_no_longer_says_the_node_runs_on_the_host_network(self): guide = (ROOT / "docs" / "guides" / "mysterium.md").read_text(encoding="utf-8") From 22f8782a9ff2840b90f82949abe2818d82af70a5 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 8 Nov 2026 19:18:07 +0100 Subject: [PATCH 4/4] fix(mysterium): read existing reservations first, name the .conf file, and keep a hold set during a refused deploy The reserve route now says to read the list before sysctl -w replaces it, uses sudo, and names a .conf file, the only kind systemd-sysctl loads. The refusal restores the popped hold only when no newer one was set meanwhile. The guide says the firewall on the operator's own network is theirs. --- UPGRADING.md | 10 ++++++---- app/drift.py | 11 +++++++---- app/orchestrator.py | 5 +++-- docs/guides/mysterium.md | 2 +- tests/test_mysterium_fenced.py | 34 +++++++++++++++++++++++++++------- 5 files changed, 44 insertions(+), 18 deletions(-) diff --git a/UPGRADING.md b/UPGRADING.md index 3d42fcd8..a0088015 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -53,10 +53,12 @@ drift, and **Check isolation** fails on it. would stay; to take the default itself, remove the service from that machine (its data volume stays) and deploy again. Or keep the range and reserve it on the host instead, with no router - change: `sudo sysctl -w net.ipv4.ip_local_reserved_ports=56000-56100` (use - your range, with a dash, added to any ranges already listed there), and the - line `net.ipv4.ip_local_reserved_ports = 56000-56100` in a file under - `/etc/sysctl.d/` so it survives a reboot. It stops automatic picks only; a + change. First read `sysctl net.ipv4.ip_local_reserved_ports`: if it lists + ranges, add yours (with a dash) to them, comma-separated, and use the whole + list in both places, or the write drops them. Then run + `sudo sysctl -w net.ipv4.ip_local_reserved_ports=56000-56100` and put the + line `net.ipv4.ip_local_reserved_ports = 56000-56100` in + `/etc/sysctl.d/90-mysterium-ports.conf` so it survives a reboot. It stops automatic picks only; a program that binds one of the ports by number still fails the start. The worker cannot see the host's reservation, so Fleet keeps warning about the range; once it is reserved, redeploy anyway. diff --git a/app/drift.py b/app/drift.py index f5363ce2..7653611c 100644 --- a/app/drift.py +++ b/app/drift.py @@ -131,6 +131,8 @@ def _names(values: set[str]) -> str: #: Per network namespace: the worker, on its own bridge, reads its own value and #: never the host's, so the warning names the reservation instead of checking it. RESERVED_PORTS_SYSCTL = "net.ipv4.ip_local_reserved_ports" +#: systemd-sysctl reads only *.conf files there. +RESERVED_PORTS_FILE = "/etc/sysctl.d/90-mysterium-ports.conf" def move_advice( @@ -176,10 +178,11 @@ def move_advice( "redeploy keeps: on its own network every port in it is published, and one a host program holds fails " f"the start. Before you redeploy, set {key} to a free range below {low} other than the default{example} " "(a value equal to the default keeps the recorded one), and change the router forward to match. " - f"Or keep this range and reserve it on the host, so the kernel stops handing its ports out: run " - f"sysctl -w {RESERVED_PORTS_SYSCTL}={m[1]}-{m[2]} (add it to any ranges already listed there, " - f"comma-separated), and put the line {RESERVED_PORTS_SYSCTL} = {m[1]}-{m[2]} in a file under " - "/etc/sysctl.d/ so it survives a reboot. That stops automatic picks only: a program that binds one of " + f"Or keep this range and reserve it on the host, so the kernel stops handing its ports out. First read " + f"sysctl {RESERVED_PORTS_SYSCTL}: if it lists ranges, add {m[1]}-{m[2]} to them, comma-separated, and use " + f"the whole list below, or the write drops them. Then run sudo sysctl -w {RESERVED_PORTS_SYSCTL}=" + f"{m[1]}-{m[2]}, and put the line {RESERVED_PORTS_SYSCTL} = {m[1]}-{m[2]} in " + f"{RESERVED_PORTS_FILE} so it survives a reboot. That stops automatic picks only: a program that binds one of " "these ports by number still fails the start. If you've reserved this range on the host, you can redeploy." ) return advice diff --git a/app/orchestrator.py b/app/orchestrator.py index 14ef975c..7369c831 100644 --- a/app/orchestrator.py +++ b/app/orchestrator.py @@ -918,9 +918,10 @@ def deploy_raw( if running and identity is None and _reads_identity(slug): # Without it the new node could run on another keystore unchecked. # Nothing is replaced, so the hold popped above comes back. Only - # the memory moved: the record on disk is left as it is. + # the memory moved: the record on disk is left as it is, and a + # hold set meanwhile is newer than the one popped, so it stays. if held_before is not None: - _MISMATCH_HELD[name] = held_before + _MISMATCH_HELD.setdefault(name, held_before) raise ContainerRuntimeError( f"{name} was not replaced: its identity could not be read from the running node, so the new " "one could not be checked against it. The running container was not touched. Try again in a " diff --git a/docs/guides/mysterium.md b/docs/guides/mysterium.md index 9d987bd0..6c777191 100644 --- a/docs/guides/mysterium.md +++ b/docs/guides/mysterium.md @@ -22,7 +22,7 @@ MystNodes (Mysterium Network) is a decentralized VPN and proxy network built on > **One node per public IP.** Mysterium strictly enforces one active node per public IP address. Additional nodes on the same IP show as offline and earn nothing. Do not run on a phone if a Docker node is already running on the same network. Use separate public IPs (e.g. dual WAN, different locations) for additional nodes. Fleet warns before a second node is deployed behind an address that already has one. -> **Port forwarding recommended.** Forward the node's UDP range to maximize earnings; see **Port forwarding** below. Without it many VPN and proxy sessions fail to connect, which cuts income. The node runs behind CashPilot's LAN fence, on its fenced bridge or your own network (see [Isolation](../isolation.md)), with the `NET_ADMIN`, `SETUID` and `SETGID` capabilities. +> **Port forwarding recommended.** Forward the node's UDP range to maximize earnings; see **Port forwarding** below. Without it many VPN and proxy sessions fail to connect, which cuts income. The node runs on CashPilot's fenced bridge, or on your own network (`CASHPILOT_CONTAINER_NETWORK`), where the firewall rules are yours to supply (see [Isolation](../isolation.md)), with the `NET_ADMIN`, `SETUID` and `SETGID` capabilities. ## Cashing out diff --git a/tests/test_mysterium_fenced.py b/tests/test_mysterium_fenced.py index 7e3909ae..ff98f662 100644 --- a/tests/test_mysterium_fenced.py +++ b/tests/test_mysterium_fenced.py @@ -1219,6 +1219,24 @@ def read_and_race(container, slug): _raw_deploy(old=old) assert orchestrator._identity_snapshot("cashpilot-mysterium") == "MISMATCH written meanwhile" + def test_the_refusal_keeps_a_hold_set_while_it_ran(self, markers): + orchestrator._MISMATCH_HELD["cashpilot-mysterium"] = "the older reason" + old = _node() + old.exec_run.return_value = (1, b"") + old.attrs["Mounts"] = [] + real = orchestrator.read_node_identity + + def read_and_race(container, slug): + orchestrator._MISMATCH_HELD["cashpilot-mysterium"] = "the newer reason" + return real(container, slug) + + with ( + patch.object(orchestrator, "read_node_identity", side_effect=read_and_race), + pytest.raises(orchestrator.ContainerRuntimeError, match="not touched"), + ): + _raw_deploy(old=old) + assert orchestrator._MISMATCH_HELD["cashpilot-mysterium"] == "the newer reason" + def test_control_a_stopped_node_is_replaced_without_the_check(self, markers): old = _node(status="exited") old.exec_run.return_value = (1, b"") @@ -1700,9 +1718,12 @@ def test_a_recorded_range_in_the_ephemeral_range_is_flagged_before_the_redeploy( def test_the_warning_offers_reserving_the_range_on_the_host_instead(self): warning = drift.move_advice(MYST, _shape(network_mode="host"), HOST_RECORD, lan_fence.MODE_ON)["warning"] assert "keep this range and reserve it on the host" in warning - assert "sysctl -w net.ipv4.ip_local_reserved_ports=56000-56100" in warning - assert "net.ipv4.ip_local_reserved_ports = 56000-56100 in a file under /etc/sysctl.d/" in warning - assert "add it to any ranges already listed there" in warning + assert "sudo sysctl -w net.ipv4.ip_local_reserved_ports=56000-56100" in warning + # systemd-sysctl skips a file without .conf, so the file is named. + assert "net.ipv4.ip_local_reserved_ports = 56000-56100 in /etc/sysctl.d/90-mysterium-ports.conf" in warning + # Read before the write: sysctl -w replaces the whole list. + assert warning.index("First read sysctl net.ipv4.ip_local_reserved_ports") < warning.index("sudo sysctl -w") + assert "add 56000-56100 to them, comma-separated" in warning # The kernel keeps it from automatic picks only; an explicit bind still takes the port. assert "stops automatic picks only" in warning and "binds one of these ports by number" in warning # The worker cannot read the host's reservation, so it says when to ignore the warning. @@ -1718,10 +1739,9 @@ def test_the_guide_and_the_upgrade_notes_give_the_same_reservation(self): text = (ROOT / doc).read_text(encoding="utf-8") assert "sysctl -w net.ipv4.ip_local_reserved_ports=56000-56100" in text, doc assert "net.ipv4.ip_local_reserved_ports = 56000-56100" in text, doc - assert "/etc/sysctl.d/" in text, doc - guide = (ROOT / "docs" / "guides" / "mysterium.md").read_text(encoding="utf-8") - # Read before the write: the sysctl -w replaces the whole list. - assert guide.index("read\n`sysctl net.ipv4.ip_local_reserved_ports`") < guide.index("sudo sysctl -w") + assert "/etc/sysctl.d/90-mysterium-ports.conf" in text, doc + flat = " ".join(text.split()) + assert flat.index("read `sysctl net.ipv4.ip_local_reserved_ports`") < flat.index("sudo sysctl -w"), doc def test_the_guide_no_longer_says_the_node_runs_on_the_host_network(self): guide = (ROOT / "docs" / "guides" / "mysterium.md").read_text(encoding="utf-8")