diff --git a/UPGRADING.md b/UPGRADING.md index 44edcec5..a0088015 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -52,6 +52,16 @@ drift, and **Check isolation** fails on it. (`23000:23029`) reads as a field left as it was, so the range on record would stay; to take the default itself, remove the service from that machine (its data volume stays) and deploy again. + Or keep the range and reserve it on the host instead, with no router + change. First read `sysctl net.ipv4.ip_local_reserved_ports`: if it lists + ranges, add yours (with a dash) to them, comma-separated, and use the whole + list in both places, or the write drops them. Then run + `sudo sysctl -w net.ipv4.ip_local_reserved_ports=56000-56100` and put the + line `net.ipv4.ip_local_reserved_ports = 56000-56100` in + `/etc/sysctl.d/90-mysterium-ports.conf` so it survives a reboot. It stops automatic picks only; a + program that binds one of the ports by number still fails the start. The worker cannot see the host's + reservation, so Fleet keeps warning about the range; once it is reserved, + redeploy anyway. The worker publishes the range 1:1 through Docker, and a node's range on record (`56000:56100` through 2.0.6, `56000:56029` in 2.0.7 and 2.1.0) sits in Linux's ephemeral range, 32768-60999, where any program on the host can diff --git a/app/drift.py b/app/drift.py index 518cd64f..7653611c 100644 --- a/app/drift.py +++ b/app/drift.py @@ -128,6 +128,13 @@ def _names(values: set[str]) -> str: return ", ".join(sorted(values)) +#: Per network namespace: the worker, on its own bridge, reads its own value and +#: never the host's, so the warning names the reservation instead of checking it. +RESERVED_PORTS_SYSCTL = "net.ipv4.ip_local_reserved_ports" +#: systemd-sysctl reads only *.conf files there. +RESERVED_PORTS_FILE = "/etc/sysctl.d/90-mysterium-ports.conf" + + def move_advice( svc: dict[str, Any] | None, shape: dict[str, Any] | None, @@ -170,7 +177,13 @@ def move_advice( f"Its recorded {key}, {value}, overlaps Linux's ephemeral range ({low}-{EPHEMERAL_PORTS[1]}), which the " "redeploy keeps: on its own network every port in it is published, and one a host program holds fails " f"the start. Before you redeploy, set {key} to a free range below {low} other than the default{example} " - "(a value equal to the default keeps the recorded one), and change the router forward to match." + "(a value equal to the default keeps the recorded one), and change the router forward to match. " + f"Or keep this range and reserve it on the host, so the kernel stops handing its ports out. First read " + f"sysctl {RESERVED_PORTS_SYSCTL}: if it lists ranges, add {m[1]}-{m[2]} to them, comma-separated, and use " + f"the whole list below, or the write drops them. Then run sudo sysctl -w {RESERVED_PORTS_SYSCTL}=" + f"{m[1]}-{m[2]}, and put the line {RESERVED_PORTS_SYSCTL} = {m[1]}-{m[2]} in " + f"{RESERVED_PORTS_FILE} so it survives a reboot. That stops automatic picks only: a program that binds one of " + "these ports by number still fails the start. If you've reserved this range on the host, you can redeploy." ) return advice diff --git a/app/orchestrator.py b/app/orchestrator.py index a58fa2d8..7369c831 100644 --- a/app/orchestrator.py +++ b/app/orchestrator.py @@ -917,6 +917,11 @@ def deploy_raw( identity = read_node_identity(old, slug) if running else None if running and identity is None and _reads_identity(slug): # Without it the new node could run on another keystore unchecked. + # Nothing is replaced, so the hold popped above comes back. Only + # the memory moved: the record on disk is left as it is, and a + # hold set meanwhile is newer than the one popped, so it stays. + if held_before is not None: + _MISMATCH_HELD.setdefault(name, held_before) raise ContainerRuntimeError( f"{name} was not replaced: its identity could not be read from the running node, so the new " "one could not be checked against it. The running container was not touched. Try again in a " diff --git a/docs/guides/mysterium.md b/docs/guides/mysterium.md index c695a796..6c777191 100644 --- a/docs/guides/mysterium.md +++ b/docs/guides/mysterium.md @@ -20,9 +20,9 @@ MystNodes (Mysterium Network) is a decentralized VPN and proxy network built on > Earnings in MYST tokens. Residential IPs earn significantly more. Node WebUI at port 4449 for management, published on the machine it runs on only (127.0.0.1); from another machine open an SSH tunnel first (`ssh -L 4449:127.0.0.1:4449 `, then browse `http://127.0.0.1:4449`), or put a reverse proxy in front (see **WebUI address** below). VPS accepted. Important: after first run, set your beneficiary (settlement) wallet via the node WebUI or CLI to match your mystnodes.com account -- this links on-chain earnings to your cloud dashboard. -> **One node per public IP.** Mysterium strictly enforces one active node per public IP address. Additional nodes on the same IP show as offline and earn nothing. Do not run on a phone if a Docker node is already running on the same network. Use separate public IPs (e.g. dual WAN, different locations) for additional nodes. +> **One node per public IP.** Mysterium strictly enforces one active node per public IP address. Additional nodes on the same IP show as offline and earn nothing. Do not run on a phone if a Docker node is already running on the same network. Use separate public IPs (e.g. dual WAN, different locations) for additional nodes. Fleet warns before a second node is deployed behind an address that already has one. -> **Port forwarding recommended.** Forward the node's UDP range to maximize earnings; see **Port forwarding** below. Without it many VPN and proxy sessions fail to connect, which cuts income. The Docker image runs with `--net host` and the `NET_ADMIN`, `SETUID` and `SETGID` capabilities. +> **Port forwarding recommended.** Forward the node's UDP range to maximize earnings; see **Port forwarding** below. Without it many VPN and proxy sessions fail to connect, which cuts income. The node runs on CashPilot's fenced bridge, or on your own network (`CASHPILOT_CONTAINER_NETWORK`), where the firewall rules are yours to supply (see [Isolation](../isolation.md)), with the `NET_ADMIN`, `SETUID` and `SETGID` capabilities. ## Cashing out @@ -104,6 +104,26 @@ as a field left as it was, so the recorded range would stay. To take the default itself, remove the service from that machine (its data volume stays) and deploy again. +Or keep the range on record and reserve it on the host, so the kernel stops +handing its ports out to other programs, then redeploy without touching the +router. For +`56000:56100`, on the machine running the node, first read +`sysctl net.ipv4.ip_local_reserved_ports`: if it already lists ranges, add +`56000-56100` to them, comma-separated, and use that whole list in both +commands below, or the first one drops the reservations already there. + +```bash +sudo sysctl -w net.ipv4.ip_local_reserved_ports=56000-56100 +echo 'net.ipv4.ip_local_reserved_ports = 56000-56100' | sudo tee /etc/sysctl.d/90-mysterium-ports.conf +``` + +The file under `/etc/sysctl.d/` keeps it after a reboot. A reservation only +stops the kernel handing those ports out on its own: a program on the machine +that binds one of them by number still takes it, and the node's start fails as +before. The worker cannot read the host's setting (it runs on its own network, +which has its own copy), so the Fleet warning about the range stays after you +reserve it; once the range is reserved, redeploy anyway. + **Check it.** After a few sessions, on the machine running the node: ```bash diff --git a/docs/research/per-ip-device-limits.md b/docs/research/per-ip-device-limits.md index df50d560..1aa30c06 100644 --- a/docs/research/per-ip-device-limits.md +++ b/docs/research/per-ip-device-limits.md @@ -26,6 +26,7 @@ reviewer with a link opens the page and sees it; without one, the error survives | iproyal (Pawns.app) | **1** | hard limit, in binding Terms | *"Pawns.app limits the number of devices per single IP address to one (1)."* | | spide | **1** | hard limit, in binding Terms | *"Spide Network limits the number of devices per single IP address to one."* Restrictions also forbid *"connect[ing] more than the allowed number of devices"*. The same paragraph adds that traffic through one IP *"will be distributed between devices utilizing the same IP address"*. | | earnfm | **1** | help-centre guidance, not a Terms clause | *"One Device Per Network: Each device **should** be connected to a unique IP or network."* Phrased as guidance; no device-per-IP clause found in the Terms, so treat enforcement as unproven. | +| mysterium (MystNodes) | **1** | hard limit, enforced | *"You can only run one MystNodes per public IP address"*; *"Each node needs its own unique public IP."* Extra nodes on one IP show offline. (earlier: *"We do not support multiple devices under a single IP address"*, ) | | ebesucher | **1** | hard limit | Multi-device is allowed only if *"each device ... has to be connected to the Internet through a unique IP address."* | ## Documented as sharing, with no number diff --git a/services/bandwidth/mysterium.yml b/services/bandwidth/mysterium.yml index 8030cf6b..5ffded44 100644 --- a/services/bandwidth/mysterium.yml +++ b/services/bandwidth/mysterium.yml @@ -163,7 +163,9 @@ docker: requirements: residential_ip: false - devices_per_ip: 0 + # MystNodes help centre: one node per public IP; the others show offline + # (docs/research/per-ip-device-limits.md). + devices_per_ip: 1 min_bandwidth: "" gpu: false min_storage: "" diff --git a/tests/test_mysterium_fenced.py b/tests/test_mysterium_fenced.py index 3c2a5a85..ff98f662 100644 --- a/tests/test_mysterium_fenced.py +++ b/tests/test_mysterium_fenced.py @@ -1190,6 +1190,53 @@ def test_a_running_node_whose_identity_cannot_be_read_is_not_replaced(self, mark old.stop.assert_not_called() old.remove.assert_not_called() + def test_the_refusal_keeps_a_hold_kept_only_in_memory(self, markers): + # No record on disk: the hold could not be written, so memory is all there is. + orchestrator._MISMATCH_HELD["cashpilot-mysterium"] = "its identity is 0xb, not 0xa" + old = _node() + old.exec_run.return_value = (1, b"") + old.attrs["Mounts"] = [] + with pytest.raises(orchestrator.ContainerRuntimeError, match="not touched"): + _raw_deploy(old=old) + assert orchestrator._MISMATCH_HELD.get("cashpilot-mysterium") == "its identity is 0xb, not 0xa" + assert "0xb" in orchestrator.identity_problem(_node(status="exited")) + + def test_the_refusal_leaves_the_record_on_disk_alone(self, markers): + # Written by someone else while the deploy ran: the refusal must not undo it. + old = _node() + old.exec_run.return_value = (1, b"") + old.attrs["Mounts"] = [] + real = orchestrator.read_node_identity + + def read_and_race(container, slug): + orchestrator._remember_identity("cashpilot-mysterium", "MISMATCH written meanwhile") + return real(container, slug) + + with ( + patch.object(orchestrator, "read_node_identity", side_effect=read_and_race), + pytest.raises(orchestrator.ContainerRuntimeError, match="not touched"), + ): + _raw_deploy(old=old) + assert orchestrator._identity_snapshot("cashpilot-mysterium") == "MISMATCH written meanwhile" + + def test_the_refusal_keeps_a_hold_set_while_it_ran(self, markers): + orchestrator._MISMATCH_HELD["cashpilot-mysterium"] = "the older reason" + old = _node() + old.exec_run.return_value = (1, b"") + old.attrs["Mounts"] = [] + real = orchestrator.read_node_identity + + def read_and_race(container, slug): + orchestrator._MISMATCH_HELD["cashpilot-mysterium"] = "the newer reason" + return real(container, slug) + + with ( + patch.object(orchestrator, "read_node_identity", side_effect=read_and_race), + pytest.raises(orchestrator.ContainerRuntimeError, match="not touched"), + ): + _raw_deploy(old=old) + assert orchestrator._MISMATCH_HELD["cashpilot-mysterium"] == "the newer reason" + def test_control_a_stopped_node_is_replaced_without_the_check(self, markers): old = _node(status="exited") old.exec_run.return_value = (1, b"") @@ -1668,6 +1715,38 @@ def test_a_recorded_range_in_the_ephemeral_range_is_flagged_before_the_redeploy( assert "Before you redeploy" in warning and "below 32768 other than the default" in warning assert "23100:23129" in warning + def test_the_warning_offers_reserving_the_range_on_the_host_instead(self): + warning = drift.move_advice(MYST, _shape(network_mode="host"), HOST_RECORD, lan_fence.MODE_ON)["warning"] + assert "keep this range and reserve it on the host" in warning + assert "sudo sysctl -w net.ipv4.ip_local_reserved_ports=56000-56100" in warning + # systemd-sysctl skips a file without .conf, so the file is named. + assert "net.ipv4.ip_local_reserved_ports = 56000-56100 in /etc/sysctl.d/90-mysterium-ports.conf" in warning + # Read before the write: sysctl -w replaces the whole list. + assert warning.index("First read sysctl net.ipv4.ip_local_reserved_ports") < warning.index("sudo sysctl -w") + assert "add 56000-56100 to them, comma-separated" in warning + # The kernel keeps it from automatic picks only; an explicit bind still takes the port. + assert "stops automatic picks only" in warning and "binds one of these ports by number" in warning + # The worker cannot read the host's reservation, so it says when to ignore the warning. + assert warning.endswith("If you've reserved this range on the host, you can redeploy.") + + def test_the_reservation_names_the_recorded_range_not_a_fixed_one(self): + record = {**HOST_RECORD, "env": {"UDP_PORTS": "40000:40020"}} + warning = drift.move_advice(MYST, _shape(network_mode="host"), record, lan_fence.MODE_ON)["warning"] + assert "ip_local_reserved_ports=40000-40020" in warning and "56000" not in warning + + def test_the_guide_and_the_upgrade_notes_give_the_same_reservation(self): + for doc in ("docs/guides/mysterium.md", "UPGRADING.md"): + text = (ROOT / doc).read_text(encoding="utf-8") + assert "sysctl -w net.ipv4.ip_local_reserved_ports=56000-56100" in text, doc + assert "net.ipv4.ip_local_reserved_ports = 56000-56100" in text, doc + assert "/etc/sysctl.d/90-mysterium-ports.conf" in text, doc + flat = " ".join(text.split()) + assert flat.index("read `sysctl net.ipv4.ip_local_reserved_ports`") < flat.index("sudo sysctl -w"), doc + + def test_the_guide_no_longer_says_the_node_runs_on_the_host_network(self): + guide = (ROOT / "docs" / "guides" / "mysterium.md").read_text(encoding="utf-8") + assert "--net host" not in guide + def test_control_a_range_below_it_or_none_on_record_is_not(self): for record in (NEW_RECORD, None, {**HOST_RECORD, "env": {}}): assert drift.move_advice(MYST, _shape(network_mode="host"), record, lan_fence.MODE_ON)["warning"] is None diff --git a/tests/test_preflight.py b/tests/test_preflight.py index 3f64572c..6f58d989 100644 --- a/tests/test_preflight.py +++ b/tests/test_preflight.py @@ -221,7 +221,7 @@ def test_the_flag_is_not_set_on_services_merely_wanting_a_residential_ip(self): class TestSourcedPerIpLimits: """Values must trace to a provider statement; see docs/research/per-ip-device-limits.md.""" - @pytest.mark.parametrize("slug", ["honeygain", "iproyal", "spide", "earnfm", "ebesucher"]) + @pytest.mark.parametrize("slug", ["honeygain", "iproyal", "spide", "earnfm", "ebesucher", "mysterium"]) def test_the_documented_services_declare_one_per_ip(self, slug): """spide was MISSING from this list, and that is how the bug shipped. @@ -269,6 +269,14 @@ def test_services_with_no_provider_source_stay_absent(self, slug): assert svc is not None, f"{slug} is not in the catalog" assert "devices_per_ip" not in (svc.get("requirements") or {}) + def test_a_second_mystnodes_on_one_machine_earns_nothing(self): + """MystNodes shows every node but one per public IP offline; it read as unlimited.""" + from app import catalog + + out = preflight.assess(catalog.get_service("mysterium"), already_deployed_slugs={"mysterium"}) + assert out["verdict"] == preflight.EARNS_NOTHING + assert "one device per IP" in " ".join(f["message"] for f in out["findings"]) + def test_a_second_device_behind_one_ip_now_gets_the_strongest_verdict(self): """The point of the bead: the verdict was weak exactly where the mistake is most likely, because the limit was never recorded."""