mirror of GeiserX/CashPilot#679

UPGRADING.md

@@ -52,6 +52,16 @@ drift, and **Check isolation** fails on it.
    (`23000:23029`) reads as a field left as it was, so the range on record
    would stay; to take the default itself, remove the service from that
    machine (its data volume stays) and deploy again.
+   Or keep the range and reserve it on the host instead, with no router
+   change. First read `sysctl net.ipv4.ip_local_reserved_ports`: if it lists
+   ranges, add yours (with a dash) to them, comma-separated, and use the whole
+   list in both places, or the write drops them. Then run
+   `sudo sysctl -w net.ipv4.ip_local_reserved_ports=56000-56100` and put the
+   line `net.ipv4.ip_local_reserved_ports = 56000-56100` in
+   `/etc/sysctl.d/90-mysterium-ports.conf` so it survives a reboot. It stops automatic picks only; a
+   program that binds one of the ports by number still fails the start. The worker cannot see the host's
+   reservation, so Fleet keeps warning about the range; once it is reserved,
+   redeploy anyway.
    The worker publishes the range 1:1 through Docker, and a node's range on
    record (`56000:56100` through 2.0.6, `56000:56029` in 2.0.7 and 2.1.0) sits in
    Linux's ephemeral range, 32768-60999, where any program on the host can

app/drift.py

@@ -128,6 +128,13 @@ def _names(values: set[str]) -> str:
     return ", ".join(sorted(values))
 
 
+#: Per network namespace: the worker, on its own bridge, reads its own value and
+#: never the host's, so the warning names the reservation instead of checking it.
+RESERVED_PORTS_SYSCTL = "net.ipv4.ip_local_reserved_ports"
+#: systemd-sysctl reads only *.conf files there.
+RESERVED_PORTS_FILE = "/etc/sysctl.d/90-mysterium-ports.conf"
+
+
 def move_advice(
     svc: dict[str, Any] | None,
     shape: dict[str, Any] | None,
@@ -170,7 +177,13 @@ def move_advice(
             f"Its recorded {key}, {value}, overlaps Linux's ephemeral range ({low}-{EPHEMERAL_PORTS[1]}), which the "
             "redeploy keeps: on its own network every port in it is published, and one a host program holds fails "
             f"the start. Before you redeploy, set {key} to a free range below {low} other than the default{example} "
-            "(a value equal to the default keeps the recorded one), and change the router forward to match."
+            "(a value equal to the default keeps the recorded one), and change the router forward to match. "
+            f"Or keep this range and reserve it on the host, so the kernel stops handing its ports out. First read "
+            f"sysctl {RESERVED_PORTS_SYSCTL}: if it lists ranges, add {m[1]}-{m[2]} to them, comma-separated, and use "
+            f"the whole list below, or the write drops them. Then run sudo sysctl -w {RESERVED_PORTS_SYSCTL}="
+            f"{m[1]}-{m[2]}, and put the line {RESERVED_PORTS_SYSCTL} = {m[1]}-{m[2]} in "
+            f"{RESERVED_PORTS_FILE} so it survives a reboot. That stops automatic picks only: a program that binds one of "
+            "these ports by number still fails the start. If you've reserved this range on the host, you can redeploy."
         )
     return advice
 

app/orchestrator.py

@@ -917,6 +917,11 @@ def deploy_raw(
             identity = read_node_identity(old, slug) if running else None
             if running and identity is None and _reads_identity(slug):
                 # Without it the new node could run on another keystore unchecked.
+                # Nothing is replaced, so the hold popped above comes back. Only
+                # the memory moved: the record on disk is left as it is, and a
+                # hold set meanwhile is newer than the one popped, so it stays.
+                if held_before is not None:
+                    _MISMATCH_HELD.setdefault(name, held_before)
                 raise ContainerRuntimeError(
                     f"{name} was not replaced: its identity could not be read from the running node, so the new "
                     "one could not be checked against it. The running container was not touched. Try again in a "

docs/guides/mysterium.md

@@ -20,9 +20,9 @@ MystNodes (Mysterium Network) is a decentralized VPN and proxy network built on
 
 > Earnings in MYST tokens. Residential IPs earn significantly more. Node WebUI at port 4449 for management, published on the machine it runs on only (127.0.0.1); from another machine open an SSH tunnel first (`ssh -L 4449:127.0.0.1:4449 <host>`, then browse `http://127.0.0.1:4449`), or put a reverse proxy in front (see **WebUI address** below). VPS accepted. Important: after first run, set your beneficiary (settlement) wallet via the node WebUI or CLI to match your mystnodes.com account -- this links on-chain earnings to your cloud dashboard.
 
-> **One node per public IP.** Mysterium strictly enforces one active node per public IP address. Additional nodes on the same IP show as offline and earn nothing. Do not run on a phone if a Docker node is already running on the same network. Use separate public IPs (e.g. dual WAN, different locations) for additional nodes.
+> **One node per public IP.** Mysterium strictly enforces one active node per public IP address. Additional nodes on the same IP show as offline and earn nothing. Do not run on a phone if a Docker node is already running on the same network. Use separate public IPs (e.g. dual WAN, different locations) for additional nodes. Fleet warns before a second node is deployed behind an address that already has one.
 
-> **Port forwarding recommended.** Forward the node's UDP range to maximize earnings; see **Port forwarding** below. Without it many VPN and proxy sessions fail to connect, which cuts income. The Docker image runs with `--net host` and the `NET_ADMIN`, `SETUID` and `SETGID` capabilities.
+> **Port forwarding recommended.** Forward the node's UDP range to maximize earnings; see **Port forwarding** below. Without it many VPN and proxy sessions fail to connect, which cuts income. The node runs on CashPilot's fenced bridge, or on your own network (`CASHPILOT_CONTAINER_NETWORK`), where the firewall rules are yours to supply (see [Isolation](../isolation.md)), with the `NET_ADMIN`, `SETUID` and `SETGID` capabilities.
 
 ## Cashing out
 
@@ -104,6 +104,26 @@ as a field left as it was, so the recorded range would stay. To take the
 default itself, remove the service from that machine (its data volume stays)
 and deploy again.
 
+Or keep the range on record and reserve it on the host, so the kernel stops
+handing its ports out to other programs, then redeploy without touching the
+router. For
+`56000:56100`, on the machine running the node, first read
+`sysctl net.ipv4.ip_local_reserved_ports`: if it already lists ranges, add
+`56000-56100` to them, comma-separated, and use that whole list in both
+commands below, or the first one drops the reservations already there.
+
+```bash
+sudo sysctl -w net.ipv4.ip_local_reserved_ports=56000-56100
+echo 'net.ipv4.ip_local_reserved_ports = 56000-56100' | sudo tee /etc/sysctl.d/90-mysterium-ports.conf
+```
+
+The file under `/etc/sysctl.d/` keeps it after a reboot. A reservation only
+stops the kernel handing those ports out on its own: a program on the machine
+that binds one of them by number still takes it, and the node's start fails as
+before. The worker cannot read the host's setting (it runs on its own network,
+which has its own copy), so the Fleet warning about the range stays after you
+reserve it; once the range is reserved, redeploy anyway.
+
 **Check it.** After a few sessions, on the machine running the node:
 
 ```bash

docs/research/per-ip-device-limits.md

@@ -26,6 +26,7 @@ reviewer with a link opens the page and sees it; without one, the error survives
 | iproyal (Pawns.app) | **1** | hard limit, in binding Terms | *"Pawns.app limits the number of devices per single IP address to one (1)."* <https://pawns.app/terms-of-use/> |
 | spide | **1** | hard limit, in binding Terms | *"Spide Network limits the number of devices per single IP address to one."* Restrictions also forbid *"connect[ing] more than the allowed number of devices"*. The same paragraph adds that traffic through one IP *"will be distributed between devices utilizing the same IP address"*. <https://spide.network/terms-of-use.html> |
 | earnfm | **1** | help-centre guidance, not a Terms clause | *"One Device Per Network: Each device **should** be connected to a unique IP or network."* Phrased as guidance; no device-per-IP clause found in the Terms, so treat enforcement as unproven. <https://help.earn.fm/portal/en/kb/articles/do-i-earn-more-if-i-run-earnfm-on-multiple-devices> |
+| mysterium (MystNodes) | **1** | hard limit, enforced | *"You can only run one MystNodes per public IP address"*; *"Each node needs its own unique public IP."* Extra nodes on one IP show offline. <https://help.mystnodes.com/en/articles/3442543-multiple-nodes-on-a-single-ip-address> (earlier: *"We do not support multiple devices under a single IP address"*, <https://help.mysterium.network/en/articles/3442543-multiple-nodes-on-a-single-ip-address>) |
 | ebesucher | **1** | hard limit | Multi-device is allowed only if *"each device ... has to be connected to the Internet through a unique IP address."* <https://www.ebesucher.com/blog/besuchertausch/multigeraete-funktion> |
 
 ## Documented as sharing, with no number

services/bandwidth/mysterium.yml

@@ -163,7 +163,9 @@ docker:
 
 requirements:
   residential_ip: false
-  devices_per_ip: 0
+  # MystNodes help centre: one node per public IP; the others show offline
+  # (docs/research/per-ip-device-limits.md).
+  devices_per_ip: 1
   min_bandwidth: ""
   gpu: false
   min_storage: ""

tests/test_mysterium_fenced.py

@@ -1190,6 +1190,53 @@ def test_a_running_node_whose_identity_cannot_be_read_is_not_replaced(self, mark
         old.stop.assert_not_called()
         old.remove.assert_not_called()
 
+    def test_the_refusal_keeps_a_hold_kept_only_in_memory(self, markers):
+        # No record on disk: the hold could not be written, so memory is all there is.
+        orchestrator._MISMATCH_HELD["cashpilot-mysterium"] = "its identity is 0xb, not 0xa"
+        old = _node()
+        old.exec_run.return_value = (1, b"")
+        old.attrs["Mounts"] = []
+        with pytest.raises(orchestrator.ContainerRuntimeError, match="not touched"):
+            _raw_deploy(old=old)
+        assert orchestrator._MISMATCH_HELD.get("cashpilot-mysterium") == "its identity is 0xb, not 0xa"
+        assert "0xb" in orchestrator.identity_problem(_node(status="exited"))
+
+    def test_the_refusal_leaves_the_record_on_disk_alone(self, markers):
+        # Written by someone else while the deploy ran: the refusal must not undo it.
+        old = _node()
+        old.exec_run.return_value = (1, b"")
+        old.attrs["Mounts"] = []
+        real = orchestrator.read_node_identity
+
+        def read_and_race(container, slug):
+            orchestrator._remember_identity("cashpilot-mysterium", "MISMATCH written meanwhile")
+            return real(container, slug)
+
+        with (
+            patch.object(orchestrator, "read_node_identity", side_effect=read_and_race),
+            pytest.raises(orchestrator.ContainerRuntimeError, match="not touched"),
+        ):
+            _raw_deploy(old=old)
+        assert orchestrator._identity_snapshot("cashpilot-mysterium") == "MISMATCH written meanwhile"
+
+    def test_the_refusal_keeps_a_hold_set_while_it_ran(self, markers):
+        orchestrator._MISMATCH_HELD["cashpilot-mysterium"] = "the older reason"
+        old = _node()
+        old.exec_run.return_value = (1, b"")
+        old.attrs["Mounts"] = []
+        real = orchestrator.read_node_identity
+
+        def read_and_race(container, slug):
+            orchestrator._MISMATCH_HELD["cashpilot-mysterium"] = "the newer reason"
+            return real(container, slug)
+
+        with (
+            patch.object(orchestrator, "read_node_identity", side_effect=read_and_race),
+            pytest.raises(orchestrator.ContainerRuntimeError, match="not touched"),
+        ):
+            _raw_deploy(old=old)
+        assert orchestrator._MISMATCH_HELD["cashpilot-mysterium"] == "the newer reason"
+
     def test_control_a_stopped_node_is_replaced_without_the_check(self, markers):
         old = _node(status="exited")
         old.exec_run.return_value = (1, b"")
@@ -1668,6 +1715,38 @@ def test_a_recorded_range_in_the_ephemeral_range_is_flagged_before_the_redeploy(
         assert "Before you redeploy" in warning and "below 32768 other than the default" in warning
         assert "23100:23129" in warning
 
+    def test_the_warning_offers_reserving_the_range_on_the_host_instead(self):
+        warning = drift.move_advice(MYST, _shape(network_mode="host"), HOST_RECORD, lan_fence.MODE_ON)["warning"]
+        assert "keep this range and reserve it on the host" in warning
+        assert "sudo sysctl -w net.ipv4.ip_local_reserved_ports=56000-56100" in warning
+        # systemd-sysctl skips a file without .conf, so the file is named.
+        assert "net.ipv4.ip_local_reserved_ports = 56000-56100 in /etc/sysctl.d/90-mysterium-ports.conf" in warning
+        # Read before the write: sysctl -w replaces the whole list.
+        assert warning.index("First read sysctl net.ipv4.ip_local_reserved_ports") < warning.index("sudo sysctl -w")
+        assert "add 56000-56100 to them, comma-separated" in warning
+        # The kernel keeps it from automatic picks only; an explicit bind still takes the port.
+        assert "stops automatic picks only" in warning and "binds one of these ports by number" in warning
+        # The worker cannot read the host's reservation, so it says when to ignore the warning.
+        assert warning.endswith("If you've reserved this range on the host, you can redeploy.")
+
+    def test_the_reservation_names_the_recorded_range_not_a_fixed_one(self):
+        record = {**HOST_RECORD, "env": {"UDP_PORTS": "40000:40020"}}
+        warning = drift.move_advice(MYST, _shape(network_mode="host"), record, lan_fence.MODE_ON)["warning"]
+        assert "ip_local_reserved_ports=40000-40020" in warning and "56000" not in warning
+
+    def test_the_guide_and_the_upgrade_notes_give_the_same_reservation(self):
+        for doc in ("docs/guides/mysterium.md", "UPGRADING.md"):
+            text = (ROOT / doc).read_text(encoding="utf-8")
+            assert "sysctl -w net.ipv4.ip_local_reserved_ports=56000-56100" in text, doc
+            assert "net.ipv4.ip_local_reserved_ports = 56000-56100" in text, doc
+            assert "/etc/sysctl.d/90-mysterium-ports.conf" in text, doc
+            flat = " ".join(text.split())
+            assert flat.index("read `sysctl net.ipv4.ip_local_reserved_ports`") < flat.index("sudo sysctl -w"), doc
+
+    def test_the_guide_no_longer_says_the_node_runs_on_the_host_network(self):
+        guide = (ROOT / "docs" / "guides" / "mysterium.md").read_text(encoding="utf-8")
+        assert "--net host" not in guide
+
     def test_control_a_range_below_it_or_none_on_record_is_not(self):
         for record in (NEW_RECORD, None, {**HOST_RECORD, "env": {}}):
             assert drift.move_advice(MYST, _shape(network_mode="host"), record, lan_fence.MODE_ON)["warning"] is None

tests/test_preflight.py

@@ -221,7 +221,7 @@ def test_the_flag_is_not_set_on_services_merely_wanting_a_residential_ip(self):
 class TestSourcedPerIpLimits:
     """Values must trace to a provider statement; see docs/research/per-ip-device-limits.md."""
 
-    @pytest.mark.parametrize("slug", ["honeygain", "iproyal", "spide", "earnfm", "ebesucher"])
+    @pytest.mark.parametrize("slug", ["honeygain", "iproyal", "spide", "earnfm", "ebesucher", "mysterium"])
     def test_the_documented_services_declare_one_per_ip(self, slug):
         """spide was MISSING from this list, and that is how the bug shipped.
 
@@ -269,6 +269,14 @@ def test_services_with_no_provider_source_stay_absent(self, slug):
         assert svc is not None, f"{slug} is not in the catalog"
         assert "devices_per_ip" not in (svc.get("requirements") or {})
 
+    def test_a_second_mystnodes_on_one_machine_earns_nothing(self):
+        """MystNodes shows every node but one per public IP offline; it read as unlimited."""
+        from app import catalog
+
+        out = preflight.assess(catalog.get_service("mysterium"), already_deployed_slugs={"mysterium"})
+        assert out["verdict"] == preflight.EARNS_NOTHING
+        assert "one device per IP" in " ".join(f["message"] for f in out["findings"])
+
     def test_a_second_device_behind_one_ip_now_gets_the_strongest_verdict(self):
         """The point of the bead: the verdict was weak exactly where the
         mistake is most likely, because the limit was never recorded."""