diff --git a/UPGRADING.md b/UPGRADING.md index d4ca2059..44edcec5 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -46,19 +46,28 @@ drift, and **Check isolation** fails on it. worker refuses to redeploy Mysterium, since on Docker's default bridge the other earners could reach its WebUI; the running node is left as it is. On your own network (`CASHPILOT_CONTAINER_NETWORK`) it runs under your rules. -3. Redeploy Mysterium once from the dashboard. It keeps its data volume or +3. Move the UDP range below 32768 first: set `UDP_PORTS` to a + free range below 32768 other than the default, such as `23100:23129`, + and change the router forward to match. A value equal to the default + (`23000:23029`) reads as a field left as it was, so the range on record + would stay; to take the default itself, remove the service from that + machine (its data volume stays) and deploy again. + The worker publishes the range 1:1 through Docker, and a node's range on + record (`56000:56100` through 2.0.6, `56000:56029` in 2.0.7 and 2.1.0) sits in + Linux's ephemeral range, 32768-60999, where any program on the host can + hold a port for a moment. One taken port fails the whole start. The + worker then puts the old node back and names the port, so nothing is + lost, but the node does not move. A node that keeps its old range still + works while no host program holds one of its ports. 30 ports also save + about 175 MB of memory in Docker's port proxy against 101. UPnP no longer + works for the node, so forward the range by hand if you relied on it. +4. Redeploy Mysterium once from the dashboard. It keeps its data volume or data folder, so it keeps its identity and its earnings: the worker reads the identity from the running node before it removes it, compares it with the new one's, and stops the new node if they differ. That needs both reads, so back up the keystore first anyway ([Backing up node identities](docs/backup.md)) and check the identity after (`docker exec cashpilot-mysterium wget -qO- http://127.0.0.1:4050/identities`). -4. Nothing changes on the router: the worker publishes the UDP range 1:1. The - default range is now `56000:56029`, inside the old `56000:56100`, so the - old forward keeps working. A node with `56000:56100` on record keeps it; - setting `UDP_PORTS` to `56000:56029` before the redeploy saves about 175 MB - of memory in Docker's port proxy. UPnP no longer works for the node, so - forward the range by hand if you relied on it. **If you reach the WebUI through a reverse proxy.** `UI_ADDRESS` is now the host addresses port 4449 is published on. A Docker network gateway in it is diff --git a/app/lan_fence.py b/app/lan_fence.py index 0f26f846..f27074fa 100644 --- a/app/lan_fence.py +++ b/app/lan_fence.py @@ -776,7 +776,8 @@ def reattach(client: Any, net: Any) -> list[str]: try: net.disconnect(c, force=True) net.connect(c) - if "network" in error and "not found" in error: + # A fence-gated one is started by the guard alone, after all its checks. + if "network" in error and "not found" in error and not _is_gated(c): c.start() moved.append(c.name) except Exception as exc: # noqa: BLE001 - logged; the next check tries again @@ -786,6 +787,13 @@ def reattach(client: Any, net: Any) -> list[str]: return moved +def _is_gated(container: Any) -> bool: + from app.constants import LABEL_SERVICE + + slug = str((getattr(container, "labels", None) or {}).get(LABEL_SERVICE) or "") + return _orchestrator()._gated_container(slug, container) + + def _subnet(net: Any) -> str: for conf in ((net.attrs or {}).get("IPAM") or {}).get("Config") or []: subnet = conf.get("Subnet") or "" @@ -1158,12 +1166,14 @@ def guard() -> dict[str, str]: """ orch = _orchestrator() held: dict[str, str] = {} + scanned = False try: if not orch.docker_available(): _state["unfenced"] = [] return {} client = orch._get_client() _state["unfenced"] = _unfenced(client) + scanned = True gated = _gated(client) if not gated: _state["guard"] = {} @@ -1190,27 +1200,81 @@ def guard() -> dict[str, str]: with contextlib.suppress(RuntimeError, IsolationConflict): apply_fence(client) stopped = orch.user_stopped() + failed = orch.deploy_failed() for c in gated: running = c.status == "running" and c.name not in stopped_now - problem = fence or orch.gated_spec_problem(c, mode) or (None if deploying else orch.identity_problem(c)) + problem = ( + fence + or (FAILED_DEPLOY if c.name in failed else None) + or orch.gated_spec_problem(c, mode) + or (None if deploying else orch.identity_problem(c)) + ) if problem: held[c.name] = problem if running: _guard_stop(c, problem) - elif not deploying and c.status in ("exited", "created") and c.name not in stopped: + elif ( + not deploying + and c.status in ("exited", "created") + and c.name not in stopped + # The old node a deploy moved aside: put back by its roll-back, never by the guard. + and not c.name.endswith(orch.REPLACED_SUFFIX) + ): c.start() - logger.info("Started %s: the LAN fence is confirmed", c.name) + if problem := _started_unattached(c, mode): + # Up with no address on its network: never started again + # by the guard until a deploy succeeds. + held[c.name] = problem + _guard_stop(c, problem) + if not problem.startswith(UNREAD): + orch._mark_failed(c.name, True) + else: + logger.info("Started %s: the LAN fence is confirmed and it is on its network", c.name) finally: if not deploying: orch.GATED_DEPLOY_LOCK.release() except Exception as exc: # noqa: BLE001 - the next pass tries again logger.warning("Fence guard pass failed: %s", exc) - # Not a fresh scan: no Fleet button on a report that may be stale. - _state["unfenced"] = [] + # No Fleet button on a report the scan could not refresh. A later step + # failing (another node's start) leaves a fresh report as it is. + if not scanned: + _state["unfenced"] = [] _state["guard"] = held return held +#: Why the guard holds a container a failed deploy left behind. +FAILED_DEPLOY = "its last deploy failed and left this container behind. Redeploy it from the dashboard" + + +#: A start the guard could not read back: stopped, but tried again next pass. +UNREAD = "it could not be read back after its start" + + +def _started_unattached(container: Any, mode: str | None) -> str | None: + """Why a container just started must stop: not running, or no address on its network. + + Judged in ``on`` and ``operator`` mode, the only ones a fence-gated + container runs in (mode_problem). + """ + if mode not in (MODE_ON, MODE_OPERATOR): + return None + expected = NETWORK_NAME if mode == MODE_ON else _orchestrator()._CONTAINER_NETWORK + try: + container.reload() + except Exception as exc: # noqa: BLE001 - not confirmed is not attached + return f"{UNREAD} ({exc})" + attrs = container.attrs if isinstance(getattr(container, "attrs", None), dict) else {} + state = attrs.get("State") if isinstance(attrs.get("State"), dict) else {} + nets = ((attrs.get("NetworkSettings") or {}).get("Networks") or {}) if isinstance(attrs, dict) else {} + address = str(((nets.get(expected) or {}) if isinstance(nets, dict) else {}).get("IPAddress") or "") + if state and not state.get("Running"): + return "it exited right after its start" + if not address: + return f"it started with no address on {expected}, so it has no network" + return None + + def _guard_stop(container: Any, reason: str) -> None: from app.constants import LABEL_SERVICE diff --git a/app/orchestrator.py b/app/orchestrator.py index 5087fbeb..67a604cd 100644 --- a/app/orchestrator.py +++ b/app/orchestrator.py @@ -825,7 +825,8 @@ def deploy_raw( # A mode that cannot run it at all (off, legacy) is refused here too, # while the running node is still running. - if problem := lan_fence.mode_problem(lan_fence.resolve_mode(client), service_name(slug)): + gated_mode = lan_fence.resolve_mode(client) + if problem := lan_fence.mode_problem(gated_mode, service_name(slug)): raise ContainerNetworkError(f"{slug} was not deployed: {problem} The running container was not touched.") # The pull is on this side of the line for the same reason: a registry outage @@ -851,7 +852,15 @@ def deploy_raw( entrypoint = _get_entrypoint(slug) _refuse_init_image(image, image_config, entrypoint, command) - # Remove any existing container with the same name + # Replace any existing container with the same name. It is stopped and + # moved aside, not removed, until its replacement runs: Docker can still + # refuse the new one (a host port another program holds), and then the old + # one goes back as it was (_roll_back). + replaced = None + replaced_running = False + identity_before = _identity_snapshot(name) + held_before = _MISMATCH_HELD.get(name) + stopped_before = name in user_stopped() try: old = client.containers.get(name) # The name alone is not proof the container is this copy of this @@ -865,6 +874,8 @@ def deploy_raw( f"{name} already belongs to {old_slug} (copy {_instance_of(old)}), not to copy {instance} of " f"{slug}. It was left running; remove it or deploy under another name." ) + if problem := _aside_taken(client, old, name, slug): + raise ContainerRuntimeError(problem) volumes, kept = _keep_live_mounts(old, volumes, moved_mounts, keep_targets) for entry in kept: logger.warning( @@ -913,12 +924,36 @@ def deploy_raw( "without the check." ) _remember_identity(name, identity) + replaced_running = getattr(old, "status", "") == "running" try: old.stop(timeout=_get_stop_timeout(slug)) - except APIError as exc: + except NotFound: + raise + except Exception as exc: # noqa: BLE001 - a timeout is no proof it stopped either logger.warning("Could not stop %s cleanly before replacing it: %s", name, exc) - logger.info("Removing existing container %s", name) - _remove_container(old, slug) + if _still_running(old): + # Its replacement would run beside it on the same identity volume. + _restore_identity(name, identity_before, held_before) + raise ContainerRuntimeError( + f"{name} could not be stopped ({exc}), so it was not replaced and is still running. " + "Try again in a minute.", + 409, + ) from exc + try: + replaced = _move_aside(client, old, name, slug) + except NotFound: + raise + except Exception as exc: # noqa: BLE001 - whatever failed, the old node must run again + # A rename that timed out may still have happened: give it its name back. + with contextlib.suppress(Exception): + old.reload() + if str((old.attrs or {}).get("Name") or "").lstrip("/") != name: + old.rename(name) + _restore_identity(name, identity_before, held_before) + restored = "The previous container was not replaced and " + _start_again( + client, old, replaced_running, gated, name + ) + raise _deploy_failure(exc, restored, slug) from exc except NotFound: _ensure_volumes(client, volumes) @@ -1006,18 +1041,243 @@ def deploy_raw( runtime=runtime or None, **resource_kwargs, ) - container = client.containers.create(**run_kwargs) if gated else client.containers.run(detach=True, **run_kwargs) - + created = None + try: + container = ( + client.containers.create(**run_kwargs) if gated else client.containers.run(detach=True, **run_kwargs) + ) + created = container + if gated: + _mark_stopped(name, False) + # Left created, not started, when the fence is not confirmed: the guard + # starts it as soon as it is, and nothing runs before that. + _require_fence(slug) + container.start() + from app import lan_fence + + if problem := lan_fence._started_unattached(container, gated_mode): + raise ContainerRuntimeError(f"{name} did not come up on its network: {problem}.") + except ContainerNetworkError: + # The fence went between the checks above and the start: the new node is + # created and the guard starts it once the fence reads complete. + if gated: + _mark_failed(name, False) + _remove_replaced(replaced, name, slug) + raise + except Exception as exc: + restored = _roll_back( + client, + name, + slug, + replaced, + replaced_running, + identity_before, + stopped_before, + held_before=held_before, + created=created, + gated=gated, + ) + raise _deploy_failure(exc, restored, slug) from exc if gated: - _mark_stopped(name, False) - # Left created, not started, when the fence is not confirmed: the guard - # starts it as soon as it is, and nothing runs before that. - _require_fence(slug) - container.start() + _mark_failed(name, False) + _remove_replaced(replaced, name, slug) logger.info("Container %s started: %s", name, container.short_id) return container.id +#: Suffix of a container moved aside while its replacement is created. +REPLACED_SUFFIX = "-replaced" + + +def _move_aside(client: Any, old: Any, name: str, slug: str) -> Any | None: + """Rename the stopped ``old`` out of its name; remove it if it cannot be renamed. The renamed one, or None.""" + aside = f"{name}{REPLACED_SUFFIX}" + with contextlib.suppress(NotFound): + # One left by a deploy that did not finish is older than ``old``. + leftover = client.containers.get(aside) + if leftover is not old and getattr(leftover, "id", None) != getattr(old, "id", None): + _remove_container(leftover, slug) + # A rename Docker refuses raises: the old one keeps its name and is never + # removed before its replacement runs. + old.rename(aside) + logger.info("Moved %s aside as %s until its replacement runs", name, aside) + return old + + +def _aside_taken(client: Any, old: Any, name: str, slug: str) -> str | None: + """Why the name ``old`` would be moved aside to belongs to another container, or None.""" + aside = f"{name}{REPLACED_SUFFIX}" + try: + leftover = client.containers.get(aside) + except NotFound: + return None + if leftover is old or getattr(leftover, "id", None) == getattr(old, "id", None): + return None + labels = leftover.labels if isinstance(getattr(leftover, "labels", None), dict) else {} + if labels.get(LABEL_SERVICE) == slug and _instance_of(leftover) == _instance_of(old): + return None + return ( + f"{name} was not replaced: {aside}, the name it is moved aside to while its replacement starts, belongs to " + f"{labels.get(LABEL_SERVICE) or 'a container CashPilot did not deploy'}. The running container was not " + "touched; remove or rename that container, then redeploy." + ) + + +def _still_running(container: Any) -> bool: + try: + container.reload() + except NotFound: + return False + except Exception: # noqa: BLE001 - not known stopped is treated as running + return True + return getattr(container, "status", "") == "running" + + +def _remove_replaced(replaced: Any | None, name: str, slug: str) -> None: + """Remove the old container once its replacement is in place; a failure leaves a leftover only.""" + if replaced is None: + return + try: + _remove_container(replaced, slug) + except Exception as exc: # noqa: BLE001 - the guard never starts it, and the next deploy removes it + logger.warning("Could not remove %s%s after its replacement was created: %s", name, REPLACED_SUFFIX, exc) + + +def _start_again(client: Any, old: Any, was_running: bool, gated: bool, name: str) -> str: + """Start the previous container again if it ran, a gated one only behind a complete fence. Says how it is.""" + if not was_running: + return "is stopped as it was." + if gated and not predates_fence(old): + from app import lan_fence + + try: + fence = lan_fence.fence_problem(client) + except Exception as exc: # noqa: BLE001 - not confirmed is not complete + fence = f"the fence could not be checked ({exc})" + if fence: + logger.warning("Not starting %s again yet: %s", name, fence) + return f"is kept stopped until the LAN fence is complete again ({fence}); the worker starts it then." + try: + old.start() + except Exception as exc: # noqa: BLE001 - reported to the caller + logger.warning("Could not start %s again: %s", name, exc) + return f"could not be started again ({exc})." + return "is running again." + + +def _roll_back( + client: Any, + name: str, + slug: str, + replaced: Any | None, + was_running: bool, + identity_before: str | None, + stopped_before: bool, + *, + held_before: str | None = None, + created: Any | None = None, + gated: bool, +) -> str: + """Undo a deploy whose new container could not be created or started. Says what was put back. + + The half-made container goes, the old one gets its name back and runs + again if it ran, and the identity and stop records are as they were. A + leftover that cannot be removed is marked, so the guard never starts it. + """ + leftover_gone = True + taken = False + try: + current = client.containers.get(name) + # Only what this deploy made: its own container, or (when Docker + # failed inside the create-and-start call) one that is not running. + # A running one there is another deploy's, and it is left alone. + if created is not None: + ours = getattr(current, "id", None) == getattr(created, "id", None) + else: + ours = getattr(current, "status", "") != "running" + if ours: + _remove_container(current, slug) + else: + leftover_gone, taken = False, True + except NotFound: + pass + except Exception as exc: # noqa: BLE001 - reported below + leftover_gone = False + logger.warning("Could not remove the container the failed deploy of %s left: %s", name, exc) + _restore_identity(name, identity_before, held_before) + _mark_stopped(name, stopped_before) + if gated and not leftover_gone and not taken: + _mark_failed(name, True) + if replaced is None: + return "No container ran under that name before, so nothing was put back." + if taken: + return ( + f"Another deploy took the name meanwhile, so the previous container is kept, stopped, as " + f"{name}{REPLACED_SUFFIX}." + ) + if not leftover_gone: + return ( + f"The previous container is kept, stopped, as {name}{REPLACED_SUFFIX}: the failed one still holds " + "the name. Remove it, then redeploy." + ) + try: + replaced.rename(name) + except Exception as exc: # noqa: BLE001 - reported to the caller + logger.warning("Could not put %s back: %s", name, exc) + return f"The previous container could not be put back ({exc}); it is kept as {name}{REPLACED_SUFFIX}." + logger.info("Put the previous %s back after a failed deploy", name) + return "The previous container was put back and " + _start_again(client, replaced, was_running, gated, name) + + +def _deploy_failure(exc: BaseException, restored: str, slug: str = "") -> ContainerRuntimeError: + """The error a failed create or start answers with, after the roll-back.""" + if isinstance(exc, ContainerRuntimeError): + return ContainerRuntimeError(f"{exc} {restored}", exc.status_code) + refusal = proxy_refusal(exc) if isinstance(exc, APIError) else None + if refusal: + return ContainerRuntimeError(f"{refusal} {restored}", 403) + if isinstance(exc, APIError): + reason = " ".join(str(exc.explanation or exc).split())[:300] + return ContainerRuntimeError( + f"Docker refused the container: {reason}{_ephemeral_hint(reason, slug)} {restored}" + ) + return ContainerRuntimeError(f"The container could not be created: {exc}. {restored}", 500) + + +#: Linux's default ephemeral range (net.ipv4.ip_local_port_range). +EPHEMERAL_PORTS = (32768, 60999) + + +def _ephemeral_hint(reason: str, slug: str = "") -> str: + """Why a taken UDP port in Linux's ephemeral range failed the start, and what to do. + + The setting to change and its default come from the service's catalog + entry (``udp_port_range``); a service without one gets the general advice. + """ + if "already" not in reason: + return "" + for port in re.findall(r":(\d+)/udp", reason): + if EPHEMERAL_PORTS[0] <= int(port) <= EPHEMERAL_PORTS[1]: + why = ( + f" UDP port {port} is in Linux's ephemeral range ({EPHEMERAL_PORTS[0]}-{EPHEMERAL_PORTS[1]}), where " + "any program on the host can hold a port for a moment, and one taken port fails the whole start." + ) + docker_conf = ((get_service(slug) if get_service and slug else None) or {}).get("docker") or {} + rng = docker_conf.get("udp_port_range") + key = str(rng.get("env") or "") if isinstance(rng, dict) else "" + if not key: + return why + f" Publish it on a port below {EPHEMERAL_PORTS[0]} instead." + default = next( + (str(v.get("default") or "") for v in docker_conf.get("env") or [] if v.get("key") == key), "" + ) + shown = f" (the default is {default})" if default else "" + return why + ( + f" Set {key} to a range below {EPHEMERAL_PORTS[0]}{shown} and forward that range on your router " + "instead." + ) + return "" + + def _named_volumes(volumes: dict[str, Any] | list[str] | None) -> list[str]: """The named volumes (not host folders) a deploy mounts.""" if not volumes: @@ -1192,6 +1452,49 @@ def _mark_stopped(name: str, stopped: bool, *, strict: bool = False) -> None: logger.warning("Could not record that %s was %s: %s", name, "stopped" if stopped else "started", exc) +#: Fence-gated containers a failed deploy left that could not be removed. The +#: guard never starts one; the next deploy that succeeds clears it. +FAILED_MARKER = os.path.join(os.getenv("CASHPILOT_DATA_DIR", "/data"), ".fence_gated_failed") + + +#: The last mark the file could not take, by container name. A deploy that +#: worked must lift the mark even when /data cannot be written, or the guard +#: would keep stopping the node the owner just redeployed. +_FAILED_UNWRITTEN: dict[str, bool] = {} + + +def _failed_on_disk() -> set[str]: + try: + with open(FAILED_MARKER, encoding="utf-8") as fh: + return {line.strip() for line in fh if line.strip()} + except OSError: + return set() + + +def deploy_failed() -> set[str]: + names = _failed_on_disk() + for name, failed in _FAILED_UNWRITTEN.items(): + names = names | {name} if failed else names - {name} + return names + + +def _mark_failed(name: str, failed: bool) -> None: + names = _failed_on_disk() + if (name in names) == failed: + _FAILED_UNWRITTEN.pop(name, None) + return + names = names | {name} if failed else names - {name} + try: + with open(FAILED_MARKER, "w", encoding="utf-8") as fh: + fh.write("".join(f"{n}\n" for n in sorted(names))) + _FAILED_UNWRITTEN.pop(name, None) + except OSError as exc: + _FAILED_UNWRITTEN[name] = failed + logger.warning( + "Could not record that the last deploy of %s %s: %s", name, "failed" if failed else "worked", exc + ) + + def predates_fence(container: Any) -> bool: """A fence-gated service's container from 2.0.6 or earlier: on the host's network. @@ -1283,6 +1586,37 @@ def _identity_file(name: str) -> str: _MISMATCH_HELD: dict[str, str] = {} +def _identity_snapshot(name: str) -> str | None: + """The identity record exactly as it is, or None when there is none.""" + try: + with open(_identity_file(name), encoding="utf-8") as fh: + return fh.read() + except OSError: + return None + + +def _restore_identity(name: str, snapshot: str | None, held: str | None = None) -> None: + """Put the identity record back as _identity_snapshot found it, and a hold kept only in memory.""" + path = _identity_file(name) + try: + if snapshot is None: + if os.path.exists(path): + os.remove(path) + else: + os.makedirs(IDENTITY_DIR, exist_ok=True) + with open(path, "w", encoding="utf-8") as fh: + fh.write(snapshot) + except OSError as exc: + logger.warning("Could not restore the identity record of %s: %s", name, exc) + word, _, reason = (snapshot or "").strip().partition(" ") + if held is not None: + _MISMATCH_HELD[name] = held + elif word == "MISMATCH": + _MISMATCH_HELD[name] = reason + else: + _MISMATCH_HELD.pop(name, None) + + def _pending_identity(name: str) -> str | None: """The identity a replacement still has to match, or None (no record, or a held mismatch).""" try: @@ -1377,7 +1711,11 @@ def _require_startable(container: Any, slug: str) -> None: mode = lan_fence.resolve_mode(_get_client()) except RuntimeError as exc: raise ContainerNetworkError(f"{slug} was not started: the isolation mode could not be read ({exc})") from None - problem = gated_spec_problem(container, mode) or identity_problem(container) + problem = ( + (lan_fence.FAILED_DEPLOY if getattr(container, "name", None) in deploy_failed() else None) + or gated_spec_problem(container, mode) + or identity_problem(container) + ) if problem: raise ContainerNetworkError(f"{slug} was not started: {problem}") @@ -1422,8 +1760,41 @@ def gated_spec_problem(container: Any, mode: str | None = None) -> str | None: if mode in (lan_fence.MODE_ON, lan_fence.MODE_OPERATOR): settings = attrs.get("NetworkSettings") if isinstance(attrs.get("NetworkSettings"), dict) else {} attached = settings.get("Networks") if isinstance(settings.get("Networks"), dict) else {} + if network_mode not in attached: + return f"it is not attached to {network_mode}, so it would start with no network" if extra := sorted(set(attached) - {network_mode}): return f"it is also attached to {', '.join(extra)}, where the fence does not cover it" + bindings = host.get("PortBindings") if isinstance(host.get("PortBindings"), dict) else {} + ui = docker_conf.get("ui_port") + if isinstance(ui, dict) and f"{ui.get('port')}/tcp" not in bindings: + return f"it does not publish its WebUI port {ui.get('port')}" + rng = docker_conf.get("udp_port_range") + if isinstance(rng, dict): + wanted = _env_port_range(config, str(rng.get("env") or "")) + if wanted is None and not any(k.endswith("/udp") for k in bindings): + return "it publishes none of its UDP ports" + if wanted is not None: + missing = [p for p in range(wanted[0], wanted[1] + 1) if f"{p}/udp" not in bindings] + if missing: + return ( + f"it does not publish UDP {missing[0]}" + + (f" and {len(missing) - 1} more" if len(missing) > 1 else "") + + f" of its range {wanted[0]}:{wanted[1]}" + ) + return None + + +def _env_port_range(config: dict[str, Any], key: str) -> tuple[int, int] | None: + """The start:end range the container's own env sets in ``key``, or None when it sets none it can read.""" + if not key: + return None + for entry in config.get("Env") or []: + name, _, value = str(entry).partition("=") + if name == key: + m = re.fullmatch(r"\s*(\d{1,5}):(\d{1,5})\s*", value) + if m and int(m[1]) <= int(m[2]) <= 65535: + return int(m[1]), int(m[2]) + return None return None diff --git a/docs/guides/mysterium.md b/docs/guides/mysterium.md index a3a6f724..c695a796 100644 --- a/docs/guides/mysterium.md +++ b/docs/guides/mysterium.md @@ -64,20 +64,20 @@ In the CashPilot web UI, open **Catalog** and click **Add** on the **MystNodes** | Variable | Default | What it does | |---|---|---| | `UI_ADDRESS` | `127.0.0.1` | The host addresses the node's WebUI (port 4449) is published on. Comma separated, no spaces. | -| `UDP_PORTS` | `56000:56029` | The UDP ports the node uses for sessions, as `start:end`. Forward the same range. To go back to the default after changing it, remove the service from that machine (its data volume stays) and deploy again. | +| `UDP_PORTS` | `23000:23029` | The UDP ports the node uses for sessions, as `start:end`. Forward the same range. Keep it below 32768 (see *Port forwarding*). To go back to the default after changing it, remove the service from that machine (its data volume stays) and deploy again. | ### Port forwarding Each session first tries to reach the node directly on its UDP range, and falls back to hole punching when it cannot, which many buyers fail at. The node uses -only the range in `UDP_PORTS` (default `56000:56029`), so forward exactly that +only the range in `UDP_PORTS` (default `23000:23029`), so forward exactly that range. Create one rule on your router: | Field | Value | |---|---| | Protocol | UDP | -| External ports | 56000-56029 (your `UDP_PORTS`) | -| Internal ports | the same, 56000-56029 | +| External ports | 23000-23029 (your `UDP_PORTS`) | +| Internal ports | the same, 23000-23029 | | Internal IP | the LAN address of the machine running the node | Give that machine a fixed LAN address (a DHCP reservation), or the rule points @@ -86,18 +86,23 @@ the LAN fence, which drops the router, so it cannot ask the router for a forward. Forward the range by hand. DMZ is the last resort. The node runs on CashPilot's fenced bridge, and the worker publishes the range -1:1 on the machine (`56000/udp` to `56000/udp`, and so on), so the forward on +1:1 on the machine (`23000/udp` to `23000/udp`, and so on), so the forward on the router is the same as when the node shared the host's network. +Keep the range below 32768. Ports 32768 to 60999 are Linux's ephemeral range: any program on the machine (a DNS lookup, a game client, a browser) can hold one of them for a moment. On the host's network the node just skipped a taken port, but Docker publishes the range as a whole, so one taken port fails the node's start. The worker then puts the previous node back and says which port was taken. + Each session takes about two ports, so the default 30 ports carry about 15 sessions at once, more than a home node gets. Every published port costs about 2.5 MB of memory on the machine (Docker starts two small proxy processes per port; 101 ports measured about 250 MB), so a wider range only costs more. A -range of 20 to 30 ports is enough. Through 2.0.6 the default was -`56000:56100`: a router forward for that range keeps working, since the new -default is inside it. A node that has `56000:56100` on record keeps it on a -redeploy; to save the memory, set `UDP_PORTS` to `56000:56029` and redeploy, -with no change on the router. +range of 20 to 30 ports is enough. The default was `56000:56100` through +2.0.6 and `56000:56029` in 2.0.7 and 2.1.0, both inside the ephemeral range. A node keeps +the range it has on record on a redeploy. To move it, set `UDP_PORTS` to a +free range below 32768 other than the default, such as `23100:23129`, change +the router forward to match, and redeploy. A value equal to the default reads +as a field left as it was, so the recorded range would stay. To take the +default itself, remove the service from that machine (its data volume stays) +and deploy again. **Check it.** After a few sessions, on the machine running the node: @@ -239,7 +244,7 @@ different one starts a new node identity. ```bash docker create --name cashpilot-mysterium \ --network cashpilot-isolated --restart no --runtime runc \ - -p 127.0.0.1:4449:4449 -p 56000-56029:56000-56029/udp \ + -p 127.0.0.1:4449:4449 -p 23000-23029:23000-23029/udp \ --cap-drop ALL --cap-add NET_ADMIN --cap-add SETUID --cap-add SETGID \ --device /dev/net/tun \ --security-opt no-new-privileges:true \ @@ -247,7 +252,7 @@ docker create --name cashpilot-mysterium \ mysteriumnetwork/myst:1.39.9-alpine \ --ui.address=0.0.0.0 --tequilapi.address=127.0.0.1 \ --firewall.protected.networks=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,127.0.0.0/8,100.64.0.0/10,169.254.0.0/16 \ - --udp.ports=56000:56029 \ + --udp.ports=23000:23029 \ service --agreed-terms-and-conditions ``` diff --git a/docs/isolation.md b/docs/isolation.md index fec0a8ec..5cd9bd10 100644 --- a/docs/isolation.md +++ b/docs/isolation.md @@ -167,9 +167,10 @@ every other earner: its namespace (flush its tables, take a LAN address, add a LAN route), what it sends to your LAN, the host, a tailnet or cloud metadata is dropped on the host. -- Its UDP range (`UDP_PORTS`, 56000:56029 by default, inside the 56000:56100 - default through 2.0.6) is published 1:1, so the forward on your router stays - valid. Each port goes to the same port in +- Its UDP range (`UDP_PORTS`, 23000:23029 by default) is published 1:1, so + the forward on your router is the same as on the host's network. Docker + fails the whole start when a host program holds one of its ports, so the + default sits below Linux's ephemeral range (32768-60999). Each port goes to the same port in the node, with the customer's own source address. At most 1000 ports. - Its WebUI listens on `0.0.0.0` inside its namespace and is published only on the host addresses in `UI_ADDRESS`, `127.0.0.1` by default. The Tequila diff --git a/services/bandwidth/mysterium.yml b/services/bandwidth/mysterium.yml index 39a9a526..8030cf6b 100644 --- a/services/bandwidth/mysterium.yml +++ b/services/bandwidth/mysterium.yml @@ -53,13 +53,17 @@ docker: same range on your router. Each session takes about two ports, so the default 30 carry about 15 sessions at once, more than a home node gets. Every published port costs about 2.5 MB of memory in Docker's port - proxy, so a wider range only costs more. A forward you already have for - a wider range that contains this one (56000-56100) keeps working. - default: "56000:56029" + proxy, so a wider range only costs more. Keep it below 32768: ports + 32768-60999 are Linux's ephemeral range, where any program on the host + can hold one for a moment, and one taken port fails the node's start. + A node that already has a range keeps it. + default: "23000:23029" # Without it the node picks from 10000:60000, so a forwarded range is - # never used and every session falls back to hole punching. 30 ports, a - # sub-range of the 56000:56100 default through 2.0.6, so the router - # forwards made for that one stay valid. + # never used and every session falls back to hole punching. 30 ports, + # below Linux's ephemeral range (32768-60999): Docker publishes the range + # 1:1 and fails the whole start on one port a host process holds, where a + # node on the host's network just skipped it. Through 2.1.0 the default + # was 56000:56029 (56000:56100 through 2.0.6); a range on record is kept. # ASCII digits only, as above; port_range checks start <= end <= 65535. pattern: '[0-9]{4,5}:[0-9]{4,5}' port_range: true @@ -175,7 +179,7 @@ earnings: monthly_high: 10 currency: USD per: "device" - notes: "Earnings in MYST tokens. Residential IPs earn significantly more. Node WebUI at port 4449 (loopback by default) for management. VPS accepted. Port forwarding UDP 56000-56029 (the UDP_PORTS default) strongly recommended -- without it, nodes get Strict NAT status and many VPN sessions fail, severely reducing income. Important: after first run, set your beneficiary (settlement) wallet via the node WebUI or CLI to match your mystnodes.com account -- this links on-chain earnings to your cloud dashboard." + notes: "Earnings in MYST tokens. Residential IPs earn significantly more. Node WebUI at port 4449 (loopback by default) for management. VPS accepted. Port forwarding UDP 23000-23029 (the UDP_PORTS default) strongly recommended -- without it, nodes get Strict NAT status and many VPN sessions fail, severely reducing income. Important: after first run, set your beneficiary (settlement) wallet via the node WebUI or CLI to match your mystnodes.com account -- this links on-chain earnings to your cloud dashboard." cashout: method: auto diff --git a/tests/test_lan_fence.py b/tests/test_lan_fence.py index f1a20e25..d5e68ba9 100644 --- a/tests/test_lan_fence.py +++ b/tests/test_lan_fence.py @@ -803,6 +803,13 @@ def test_an_earner_the_missing_network_stopped_is_moved_and_started(self): net.connect.assert_called_once_with(c) c.start.assert_called_once() + def test_a_fence_gated_one_is_moved_but_left_to_the_guard(self): + net = _network() + c = _orphan("cashpilot-mysterium", "old-id", error=self.GONE) + assert lan_fence.reattach(_docker(net, earners=[c]), net) == ["cashpilot-mysterium"] + net.connect.assert_called_once_with(c) + c.start.assert_not_called() + def test_one_the_owner_stopped_is_moved_but_left_stopped(self): net = _network() c = _orphan("cashpilot-storj", "old-id") diff --git a/tests/test_lan_isolation.py b/tests/test_lan_isolation.py index 63e91c19..1874283e 100644 --- a/tests/test_lan_isolation.py +++ b/tests/test_lan_isolation.py @@ -81,7 +81,7 @@ def test_mysterium_is_fenced_with_its_udp_range_as_the_exception(self): out = li.assess(catalog.get_service("mysterium")) assert out["verdict"] == li.NEEDS_EXCEPTIONS details = " ".join(e["detail"] for e in out["exceptions"]) - assert "56000:56029/udp" in details + assert "23000:23029/udp" in details assert "host network" not in details assert "4449" not in details, "the WebUI is published on loopback by default" diff --git a/tests/test_mysterium_fenced.py b/tests/test_mysterium_fenced.py index 2fb3ab1a..78728deb 100644 --- a/tests/test_mysterium_fenced.py +++ b/tests/test_mysterium_fenced.py @@ -115,11 +115,14 @@ def test_the_default_range_is_30_ports_each_on_itself(self): # About two ports per session: 15 at once, and about 2.5 MB of # docker-proxy memory per port, so a wider default only costs memory. assert len(ports) == 30 - assert ports["56000/udp"] == 56000 and ports["56029/udp"] == 56029 + assert ports["23000/udp"] == 23000 and ports["23029/udp"] == 23029 - def test_the_default_stays_inside_the_range_routers_forward_since_2_0_6(self): + def test_the_default_stays_below_linuxs_ephemeral_range(self): + """Docker fails the whole publish on one port a host program holds, and 32768-60999 is where they take them.""" default = next(e["default"] for e in MYST["docker"]["env"] if e["key"] == "UDP_PORTS") - assert set(catalog.expand_udp_range(default)) <= set(catalog.expand_udp_range("56000:56100")) + assert max(catalog.expand_udp_range(default).values()) < orchestrator.EPHEMERAL_PORTS[0] + # Nor in the Kubernetes NodePort range just under it. + assert max(catalog.expand_udp_range(default).values()) < 30000 @pytest.mark.parametrize("value", ["56100:56000", "1:70000", "56000", "a:b", "56000-56100"]) def test_anything_but_a_range_is_refused(self, value): @@ -272,19 +275,38 @@ def markers(tmp_path, monkeypatch): monkeypatch.setattr(orchestrator, "STOPPED_MARKER", str(tmp_path / "stopped")) monkeypatch.setattr(orchestrator, "IDENTITY_DIR", str(tmp_path / "identity")) monkeypatch.setattr(orchestrator, "_MISMATCH_HELD", {}, raising=False) + monkeypatch.setattr(orchestrator, "FAILED_MARKER", str(tmp_path / "failed"), raising=False) + monkeypatch.setattr(orchestrator, "_FAILED_UNWRITTEN", {}, raising=False) return tmp_path -def _raw_deploy(problem=None, slug="mysterium", old=None, mode=lan_fence.MODE_ON, **kwargs): +def _up_on(network="cashpilot-isolated", running=True, address="172.30.0.2"): + """attrs of a container that came up on ``network`` (inspect after start).""" + return { + "State": {"Running": running}, + "NetworkSettings": {"Networks": {network: {"IPAddress": address}}}, + } + + +def _not_found(name): + raise orchestrator.NotFound(f"no {name}") + + +def _raw_deploy(problem=None, slug="mysterium", old=None, mode=lan_fence.MODE_ON, created=None, client=None, **kwargs): real = lan_fence.mode_problem - created = MagicMock(id="new", short_id="new") - client = MagicMock() - if old is None: - client.containers.get.side_effect = orchestrator.NotFound("nope") - else: - client.containers.get.return_value = old - client.containers.create.return_value = created - client.containers.run.return_value = created + if created is None: + created = MagicMock(id="new", short_id="new") + created.attrs = _up_on( + orchestrator._CONTAINER_NETWORK if mode == lan_fence.MODE_OPERATOR else "cashpilot-isolated" + ) + if client is None: + client = MagicMock() + if old is None: + client.containers.get.side_effect = orchestrator.NotFound("nope") + else: + client.containers.get.side_effect = lambda n: old if n == old.name else _not_found(n) + client.containers.create.return_value = created + client.containers.run.return_value = created with ( patch.object(orchestrator, "_get_client", return_value=client), patch.object(orchestrator, "_CONTAINER_RUNTIME", "runsc"), @@ -369,12 +391,12 @@ def test_the_socket_policy_keeps_host_cgroupns_off(): #: The 30-port default range, published 1:1 as the worker does. -UDP_BINDINGS = {f"{p}/udp": [{"HostIp": "", "HostPort": str(p)}] for p in range(56000, 56030)} +UDP_BINDINGS = {f"{p}/udp": [{"HostIp": "", "HostPort": str(p)}] for p in range(23000, 23030)} def _shape(network_mode="cashpilot-isolated", restart="no", image=PINNED, udp=True): attrs = { - "Config": {"Image": image, "Env": ["UI_ADDRESS=127.0.0.1", "UDP_PORTS=56000:56029"]}, + "Config": {"Image": image, "Env": ["UI_ADDRESS=127.0.0.1", "UDP_PORTS=23000:23029"]}, "HostConfig": { "CapAdd": ["NET_ADMIN", "SETUID", "SETGID"], "NetworkMode": network_mode, @@ -454,9 +476,18 @@ def _node(status="running", network_mode="cashpilot-isolated", caps=("NET_ADMIN" c.labels = dict(LABELS) c.attrs = { "Config": {"Image": image}, - "HostConfig": {"NetworkMode": network_mode, "CapAdd": list(caps), "RestartPolicy": {"Name": "no"}}, + "HostConfig": { + "NetworkMode": network_mode, + "CapAdd": list(caps), + "RestartPolicy": {"Name": "no"}, + "PortBindings": {"4449/tcp": [{"HostIp": "127.0.0.1", "HostPort": "4449"}], **UDP_BINDINGS}, + }, + "NetworkSettings": {"Networks": {network_mode: {"IPAddress": "172.30.0.2"}}}, + "State": {"Running": status == "running"}, } c.exec_run.return_value = (1, b"") + # docker-py's start; a reload then reads it running on its network. + c.start.side_effect = lambda *a, **k: c.attrs["State"].update(Running=True) return c @@ -672,6 +703,15 @@ def _start(node, action="start_service"): class TestStartRefusesAHeldNode: + @pytest.mark.parametrize("action", ["start_service", "restart_service"]) + def test_a_node_a_failed_deploy_left_is_not_started(self, markers, action): + orchestrator._mark_failed("cashpilot-mysterium", True) + node = _node(status="exited") + with pytest.raises(orchestrator.ContainerNetworkError, match="last deploy failed"): + _start(node, action) + node.start.assert_not_called() + node.restart.assert_not_called() + def test_a_node_held_for_another_identity_is_not_started(self, markers): orchestrator._remember_identity("cashpilot-mysterium", "MISMATCH its identity is 0xb, not 0xa") node = _node(status="exited") @@ -953,7 +993,7 @@ def test_drift_names_the_service_and_claims_no_capability_it_lacks(self): def test_drift_flags_a_node_that_publishes_its_webui_but_not_its_range(self): found = drift.container_drift(MYST, _shape(udp=False), None) - assert any("does not publish port" in f and "56000/udp" in f for f in found) + assert any("does not publish port" in f and "23000/udp" in f for f in found) def test_drift_judges_the_range_on_record(self): record = {"env": {"UDP_PORTS": "56000:56100"}} @@ -1118,6 +1158,17 @@ def test_a_failed_scan_drops_the_unfenced_report(markers): assert lan_fence.status()["unfenced"] == [] +def test_a_later_failure_in_the_pass_keeps_a_fresh_unfenced_report(markers): + """Another node failing to start does not hide the move of the one on the host network.""" + old = _old_node() + other = _node(status="exited") + other.name = "cashpilot-mysterium-2" + other.start.side_effect = RuntimeError("the start failed") + _guard([old, other]) + other.start.assert_called_once() + assert [u["name"] for u in lan_fence.status()["unfenced"]] == ["cashpilot-mysterium"] + + def test_the_fleet_page_does_not_offer_a_move_it_just_made(): fleet = (ROOT / "app" / "templates" / "fleet.html").read_text(encoding="utf-8") assert "_movedBehindFence.add(" in fleet @@ -1150,3 +1201,424 @@ def test_a_redeploy_lifts_a_hold_kept_only_in_memory(self, markers): assert "cashpilot-mysterium" not in orchestrator._MISMATCH_HELD # The identity to match is kept, so the restored volume is still checked. assert orchestrator._pending_identity("cashpilot-mysterium") == IDENTITY_A + + +# --------------------------------------------------------------------------- +# (n) A deploy Docker refuses puts the old node back; the guard never starts +# what a failed deploy left; the default range is out of the ephemeral one +# --------------------------------------------------------------------------- + +BIND_REFUSED = "failed to bind host port 0.0.0.0:56033/udp: address already in use" + + +class _FakeDocker: + """Containers by name, with rename and remove, so a roll-back can be followed.""" + + def __init__(self, *containers): + self.by_name = {} + self.client = MagicMock() + self.client.containers.get.side_effect = self.get + self.client.containers.create.side_effect = self.create + self.client.containers.run.side_effect = self.create + self.created = MagicMock(id="new", short_id="new") + self.created.attrs = _up_on() + for c in containers: + self.add(c) + + def add(self, c): + self.by_name[c.name] = c + if not isinstance(getattr(c, "remove_error", None), BaseException): + c.remove_error = None + + def rename(new, c=c): + self.by_name.pop(c.name, None) + c.name = new + self.by_name[new] = c + + def remove(c=c, **_): + if getattr(c, "remove_error", None): + raise c.remove_error + self.by_name.pop(c.name, None) + c.removed = True + + c.rename.side_effect = rename + c.remove.side_effect = remove + c.removed = False + + def get(self, name): + if name in self.by_name: + return self.by_name[name] + raise orchestrator.NotFound(f"no {name}") + + def create(self, **kwargs): + self.created.name = kwargs["name"] + self.add(self.created) + if getattr(self, "create_error", None): + raise self.create_error + return self.created + + +def _running_old(name="cashpilot-mysterium", labels=None, status="running"): + old = MagicMock() + old.name = name + old.status = status + old.labels = dict(labels or LABELS) + old.attrs = {"Mounts": []} + old.exec_run.return_value = _says(IDENTITY_A) + return old + + +def _api_error(text): + return orchestrator.APIError("500 Server Error", explanation=text) + + +class TestARefusedDeployPutsTheOldNodeBack: + def test_a_refused_start_puts_the_running_node_back_and_running(self, markers): + old = _running_old() + docker = _FakeDocker(old) + docker.created.start.side_effect = _api_error(BIND_REFUSED) + with pytest.raises(orchestrator.ContainerRuntimeError) as refused: + _raw_deploy(client=docker.client, old=old) + message = str(refused.value) + assert "56033/udp" in message and "ephemeral range" in message + assert "put back and is running again" in message + assert docker.get("cashpilot-mysterium") is old and not old.removed + old.start.assert_called_once() + assert docker.created.removed + + def test_the_identity_record_is_as_it_was(self, markers): + orchestrator._remember_identity("cashpilot-mysterium", "MISMATCH its identity is 0xb, not 0xa") + before = orchestrator._identity_snapshot("cashpilot-mysterium") + old = _running_old(status="exited") + docker = _FakeDocker(old) + docker.created.start.side_effect = _api_error(BIND_REFUSED) + with pytest.raises(orchestrator.ContainerRuntimeError): + _raw_deploy(client=docker.client, old=old) + assert orchestrator._identity_snapshot("cashpilot-mysterium") == before + assert "0xb" in orchestrator.identity_problem(_node(status="exited")) + + def test_a_hold_kept_only_in_memory_comes_back(self, markers): + # The record says which identity to match; the hold could not be written. + orchestrator._remember_identity("cashpilot-mysterium", IDENTITY_A) + orchestrator._MISMATCH_HELD["cashpilot-mysterium"] = "its identity is 0xb, not 0xa" + old = _running_old(status="exited") + docker = _FakeDocker(old) + docker.created.start.side_effect = _api_error(BIND_REFUSED) + with pytest.raises(orchestrator.ContainerRuntimeError): + _raw_deploy(client=docker.client, old=old) + assert orchestrator._pending_identity("cashpilot-mysterium") == IDENTITY_A + assert "0xb" in orchestrator.identity_problem(_node(status="exited")) + + def test_a_record_the_deploy_wrote_is_taken_back(self, markers): + old = _running_old() + docker = _FakeDocker(old) + docker.created.start.side_effect = _api_error(BIND_REFUSED) + with pytest.raises(orchestrator.ContainerRuntimeError): + _raw_deploy(client=docker.client, old=old) + assert orchestrator._identity_snapshot("cashpilot-mysterium") is None + + def test_a_refused_create_puts_it_back_too(self, markers): + old = _running_old() + docker = _FakeDocker(old) + docker.create_error = _api_error("Conflict. The container name is already in use") + with pytest.raises(orchestrator.ContainerRuntimeError, match="put back"): + _raw_deploy(client=docker.client, old=old) + assert docker.get("cashpilot-mysterium") is old + old.start.assert_called_once() + + def test_a_node_the_owner_had_stopped_goes_back_stopped(self, markers): + orchestrator._mark_stopped("cashpilot-mysterium", True) + old = _running_old(status="exited") + docker = _FakeDocker(old) + docker.created.start.side_effect = _api_error(BIND_REFUSED) + with pytest.raises(orchestrator.ContainerRuntimeError, match="stopped as it was"): + _raw_deploy(client=docker.client, old=old) + old.start.assert_not_called() + assert "cashpilot-mysterium" in orchestrator.user_stopped() + + def test_a_new_node_with_no_network_is_rolled_back(self, markers): + old = _running_old() + docker = _FakeDocker(old) + docker.created.attrs = _up_on(address="") + with ( + patch("app.lan_fence.resolve_mode", return_value=lan_fence.MODE_ON), + patch("app.lan_fence.network_for_deploy", return_value="cashpilot-isolated"), + pytest.raises(orchestrator.ContainerRuntimeError, match="no address on cashpilot-isolated"), + ): + _raw_deploy(client=docker.client, old=old) + assert docker.get("cashpilot-mysterium") is old + + def test_a_leftover_that_cannot_be_removed_is_marked_and_the_old_one_kept_aside(self, markers): + old = _running_old() + docker = _FakeDocker(old) + docker.created.start.side_effect = _api_error(BIND_REFUSED) + docker.created.remove_error = _api_error("removal of container is already in progress") + with pytest.raises(orchestrator.ContainerRuntimeError, match="cashpilot-mysterium-replaced"): + _raw_deploy(client=docker.client, old=old) + assert old.name == "cashpilot-mysterium-replaced" and not old.removed + assert "cashpilot-mysterium" in orchestrator.deploy_failed() + + def test_an_ordinary_earner_is_put_back_too(self, markers): + old = _running_old("cashpilot-honeygain", {"cashpilot.service": "honeygain", "cashpilot.managed": "true"}) + docker = _FakeDocker(old) + docker.create_error = _api_error("failed to bind host port 0.0.0.0:8080/tcp: address already in use") + with pytest.raises(orchestrator.ContainerRuntimeError, match="running again"): + _raw_deploy(client=docker.client, old=old, slug="honeygain") + assert docker.get("cashpilot-honeygain") is old + old.start.assert_called_once() + + def test_control_a_deploy_that_runs_removes_the_old_one_after(self, markers): + orchestrator._mark_failed("cashpilot-mysterium", True) + old = _running_old() + docker = _FakeDocker(old) + order = [] + docker.created.start.side_effect = lambda: order.append("new started") + old.remove.side_effect = lambda **_: order.append("old removed") + _raw_deploy(client=docker.client, old=old) + assert order == ["new started", "old removed"] + assert orchestrator._pending_identity("cashpilot-mysterium") == IDENTITY_A + assert orchestrator.deploy_failed() == set() + + def test_a_deploy_that_runs_lifts_a_mark_the_file_cannot_drop(self, markers, monkeypatch): + orchestrator._mark_failed("cashpilot-mysterium", True) + real_open = open + + def read_only_marker(path, mode="r", *a, **k): + if path == orchestrator.FAILED_MARKER and "w" in mode: + raise PermissionError(13, "Permission denied", path) + return real_open(path, mode, *a, **k) + + monkeypatch.setattr(orchestrator, "open", read_only_marker, raising=False) + old = _running_old() + _raw_deploy(client=_FakeDocker(old).client, old=old) + assert orchestrator.deploy_failed() == set() + node = _node(status="exited") + _guard([node]) + node.start.assert_called_once() + + +class TestAMoveAsideThatFails: + def test_an_old_node_that_can_neither_move_nor_go_runs_again(self, markers): + old = _running_old() + old.remove_error = _api_error("removal of container is already in progress") + docker = _FakeDocker(old) + old.rename.side_effect = _api_error("rename refused") + with pytest.raises(orchestrator.ContainerRuntimeError, match="not replaced and is running again"): + _raw_deploy(client=docker.client, old=old) + old.start.assert_called_once() + docker.client.containers.create.assert_not_called() + assert orchestrator._identity_snapshot("cashpilot-mysterium") is None + + def test_a_refused_rename_never_removes_the_old_node(self, markers): + old = _running_old() + docker = _FakeDocker(old) + old.rename.side_effect = _api_error("rename refused") + with pytest.raises(orchestrator.ContainerRuntimeError, match="running again"): + _raw_deploy(client=docker.client, old=old) + assert not old.removed and docker.get("cashpilot-mysterium") is old + old.start.assert_called_once() + docker.client.containers.create.assert_not_called() + + def test_a_move_aside_that_times_out_puts_the_old_node_back_running(self, markers): + old = _running_old() + docker = _FakeDocker(old) + + def renamed_then_timed_out(new): + docker.by_name.pop(old.name, None) + old.name = new + docker.by_name[new] = old + old.attrs["Name"] = "/" + new + old.rename.side_effect = lambda back: old.attrs.update(Name="/" + back) + raise ConnectionError("read timed out") + + old.rename.side_effect = renamed_then_timed_out + with pytest.raises(orchestrator.ContainerRuntimeError, match="running again"): + _raw_deploy(client=docker.client, old=old) + assert old.attrs["Name"] == "/cashpilot-mysterium" + old.start.assert_called_once() + docker.client.containers.create.assert_not_called() + + def test_a_stop_that_times_out_on_a_running_node_is_not_replaced(self, markers): + old = _running_old() + docker = _FakeDocker(old) + old.stop.side_effect = ConnectionError("read timed out") + with pytest.raises(orchestrator.ContainerRuntimeError, match="could not be stopped"): + _raw_deploy(client=docker.client, old=old) + old.rename.assert_not_called() + + def test_an_old_node_that_will_not_stop_is_not_replaced(self, markers): + old = _running_old() + docker = _FakeDocker(old) + old.stop.side_effect = _api_error("tried to kill container, but did not receive an exit event") + with pytest.raises(orchestrator.ContainerRuntimeError, match="could not be stopped"): + _raw_deploy(client=docker.client, old=old) + old.rename.assert_not_called() + docker.client.containers.create.assert_not_called() + + def test_control_a_failed_stop_of_a_node_that_did_stop_goes_ahead(self, markers): + old = _running_old() + docker = _FakeDocker(old) + + def stopped_anyway(**_): + old.status = "exited" + raise _api_error("tried to kill container, but did not receive an exit event") + + old.stop.side_effect = stopped_anyway + _raw_deploy(client=docker.client, old=old) + docker.created.start.assert_called_once() + + def test_an_aside_name_another_service_holds_is_refused_before_anything_moves(self, markers): + old = _running_old() + other = _running_old( + "cashpilot-mysterium-replaced", {"cashpilot.service": "mysterium-replaced", "cashpilot.managed": "true"} + ) + docker = _FakeDocker(old, other) + with pytest.raises(orchestrator.ContainerRuntimeError, match="belongs to mysterium-replaced"): + _raw_deploy(client=docker.client, old=old) + old.stop.assert_not_called() + assert not other.removed + + def test_a_new_node_the_fence_went_under_clears_the_failed_mark(self, markers): + orchestrator._mark_failed("cashpilot-mysterium", True) + old = _running_old() + docker = _FakeDocker(old) + with ( + patch.object(orchestrator, "_require_fence", side_effect=orchestrator.ContainerNetworkError("gone")), + pytest.raises(orchestrator.ContainerNetworkError), + ): + _raw_deploy(client=docker.client, old=old) + assert orchestrator.deploy_failed() == set() + + def test_a_name_another_deploy_took_is_left_to_it(self, markers): + old = _running_old() + docker = _FakeDocker(old) + other = MagicMock(id="other", status="running") + other.name = "cashpilot-mysterium" + + def taken(*_a, **_k): + docker.by_name["cashpilot-mysterium"] = other + raise _api_error("container is not running") + + docker.created.start.side_effect = taken + with pytest.raises(orchestrator.ContainerRuntimeError, match="Another deploy took the name"): + _raw_deploy(client=docker.client, old=old) + other.remove.assert_not_called() + assert docker.get("cashpilot-mysterium-replaced") is old + old.start.assert_not_called() + + def test_the_old_node_waits_for_the_fence_before_it_runs_again(self, markers): + old = _running_old() + docker = _FakeDocker(old) + + def fence_gone(*_a, **_k): + lan_fence.fence_problem.return_value = "the LAN fence is not complete on this host" + raise _api_error(BIND_REFUSED) + + docker.created.start.side_effect = fence_gone + with pytest.raises(orchestrator.ContainerRuntimeError, match="kept stopped until the LAN fence"): + _raw_deploy(client=docker.client, old=old) + assert docker.get("cashpilot-mysterium") is old + old.start.assert_not_called() + + def test_a_leftover_that_cannot_go_does_not_fail_a_deploy_that_runs(self, markers): + orchestrator._mark_failed("cashpilot-mysterium", True) + old = _running_old() + old.remove_error = _api_error("removal of container is already in progress") + docker = _FakeDocker(old) + _raw_deploy(client=docker.client, old=old) + docker.created.start.assert_called_once() + assert orchestrator.deploy_failed() == set() + assert docker.get("cashpilot-mysterium-replaced") is old + + +class TestTheGuardStartsOnlyAWholeNode: + def test_a_start_it_could_not_read_back_is_tried_again(self, markers): + node = _node(status="exited") + node.reload.side_effect = RuntimeError("the daemon did not answer") + held, _ = _guard([node]) + node.stop.assert_called_once() + assert lan_fence.UNREAD in held["cashpilot-mysterium"] + assert orchestrator.deploy_failed() == set() + + def test_never_one_missing_part_of_its_udp_range(self, markers): + node = _node(status="exited") + node.attrs["Config"]["Env"] = ["UDP_PORTS=23000:23029"] + del node.attrs["HostConfig"]["PortBindings"]["23015/udp"] + held, _ = _guard([node]) + node.start.assert_not_called() + assert "UDP 23015" in held["cashpilot-mysterium"] + + def test_control_one_with_its_whole_range_is_started(self, markers): + node = _node(status="exited") + node.attrs["Config"]["Env"] = ["UDP_PORTS=23000:23029"] + _guard([node]) + node.start.assert_called_once() + + def test_never_the_old_node_a_deploy_moved_aside(self, markers): + aside = _node(status="exited") + aside.name = "cashpilot-mysterium-replaced" + _guard([aside]) + aside.start.assert_not_called() + + def test_never_a_container_a_failed_deploy_left(self, markers): + orchestrator._mark_failed("cashpilot-mysterium", True) + node = _node(status="exited") + held, _ = _guard([node]) + node.start.assert_not_called() + assert held["cashpilot-mysterium"] == lan_fence.FAILED_DEPLOY + + def test_never_one_not_attached_to_its_network(self, markers): + node = _node(status="exited") + node.attrs["NetworkSettings"] = {"Networks": {}} + held, _ = _guard([node]) + node.start.assert_not_called() + assert "not attached to cashpilot-isolated" in held["cashpilot-mysterium"] + + def test_never_one_without_its_udp_publishes(self, markers): + node = _node(status="exited") + node.attrs["HostConfig"]["PortBindings"] = {"4449/tcp": [{"HostIp": "127.0.0.1", "HostPort": "4449"}]} + held, _ = _guard([node]) + node.start.assert_not_called() + assert "UDP" in held["cashpilot-mysterium"] + + def test_one_that_comes_up_with_no_address_is_stopped_and_not_started_again(self, markers, caplog): + node = _node(status="exited") + + def up_without_network(*_a, **_k): + node.attrs["State"]["Running"] = True + node.attrs["NetworkSettings"]["Networks"]["cashpilot-isolated"]["IPAddress"] = "" + + node.start.side_effect = up_without_network + with caplog.at_level("INFO"): + held, _ = _guard([node]) + node.stop.assert_called_once() + assert "no address" in held["cashpilot-mysterium"] + assert "fence is confirmed" not in caplog.text + again = _node(status="exited") + _guard([again]) + again.start.assert_not_called() + + def test_control_a_whole_node_is_started_and_logged(self, markers, caplog): + node = _node(status="exited") + with caplog.at_level("INFO"): + _guard([node]) + node.start.assert_called_once() + node.stop.assert_not_called() + assert "fence is confirmed and it is on its network" in caplog.text + + +class TestTheEphemeralRangeHint: + def test_a_taken_port_in_the_ephemeral_range_says_why_and_what_to_do(self): + hint = orchestrator._ephemeral_hint(BIND_REFUSED, "mysterium") + assert "32768-60999" in hint and "below 32768" in hint and "UDP_PORTS" in hint and "23000:23029" in hint + + def test_a_service_without_a_udp_range_setting_gets_general_advice(self): + hint = orchestrator._ephemeral_hint(BIND_REFUSED, "honeygain") + assert "32768-60999" in hint and "below 32768" in hint + assert "UDP_PORTS" not in hint and "23000" not in hint + + def test_control_a_taken_port_below_it_gets_no_hint(self): + assert orchestrator._ephemeral_hint(BIND_REFUSED.replace("56033", "23005")) == "" + + def test_control_another_refusal_gets_no_hint(self): + assert orchestrator._ephemeral_hint("no such device /dev/net/tun") == "" diff --git a/tests/test_mysterium_udp_ports.py b/tests/test_mysterium_udp_ports.py index 96201636..194531c4 100644 --- a/tests/test_mysterium_udp_ports.py +++ b/tests/test_mysterium_udp_ports.py @@ -29,16 +29,16 @@ def _udp(sent): def test_a_first_deploy_uses_the_range_the_guide_says_to_forward(client): resp, sent = _deploy(client, {"env": {}}) assert resp.status_code == 200, resp.text - assert " --udp.ports=56000:56029 service " in sent["command"] + assert " --udp.ports=23000:23029 service " in sent["command"] # Published 1:1 on the bridge, so the router's forward reaches the node unchanged. - assert _udp(sent) == [f"{p}/udp" for p in range(56000, 56030)] - assert all(sent["ports"][f"{p}/udp"] == p for p in range(56000, 56030)) + assert _udp(sent) == [f"{p}/udp" for p in range(23000, 23030)] + assert all(sent["ports"][f"{p}/udp"] == p for p in range(23000, 23030)) def test_a_node_deployed_before_the_setting_existed_gets_the_default(client): resp, sent = _deploy(client, {"env": {}}, recorded=_record({"UI_ADDRESS": "127.0.0.1"})) assert resp.status_code == 200, resp.text - assert " --udp.ports=56000:56029 service " in sent["command"] + assert " --udp.ports=23000:23029 service " in sent["command"] def test_a_typed_range_reaches_the_node(client): @@ -50,7 +50,7 @@ def test_a_typed_range_reaches_the_node(client): def test_a_blank_redeploy_keeps_the_range_this_machine_was_given(client): recorded = _record({"UDP_PORTS": "41000:41050"}) - for env in ({}, {"UDP_PORTS": ""}, {"UDP_PORTS": "56000:56029"}): + for env in ({}, {"UDP_PORTS": ""}, {"UDP_PORTS": "23000:23029"}): resp, sent = _deploy(client, {"env": env}, recorded=recorded) assert resp.status_code == 200, resp.text assert " --udp.ports=41000:41050 service " in sent["command"] @@ -83,5 +83,5 @@ def test_a_recorded_reversed_range_is_refused_too(client): def test_the_compose_rendering_uses_the_default_range(): service = compose_generator._service_to_compose(catalog.get_service("mysterium"), None, None, None) - assert " --udp.ports=56000:56029 service " in service["command"] - assert "56000-56029:56000-56029/udp" in service["ports"] + assert " --udp.ports=23000:23029 service " in service["command"] + assert "23000-23029:23000-23029/udp" in service["ports"] diff --git a/tests/test_mysterium_ui_address.py b/tests/test_mysterium_ui_address.py index 9a1b4ec7..d16debec 100644 --- a/tests/test_mysterium_ui_address.py +++ b/tests/test_mysterium_ui_address.py @@ -28,7 +28,7 @@ ) LOOPBACK = f"--ui.address=127.0.0.1 --tequilapi.address=127.0.0.1 {PROTECTED} --udp.ports=56000:56100 service --agreed-terms-and-conditions" #: What every node runs after 2.0.6: the WebUI on its own namespace, published by Docker. -COMMAND = f"--ui.address=0.0.0.0 --tequilapi.address=127.0.0.1 {PROTECTED} --udp.ports=56000:56029 service --agreed-terms-and-conditions" +COMMAND = f"--ui.address=0.0.0.0 --tequilapi.address=127.0.0.1 {PROTECTED} --udp.ports=23000:23029 service --agreed-terms-and-conditions" UI = "4449/tcp"