From 8bc7d87d4208586ee56e02a4a21e9480ef5b2153 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 14:29:12 +0100
Subject: [PATCH 01/34] fix(proxies): retry a copy the cap reset could not
start, so an offline box does not keep it stopped
When a reset could not start a copy the cap had stopped, the copy kept its
stopped_by_cap mark and nothing tried again until the next reset, a month
later. Each heartbeat from that worker now starts such a copy again while the
proxy is uncapped and the copy is reported down.
---
app/main.py | 35 ++++++++++++++++++++++++++++++-
docs/proxies.md | 4 +++-
tests/test_proxies.py | 49 ++++++++++++++++++++++++++++++++++++++++++-
3 files changed, 85 insertions(+), 3 deletions(-)
diff --git a/app/main.py b/app/main.py
index 8ef60fff..fa18a011 100644
--- a/app/main.py
+++ b/app/main.py
@@ -4366,6 +4366,8 @@ async def _run_proxy_cap_resets() -> None:
_proxy_cap_unstopped: dict[int, dict[tuple[int, str], str]] = {}
# Stops already on their way, so a slow worker is not sent a second one.
_proxy_cap_stopping: set[tuple[int, int, str]] = set()
+# Starts of copies the cap stopped, retried after a reset, on their way.
+_proxy_cap_starting: set[tuple[int, int, str]] = set()
def _running(entry: dict[str, Any]) -> bool:
@@ -4392,7 +4394,9 @@ async def _account_proxy_traffic(worker_id: int, entries: list[Any]) -> None:
return
# Only traffic of copies the UI recorded on this worker counts, so a worker
# cannot spend, or cap, a proxy it runs nothing through.
- recorded = {(c["proxy_id"], c["slug"]) for c in await database.list_proxied_copies() if c["worker_id"] == worker_id}
+ copies = [c for c in await database.list_proxied_copies() if c["worker_id"] == worker_id]
+ recorded = {(c["proxy_id"], c["slug"]) for c in copies}
+ held_by_cap = {(c["proxy_id"], c["slug"]) for c in copies if c.get("stopped_by_cap")}
now = time.time()
for proxy_id, items in by_proxy.items():
counted = [e for e in items if (proxy_id, egress.container_slug(e)) in recorded]
@@ -4403,6 +4407,18 @@ async def _account_proxy_traffic(worker_id: int, entries: list[Any]) -> None:
state, crossed = proxies.account(proxy["state"], proxy["definition"], worker_id, counted, now)
await database.update_proxy_state(proxy_id, state)
if not state.get("capped_at"):
+ # A reset whose start failed leaves the copy marked as the cap's and
+ # down; while the proxy stays uncapped, every heartbeat tries again.
+ retry = {
+ slug
+ for e in items
+ if not _running(e)
+ and (proxy_id, slug := egress.container_slug(e)) in held_by_cap
+ and (proxy_id, worker_id, slug) not in _proxy_cap_starting
+ }
+ if retry:
+ _proxy_cap_starting.update((proxy_id, worker_id, slug) for slug in retry)
+ _spawn(_restart_capped_copies(proxy_id, worker_id, retry))
continue
targets = {(worker_id, egress.container_slug(e)) for e in items if _running(e)}
if crossed:
@@ -4416,6 +4432,23 @@ async def _account_proxy_traffic(worker_id: int, entries: list[Any]) -> None:
_spawn(_stop_capped_proxy({**proxy, "state": state}, targets))
+async def _restart_capped_copies(proxy_id: int, worker_id: int, slugs: set[str]) -> None:
+ """Start copies the cap stopped and a reset could not start, unless the cap is back."""
+ try:
+ for slug in sorted(slugs):
+ async with _proxy_state_lock(proxy_id):
+ fresh = await database.get_proxy(proxy_id)
+ if not fresh or fresh["state"].get("capped_at"):
+ return
+ try:
+ await _start_proxied_copy(fresh, worker_id, slug)
+ await database.set_proxied_copy_capped(proxy_id, worker_id, slug, False)
+ except HTTPException as exc:
+ logger.warning("Could not restart %s through proxy %s after a reset: %s", slug, proxy_id, exc.detail)
+ finally:
+ _proxy_cap_starting.difference_update((proxy_id, worker_id, slug) for slug in slugs)
+
+
async def _stop_capped_proxy(proxy: dict[str, Any], targets: set[tuple[int, str]]) -> None:
pid = proxy["id"]
unstopped = _proxy_cap_unstopped.setdefault(pid, {})
diff --git a/docs/proxies.md b/docs/proxies.md
index 3790fa5b..19d261c3 100644
--- a/docs/proxies.md
+++ b/docs/proxies.md
@@ -227,7 +227,9 @@ the moment of the stop is stopped when it comes back. If a stop fails, the
alert says which copy is still running.
A reset, on the day or by hand, starts only the copies the cap stopped; a copy
-you stopped yourself stays stopped. Turning a cap on starts a fresh period, so
+you stopped yourself stays stopped. If a copy cannot be started at the reset,
+for example because its box is offline, every report from that box tries again
+until it starts or the cap holds again. Turning a cap on starts a fresh period, so
traffic from before it existed does not count. Turning it off, or raising it
above what this period has used, lifts the stop and starts those copies again.
diff --git a/tests/test_proxies.py b/tests/test_proxies.py
index f6280c76..e8808565 100644
--- a/tests/test_proxies.py
+++ b/tests/test_proxies.py
@@ -322,6 +322,7 @@ def __init__(self, copies, worker_call):
self.call = worker_call
self.alerts: list[tuple] = []
self.pushed: list[tuple] = []
+ self.mark = AsyncMock()
async def heartbeat(self, worker_id, entries):
spawned: list = []
@@ -343,7 +344,7 @@ async def push(*args):
patch.object(main.database, "get_proxy", side_effect=get_proxy),
patch.object(main.database, "update_proxy_state", side_effect=update_state),
patch.object(main.database, "list_proxied_copies", new_callable=AsyncMock, return_value=self.copies),
- patch.object(main.database, "set_proxied_copy_capped", new_callable=AsyncMock),
+ patch.object(main.database, "set_proxied_copy_capped", self.mark),
patch.object(main.database, "record_alert", side_effect=record_alert),
patch.object(main.database, "list_workers", new_callable=AsyncMock, return_value=[]),
patch.object(main, "_proxy_worker_call", self.call),
@@ -359,6 +360,7 @@ class TestCapEnforcement:
def teardown_method(self):
main._drop_proxy_cap_bell(1)
main._proxy_cap_stopping.clear()
+ main._proxy_cap_starting.clear()
def test_a_failed_stop_is_tried_again_on_the_next_heartbeat(self):
fails = {8: [HTTPException(status_code=503, detail="Worker offline")]}
@@ -406,6 +408,51 @@ def test_a_stopped_copy_is_left_alone(self):
asyncio.run(rig.heartbeat(9, [_entry(5, 0, tid="t9", status="exited", tunnel_status="exited")]))
assert call.call_count == 0
+ def test_a_copy_a_reset_could_not_start_is_started_on_the_next_heartbeat(self):
+ fails = [HTTPException(status_code=503, detail="Worker offline")]
+
+ async def worker_call(worker_id, method, path, definition, **kw):
+ if fails:
+ raise fails.pop()
+ return {}
+
+ call = AsyncMock(side_effect=worker_call)
+ rig = _CapRig(
+ [
+ {"proxy_id": 1, "worker_id": 7, "slug": "a", "stopped_by_cap": 1},
+ {"proxy_id": 1, "worker_id": 7, "slug": "b", "stopped_by_cap": 0},
+ ],
+ call,
+ )
+ down = {"status": "exited", "tunnel_status": "exited"}
+
+ async def go():
+ # The first try fails, the copy stays the cap's, the next heartbeat tries again.
+ await rig.heartbeat(7, [_entry(0, 0, **down), _entry(0, 0, tid="tb", slug="b", **down)])
+ rig.mark.assert_not_called()
+ await rig.heartbeat(7, [_entry(0, 0, **down), _entry(0, 0, tid="tb", slug="b", **down)])
+
+ asyncio.run(go())
+ # Only the copy the cap stopped is started; the owner's stopped copy b is left alone.
+ assert [(c.args[0], c.args[2]) for c in call.call_args_list] == [
+ (7, "/api/proxied/a/1/start"),
+ (7, "/api/proxied/a/1/start"),
+ ]
+ assert [c.args for c in rig.mark.call_args_list] == [(1, 7, "a", False)]
+
+ def test_a_copy_the_cap_stopped_stays_down_while_the_cap_holds(self):
+ call = AsyncMock(return_value={})
+ rig = _CapRig([{"proxy_id": 1, "worker_id": 7, "slug": "a", "stopped_by_cap": 1}], call)
+ rig.proxy["state"] = {"used_bytes": 950_000_000, "capped_at": "x", "cap_reason": "r"}
+ asyncio.run(rig.heartbeat(7, [_entry(0, 0, status="exited", tunnel_status="exited")]))
+ assert call.call_count == 0
+
+ def test_a_running_copy_the_cap_stopped_is_not_started_again(self):
+ call = AsyncMock(return_value={})
+ rig = _CapRig([{"proxy_id": 1, "worker_id": 7, "slug": "a", "stopped_by_cap": 1}], call)
+ asyncio.run(rig.heartbeat(7, [_entry(0, 0, status="running", tunnel_status="running")]))
+ assert call.call_count == 0
+
def test_the_bell_keeps_a_cap_that_still_holds_after_a_restart(self):
capped = {"id": 3, "definition": {"name": "Oxy"}, "state": {"capped_at": "x", "cap_reason": "Oxy reached 90%"}}
free = {"id": 4, "definition": {"name": "Free"}, "state": {}}
From b9458e8360b0b063656013d24772d79151d7824d Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 14:35:31 +0100
Subject: [PATCH 02/34] fix(proxies): keep the running copy when a redeploy's
tunnel cannot start
deploy() removed the old earner and tunnel before the new tunnel started, so a
tunnel that failed to start (an image missing locally, a proxy that does not
answer) left the proxy with no copy at all. The new tunnel now starts under a
staged name next to the running copy. Only once it has proved an exit does the
old pair go and the tunnel take the final name, under a lock the heartbeat's
status() shares. The heartbeat never reports a staged tunnel, and a worker
start sweeps one a dead deploy left behind.
---
app/proxy_tunnel.py | 68 ++++++++++++++++++++++++----
tests/test_proxy_tunnel.py | 90 ++++++++++++++++++++++++++++++++++++--
2 files changed, 145 insertions(+), 13 deletions(-)
diff --git a/app/proxy_tunnel.py b/app/proxy_tunnel.py
index 23a6331b..54930c71 100644
--- a/app/proxy_tunnel.py
+++ b/app/proxy_tunnel.py
@@ -31,6 +31,7 @@
import re
import socket
import tarfile
+import threading
import time
import uuid
from typing import Any
@@ -61,6 +62,13 @@
READY_TIMEOUT = 45.0
SECRET_DIR = "/run/cashpilot-secret"
_FLOATING_TAGS = {"", "latest", "stable", "edge", "main", "master"}
+#: A redeploy starts the new sidecar under its tunnel name plus this suffix, next
+#: to the copy that runs now, and gives it the final name only once the old pair
+#: is gone. The heartbeat (status()) never reports a staged sidecar.
+STAGED_SUFFIX = "-next"
+# deploy() swapping the old pair for the staged one, and status(), take turns: a
+# heartbeat must not start an earner that is being removed, or read a half swap.
+_swap_lock = threading.Lock()
class ProxyTunnelError(RuntimeError):
@@ -357,7 +365,13 @@ def _get(client: Any, name: str) -> Any | None:
def start_sidecar(
- client: Any, name: str, slug: str, proxy_id: int | None, proxy: dict[str, Any], own_ip: str | None
+ client: Any,
+ name: str,
+ slug: str,
+ proxy_id: int | None,
+ proxy: dict[str, Any],
+ own_ip: str | None,
+ hostname: str | None = None,
) -> tuple[Any, dict[str, Any]]:
"""Create, provision and start a sidecar, then wait for its exit. Removed on any failure."""
network = orchestrator.container_network("bridge")
@@ -367,7 +381,7 @@ def start_sidecar(
client.images.pull(image)
except APIError as exc:
logger.warning("Could not pull %s: %s (trying the local copy)", image, exc)
- kwargs = sidecar_kwargs(name, slug, proxy_id, network, hostname=name.removeprefix(CONTAINER_PREFIX)[:63])
+ kwargs = sidecar_kwargs(name, slug, proxy_id, network, hostname=hostname or _hostname(name))
kwargs.pop("detach")
container = client.containers.create(**kwargs)
try:
@@ -384,19 +398,38 @@ def start_sidecar(
return container, exit_info
+def _hostname(name: str) -> str:
+ return name.removeprefix(CONTAINER_PREFIX)[:63]
+
+
def deploy(slug: str, proxy_id: int, earner: dict[str, Any], proxy: dict[str, Any], own_ip: str | None) -> dict:
"""Deploy (or replace) the proxied copy of ``slug`` through proxy ``proxy_id``.
- Order: the old pair goes (earner first), the sidecar starts and proves an
- exit, and only then is the earner created inside its namespace.
+ Order: the new sidecar starts under a staged name and proves an exit while
+ the old pair keeps running; then the old pair goes (earner first), the
+ sidecar takes the final name, and only then is the earner created inside
+ its namespace. A sidecar that cannot start therefore costs nothing.
"""
client = orchestrator._get_client()
- # Checked before the old pair goes: settings that cannot work (a host that
- # does not resolve, no tunnel image) must not cost the copy running now.
+ # Checked first: settings that cannot work (a host that does not resolve,
+ # no tunnel image) fail fast, before a sidecar is created.
secret_files(proxy, own_ip, f"{slug}-p{proxy_id}")
tunnel_image()
- remove(slug, proxy_id)
- sidecar, exit_info = start_sidecar(client, tunnel_name(slug, proxy_id), slug, proxy_id, proxy, own_ip)
+ final = tunnel_name(slug, proxy_id)
+ staged = final + STAGED_SUFFIX
+ # Left by a deploy that died midway, on this name only if it is ours.
+ if (leftover := _get(client, staged)) is not None:
+ if not _is_ours(leftover, slug, proxy_id):
+ raise ProxyTunnelError(f"A container named {staged} that CashPilot did not create is in the way")
+ _remove_quietly(leftover, 2)
+ sidecar, exit_info = start_sidecar(client, staged, slug, proxy_id, proxy, own_ip, hostname=_hostname(final))
+ try:
+ with _swap_lock:
+ remove(slug, proxy_id)
+ sidecar.rename(final)
+ except Exception:
+ _remove_quietly(sidecar, 2)
+ raise
try:
container_id = orchestrator.deploy_raw(
slug=slug,
@@ -529,7 +562,8 @@ def remove_all(slug: str) -> list[str]:
def sweep_test_tunnels() -> list[str]:
- """Remove Test tunnels a worker that stopped mid-test left behind.
+ """Remove Test tunnels a worker that stopped mid-test left behind, and staged
+ sidecars of a deploy it stopped in.
Run at worker start, when no test of this worker can be in flight.
"""
@@ -544,6 +578,15 @@ def sweep_test_tunnels() -> list[str]:
continue
_remove_quietly(container, 2)
swept.append(container.name)
+ # And staged sidecars of a deploy the worker died in: they hold the proxy
+ # login, and the heartbeat never reports them.
+ staged = client.containers.list(
+ all=True, filters={"label": [f"{LABEL_MANAGED}=true", f"{LABEL_ROLE}={ROLE_TUNNEL}", LABEL_PROXY]}
+ )
+ for container in staged:
+ if str(container.name).endswith(STAGED_SUFFIX):
+ _remove_quietly(container, 2)
+ swept.append(container.name)
return swept
@@ -600,6 +643,11 @@ def status(collect_stats: bool = True) -> list[dict[str, Any]]:
stats carry no network counters, and every byte it moves crosses the
sidecar's eth0.
"""
+ with _swap_lock:
+ return _status(collect_stats)
+
+
+def _status(collect_stats: bool) -> list[dict[str, Any]]:
try:
client = orchestrator._get_client()
found = client.containers.list(all=True, filters={"label": [f"{LABEL_MANAGED}=true", LABEL_PROXY]})
@@ -608,6 +656,8 @@ def status(collect_stats: bool = True) -> list[dict[str, Any]]:
return []
pairs: dict[tuple[str, str], dict[str, Any]] = {}
for container in found:
+ if str(getattr(container, "name", "")).endswith(STAGED_SUFFIX):
+ continue # a redeploy's sidecar, not promoted yet
labels = container.labels or {}
key = (labels.get(LABEL_SERVICE, ""), labels.get(LABEL_PROXY, ""))
role = "tunnel" if labels.get(LABEL_ROLE) == ROLE_TUNNEL else "earner"
diff --git a/tests/test_proxy_tunnel.py b/tests/test_proxy_tunnel.py
index 7fea50a9..6fa558b7 100644
--- a/tests/test_proxy_tunnel.py
+++ b/tests/test_proxy_tunnel.py
@@ -298,21 +298,94 @@ def fake_deploy_raw(**kw):
assert "ports" not in kw and "volumes" not in kw
return "earner-id"
+ sidecar.rename.side_effect = lambda name: calls.append(f"renamed {name}")
+ client = MagicMock()
+ client.containers.get.side_effect = NotFound("gone")
with (
- patch.object(orchestrator, "_get_client", return_value=MagicMock()),
+ patch.object(orchestrator, "_get_client", return_value=client),
patch.object(proxy_tunnel, "tunnel_image", return_value="drumsergio/cashpilot-tunnel:1.64.0"),
patch.object(proxy_tunnel, "remove", side_effect=lambda *a: calls.append("old-removed") or []),
- patch.object(proxy_tunnel, "start_sidecar", side_effect=fake_start_sidecar),
+ patch.object(proxy_tunnel, "start_sidecar", side_effect=fake_start_sidecar) as start,
patch.object(orchestrator, "deploy_raw", side_effect=fake_deploy_raw),
):
result = proxy_tunnel.deploy("honeygain", 3, {"image": "honeygain/honeygain:1"}, _socks(), None)
- assert calls == ["old-removed", "sidecar-ready", "earner"]
+ # The new sidecar proves its exit while the old pair still runs, under a staged name.
+ assert calls == ["sidecar-ready", "old-removed", "renamed cashpilot-tunnel-honeygain-p3", "earner"]
+ assert start.call_args.args[1] == "cashpilot-tunnel-honeygain-p3-next"
+ assert start.call_args.kwargs["hostname"] == "tunnel-honeygain-p3"
assert result["container_id"] == "earner-id"
+ def test_a_sidecar_that_cannot_start_keeps_the_running_copy(self):
+ client = MagicMock()
+ client.containers.get.side_effect = NotFound("gone")
+ with (
+ patch.object(orchestrator, "_get_client", return_value=client),
+ patch.object(proxy_tunnel, "tunnel_image", return_value="drumsergio/cashpilot-tunnel:1.64.0"),
+ patch.object(proxy_tunnel, "remove") as remove,
+ patch.object(proxy_tunnel, "start_sidecar", side_effect=proxy_tunnel.ProxyTunnelError("No such image")),
+ pytest.raises(proxy_tunnel.ProxyTunnelError),
+ ):
+ proxy_tunnel.deploy("honeygain", 3, {"image": "honeygain/honeygain:1"}, _socks(), None)
+ remove.assert_not_called()
+
+ def test_a_failed_promotion_removes_the_staged_sidecar(self):
+ sidecar = MagicMock(id="sidecar-id")
+ sidecar.rename.side_effect = RuntimeError("409 Conflict")
+ client = MagicMock()
+ client.containers.get.side_effect = NotFound("gone")
+ with (
+ patch.object(orchestrator, "_get_client", return_value=client),
+ patch.object(proxy_tunnel, "tunnel_image", return_value="drumsergio/cashpilot-tunnel:1.64.0"),
+ patch.object(proxy_tunnel, "remove", return_value=[]),
+ patch.object(proxy_tunnel, "start_sidecar", return_value=(sidecar, {"ip": "93.184.215.14"})),
+ patch.object(orchestrator, "deploy_raw") as deploy_raw,
+ pytest.raises(RuntimeError),
+ ):
+ proxy_tunnel.deploy("honeygain", 3, {"image": "honeygain/honeygain:1"}, _socks(), None)
+ sidecar.remove.assert_called_once_with(force=True, v=True)
+ deploy_raw.assert_not_called()
+
+ def test_a_staged_sidecar_left_behind_is_replaced_but_never_a_stranger(self):
+ leftover = _container("cashpilot-tunnel-honeygain-p3-next", "honeygain", proxy=3, role="tunnel")
+ stranger = _container("cashpilot-tunnel-honeygain-p3-next", "other")
+ sidecar = MagicMock(id="sidecar-id")
+ for found, ok in ((leftover, True), (stranger, False)):
+ client = MagicMock()
+ client.containers.get.return_value = found
+ with (
+ patch.object(orchestrator, "_get_client", return_value=client),
+ patch.object(proxy_tunnel, "tunnel_image", return_value="drumsergio/cashpilot-tunnel:1.64.0"),
+ patch.object(proxy_tunnel, "remove", return_value=[]),
+ patch.object(proxy_tunnel, "start_sidecar", return_value=(sidecar, {"ip": "93.184.215.14"})),
+ patch.object(orchestrator, "deploy_raw", return_value="earner-id"),
+ ):
+ if ok:
+ proxy_tunnel.deploy("honeygain", 3, {"image": "honeygain/honeygain:1"}, _socks(), None)
+ else:
+ with pytest.raises(proxy_tunnel.ProxyTunnelError, match="did not create"):
+ proxy_tunnel.deploy("honeygain", 3, {"image": "honeygain/honeygain:1"}, _socks(), None)
+ assert found.remove.called is ok
+
+ def test_the_heartbeat_never_reports_a_staged_sidecar(self):
+ earner = _container("cashpilot-x-p1", "x", proxy=1)
+ tunnel = _container("cashpilot-tunnel-x-p1", "x", proxy=1, role="tunnel")
+ staged = _container("cashpilot-tunnel-x-p1-next", "x", proxy=1, role="tunnel")
+ staged.id = "id-staged"
+ client = MagicMock()
+ client.containers.list.return_value = [earner, tunnel, staged]
+ with (
+ patch.object(orchestrator, "_get_client", return_value=client),
+ patch.object(proxy_tunnel, "_read_exit", return_value=None),
+ ):
+ entries = proxy_tunnel.status(collect_stats=False)
+ assert [(e["slug"], e["tunnel_container_id"]) for e in entries] == [("x", tunnel.short_id)]
+
def test_earner_failure_removes_the_sidecar(self):
sidecar = MagicMock(id="sidecar-id")
+ client = MagicMock()
+ client.containers.get.side_effect = NotFound("gone")
with (
- patch.object(orchestrator, "_get_client", return_value=MagicMock()),
+ patch.object(orchestrator, "_get_client", return_value=client),
patch.object(proxy_tunnel, "tunnel_image", return_value="drumsergio/cashpilot-tunnel:1.64.0"),
patch.object(proxy_tunnel, "remove", return_value=[]),
patch.object(proxy_tunnel, "start_sidecar", return_value=(sidecar, {"ip": "93.184.215.14"})),
@@ -882,6 +955,15 @@ def test_leftover_test_tunnels_are_swept_and_copies_are_not(self):
leftover.remove.assert_called_once_with(force=True, v=True)
copy_tunnel.remove.assert_not_called()
+ def test_a_staged_sidecar_of_a_deploy_that_died_is_swept(self):
+ staged = _container("cashpilot-tunnel-x-p1-next", "x", proxy=1, role="tunnel")
+ live = _container("cashpilot-tunnel-x-p1", "x", proxy=1, role="tunnel")
+ client = MagicMock()
+ client.containers.list.side_effect = [[], [staged, live]]
+ with patch.object(orchestrator, "_get_client", return_value=client):
+ assert proxy_tunnel.sweep_test_tunnels() == ["cashpilot-tunnel-x-p1-next"]
+ live.remove.assert_not_called()
+
def test_a_probe_answer_that_is_not_an_ip_is_not_ready(self):
sidecar = MagicMock(status="running")
sidecar.exec_run.return_value = (0, b'{"ip":"e","country":""}\n')
From 42986fa89068a13248fd77f5b92fe5e158dcd7e4 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 14:37:36 +0100
Subject: [PATCH 03/34] fix(isolation): wait for another process's fence helper
instead of killing it
The worker force-removed any container named cashpilot-fence that carried
the fence label before starting its own. On a host that also runs the
Desktop app, that is the other side's helper while it is still running, so
the other side recorded a failed fence and refused its deploy.
The worker now waits for a helper that is running, or was created or exited
in the last 10 seconds, and removes only a leftover. A create that loses the
race on the name is tried once more. Its own helper is created and started
apart and removed by its own object, never by name.
---
app/lan_fence.py | 168 ++++++++++++++++++++++++++++++----------
tests/test_lan_fence.py | 116 +++++++++++++++++++++++++--
2 files changed, 239 insertions(+), 45 deletions(-)
diff --git a/app/lan_fence.py b/app/lan_fence.py
index c7f8b31b..8d9917b6 100644
--- a/app/lan_fence.py
+++ b/app/lan_fence.py
@@ -57,6 +57,7 @@
import re
import threading
import time
+from datetime import datetime
from pathlib import Path
from typing import Any
@@ -770,58 +771,105 @@ def _helper_image(client: Any) -> str:
return f"drumsergio/cashpilot-worker:{version.current()}"
-def _remove_helper(client: Any) -> None:
- """Remove a leftover helper by name, only when it carries the helper's label.
+#: How long a helper that was just created or has just exited still belongs to
+#: the process that started it: that process starts it, then reads its logs and
+#: removes it, all within moments. HELPER_POLL is how often a busy helper is
+#: looked at again.
+HELPER_GRACE = 10.0
+HELPER_POLL = 0.25
- A container someone else named cashpilot-fence is left alone; the run that
- follows then fails on the name instead of deleting it.
+
+def _stamp_age(stamp: Any, now: float) -> float | None:
+ """Seconds since a Docker timestamp, or None when it is unset or unreadable."""
+ try:
+ when = datetime.fromisoformat(str(stamp))
+ except ValueError:
+ return None
+ if when.tzinfo is None or when.year < 2000:
+ return None
+ return now - when.timestamp()
+
+
+def _helper_busy(attrs: dict[str, Any], now: float) -> bool:
+ """A helper another process is still using: running, or created or exited within HELPER_GRACE."""
+ state = attrs.get("State") or {}
+ if state.get("Running") or state.get("Restarting"):
+ return True
+ status = state.get("Status")
+ if status == "created":
+ stamp = attrs.get("Created")
+ elif status in ("exited", "dead"):
+ stamp = state.get("FinishedAt")
+ else:
+ return False
+ age = _stamp_age(stamp, now)
+ return age is not None and age < HELPER_GRACE
+
+
+def _clear_helper(client: Any) -> None:
+ """Make room for this process's helper.
+
+ Only a container that carries the helper's label is touched: a container
+ someone else named cashpilot-fence is left alone, and the run that follows
+ fails on the name instead of deleting it.
+
+ The Desktop app runs the same helper, by the same name and label, so on a
+ host that runs both, a helper of ours can belong to another process that is
+ still waiting for its result. That one is waited for, never killed: killing
+ it would make the other side record a failed fence and refuse its deploy.
+ Only a leftover, one idle past HELPER_GRACE or still there after the helper
+ timeout, is removed.
"""
from docker.errors import NotFound
+ deadline = time.monotonic() + HELPER_TIMEOUT + HELPER_GRACE
+ while True:
+ try:
+ found = client.containers.get(HELPER_NAME)
+ if (getattr(found, "labels", None) or {}).get(PROBE_LABEL) != "fence":
+ return
+ if not _helper_busy(found.attrs or {}, time.time()) or time.monotonic() > deadline:
+ found.remove(force=True, v=True)
+ return
+ except NotFound:
+ return
+ except Exception as exc: # noqa: BLE001 - logged; the run that follows fails on the name
+ logger.warning("Could not remove %s: %s", HELPER_NAME, exc)
+ return
+ time.sleep(HELPER_POLL)
+
+
+def _remove_own(container: Any) -> None:
+ """Remove the helper this process started, by its own ID, never by name."""
+ from docker.errors import NotFound
+
try:
- found = client.containers.get(HELPER_NAME)
- if (getattr(found, "labels", None) or {}).get(PROBE_LABEL) == "fence":
- found.remove(force=True, v=True)
+ container.remove(force=True, v=True)
except NotFound:
pass
- except Exception as exc: # noqa: BLE001 - logged; the next run removes it by name first
+ except Exception as exc: # noqa: BLE001 - logged; the next run clears it as a leftover
logger.warning("Could not remove %s: %s", HELPER_NAME, exc)
def run_helper(client: Any, action: str, subnet: str, filtered: bool = True) -> dict[str, Any]:
"""Run the helper to completion and return what it printed. Removed on every path."""
+ from docker.errors import APIError
+
with _lock:
- _remove_helper(client)
+ container = None
try:
- container = client.containers.run(
- image=_helper_image(client),
- name=HELPER_NAME,
- entrypoint=["python", "-c", HELPER_SCRIPT],
- command=[action],
- environment={
- "FENCE_IFACE": BRIDGE_INTERFACE,
- "FENCE_SUBNET": subnet,
- "FENCE_FILTER": "1" if filtered else "0",
- "FENCE_BACKEND": _BACKEND,
- "FENCE_EGRESS": _egress["ip"] or "",
- },
- network_mode="host",
- user="0:0",
- read_only=True,
- # iptables-legacy takes its lock in /run.
- tmpfs={"/run": "size=1m"},
- cap_drop=["ALL"],
- # NET_RAW only matters to iptables-legacy, which talks to the
- # kernel over a raw socket. Both run our own script, never an earner.
- cap_add=["NET_ADMIN", "NET_RAW"],
- security_opt=["no-new-privileges:true"],
- privileged=False,
- pids_limit=64,
- mem_limit="64m",
- healthcheck={"test": ["NONE"]},
- labels={PROBE_LABEL: "fence"},
- detach=True,
- )
+ # Twice: another process can create its helper between our clear and our run.
+ for attempt in range(2):
+ _clear_helper(client)
+ try:
+ container = _create_helper(client, action, subnet, filtered)
+ break
+ except APIError as exc:
+ if attempt or getattr(exc, "status_code", None) != 409:
+ raise
+ # Created and started apart, so a start that fails still leaves an
+ # ID to remove: nothing is ever removed by name after the create.
+ container.start()
status = container.wait(timeout=HELPER_TIMEOUT)
text = container.logs(stdout=True, stderr=False).decode(errors="replace").strip()
if (status or {}).get("StatusCode") != 0:
@@ -831,7 +879,49 @@ def run_helper(client: Any, action: str, subnet: str, filtered: bool = True) ->
)
return json.loads(text.splitlines()[-1])
finally:
- _remove_helper(client)
+ if container is not None:
+ _remove_own(container)
+
+
+def _create_helper(client: Any, action: str, subnet: str, filtered: bool) -> Any:
+ from docker.errors import ImageNotFound
+
+ image = _helper_image(client)
+ try:
+ return client.containers.create(image=image, **_helper_spec(action, subnet, filtered))
+ except ImageNotFound:
+ client.images.pull(image)
+ return client.containers.create(image=image, **_helper_spec(action, subnet, filtered))
+
+
+def _helper_spec(action: str, subnet: str, filtered: bool) -> dict[str, Any]:
+ return dict(
+ name=HELPER_NAME,
+ entrypoint=["python", "-c", HELPER_SCRIPT],
+ command=[action],
+ environment={
+ "FENCE_IFACE": BRIDGE_INTERFACE,
+ "FENCE_SUBNET": subnet,
+ "FENCE_FILTER": "1" if filtered else "0",
+ "FENCE_BACKEND": _BACKEND,
+ "FENCE_EGRESS": _egress["ip"] or "",
+ },
+ network_mode="host",
+ user="0:0",
+ read_only=True,
+ # iptables-legacy takes its lock in /run.
+ tmpfs={"/run": "size=1m"},
+ cap_drop=["ALL"],
+ # NET_RAW only matters to iptables-legacy, which talks to the
+ # kernel over a raw socket. Both run our own script, never an earner.
+ cap_add=["NET_ADMIN", "NET_RAW"],
+ security_opt=["no-new-privileges:true"],
+ privileged=False,
+ pids_limit=64,
+ mem_limit="64m",
+ healthcheck={"test": ["NONE"]},
+ labels={PROBE_LABEL: "fence"},
+ )
def _set_fence(state: str, result: dict[str, Any] | None = None, error: str = "") -> None:
diff --git a/tests/test_lan_fence.py b/tests/test_lan_fence.py
index 9008b934..33f2a080 100644
--- a/tests/test_lan_fence.py
+++ b/tests/test_lan_fence.py
@@ -18,6 +18,7 @@
import subprocess
import sys
from contextlib import asynccontextmanager
+from datetime import UTC, datetime, timedelta
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
@@ -26,7 +27,7 @@
os.environ.setdefault("CASHPILOT_API_KEY", "test-fleet-key")
import pytest
-from docker.errors import NotFound
+from docker.errors import APIError, ImageNotFound, NotFound
from fastapi.testclient import TestClient
from app import isolation_check, lan_fence, orchestrator, worker_api
@@ -860,13 +861,13 @@ def test_the_next_deploy_lands_behind_the_fence(self):
def test_the_helper_gets_the_egress_ip():
lan_fence.set_egress_ip("81.61.1.9")
client = MagicMock()
- helper = client.containers.run.return_value
+ helper = client.containers.create.return_value
helper.wait.return_value = {"StatusCode": 0}
helper.logs.return_value = b'{"ok": true}'
client.containers.get.side_effect = NotFound("gone")
with patch.object(lan_fence, "_helper_image", return_value="img"):
lan_fence.run_helper(client, "apply", "172.18.0.0/16")
- assert client.containers.run.call_args.kwargs["environment"]["FENCE_EGRESS"] == "81.61.1.9"
+ assert client.containers.create.call_args.kwargs["environment"]["FENCE_EGRESS"] == "81.61.1.9"
lan_fence.set_egress_ip("10.0.0.1")
assert lan_fence._egress["ip"] == "81.61.1.9", "a LAN answer is a failed lookup: never dropped, the last IP kept"
@@ -929,14 +930,117 @@ def test_a_foreign_container_by_the_helpers_name_is_left_alone(self):
foreign = MagicMock(labels={})
client = MagicMock()
client.containers.get.return_value = foreign
- lan_fence._remove_helper(client)
+ lan_fence._clear_helper(client)
foreign.remove.assert_not_called()
- ours = MagicMock(labels={lan_fence.PROBE_LABEL: "fence"})
+ ours = _helper_container("exited", finished=_ago(3600))
client.containers.get.return_value = ours
- lan_fence._remove_helper(client)
+ lan_fence._clear_helper(client)
ours.remove.assert_called_once_with(force=True, v=True)
+def _ago(seconds):
+ return (datetime.now(UTC) - timedelta(seconds=seconds)).isoformat().replace("+00:00", "123Z")
+
+
+def _helper_container(status, created=None, finished="0001-01-01T00:00:00Z"):
+ c = MagicMock(labels={lan_fence.PROBE_LABEL: "fence"})
+ c.attrs = {
+ "Created": created or _ago(3600),
+ "State": {"Status": status, "Running": status == "running", "Restarting": False, "FinishedAt": finished},
+ }
+ return c
+
+
+def _run_helper(client):
+ mine = client.containers.create.return_value
+ mine.wait.return_value = {"StatusCode": 0}
+ mine.logs.return_value = b'{"ok": true}'
+ with patch.object(lan_fence, "_helper_image", return_value="img"), patch.object(lan_fence, "HELPER_POLL", 0):
+ return lan_fence.run_helper(client, "apply", "172.18.0.0/16"), mine
+
+
+class TestAnotherProcesssHelper:
+ """The Desktop app runs the same helper, by the same name and label, on a host that can run both."""
+
+ @pytest.mark.parametrize(
+ "theirs",
+ [
+ pytest.param(lambda: _helper_container("running"), id="running"),
+ pytest.param(lambda: _helper_container("created", created=_ago(1)), id="just-created"),
+ pytest.param(lambda: _helper_container("exited", finished=_ago(1)), id="just-exited"),
+ ],
+ )
+ def test_a_helper_still_in_use_is_waited_for_never_killed(self, theirs):
+ other = theirs()
+ client = MagicMock()
+ client.containers.get.side_effect = [other, other, NotFound("gone")]
+ result, mine = _run_helper(client)
+ assert result == {"ok": True}
+ other.remove.assert_not_called()
+ assert client.containers.get.call_count == 3, "it looked again until the other side removed it"
+ client.containers.create.assert_called_once()
+ mine.remove.assert_called_once_with(force=True, v=True)
+
+ def test_a_leftover_is_removed(self):
+ stale = _helper_container("exited", finished=_ago(3600))
+ client = MagicMock()
+ client.containers.get.return_value = stale
+ _run_helper(client)
+ stale.remove.assert_called_once_with(force=True, v=True)
+
+ def test_one_still_there_after_the_helper_timeout_is_removed(self):
+ stuck = _helper_container("running")
+ client = MagicMock()
+ client.containers.get.return_value = stuck
+ with patch.object(lan_fence, "HELPER_TIMEOUT", 0), patch.object(lan_fence, "HELPER_GRACE", 0):
+ _run_helper(client)
+ stuck.remove.assert_called_once_with(force=True, v=True)
+
+ def test_only_its_own_helper_is_removed_afterwards(self):
+ """By its own object: a name lookup after the run could find the other side's new helper."""
+ client = MagicMock()
+ client.containers.get.side_effect = NotFound("gone")
+ _, mine = _run_helper(client)
+ mine.remove.assert_called_once_with(force=True, v=True)
+ assert client.containers.get.call_count == 1, "no lookup by name after the run"
+
+ def test_a_name_conflict_from_a_racing_create_is_tried_once_more(self):
+ client = MagicMock()
+ client.containers.get.side_effect = NotFound("gone")
+ mine = client.containers.create.return_value
+ response = MagicMock(status_code=409)
+ client.containers.create.side_effect = [APIError("Conflict", response=response), mine]
+ result, _ = _run_helper(client)
+ assert result == {"ok": True}
+ assert client.containers.create.call_count == 2
+
+ def test_a_helper_that_never_started_leaves_nothing_to_remove(self):
+ client = MagicMock()
+ client.containers.get.side_effect = NotFound("gone")
+ client.containers.create.side_effect = APIError("boom", response=MagicMock(status_code=500))
+ with pytest.raises(APIError):
+ _run_helper(client)
+ assert client.containers.create.call_count == 1
+
+ def test_a_helper_that_fails_to_start_is_removed_by_its_id(self):
+ client = MagicMock()
+ client.containers.get.side_effect = NotFound("gone")
+ mine = client.containers.create.return_value
+ mine.start.side_effect = APIError("no such image layer", response=MagicMock(status_code=500))
+ with pytest.raises(APIError):
+ _run_helper(client)
+ mine.remove.assert_called_once_with(force=True, v=True)
+
+ def test_a_missing_image_is_pulled_once(self):
+ client = MagicMock()
+ client.containers.get.side_effect = NotFound("gone")
+ mine = client.containers.create.return_value
+ client.containers.create.side_effect = [ImageNotFound("img"), mine]
+ result, _ = _run_helper(client)
+ assert result == {"ok": True}
+ client.images.pull.assert_called_once_with("img")
+
+
class TestEnable:
def test_off_setting_is_refused(self):
with (
From 70a46d18d3743f645a0e355e4ccac460a80c9c67 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 14:37:52 +0100
Subject: [PATCH 04/34] fix(proxies): start a copy the cap released once its
box is back
When a cap reset or a cap change lifted the stop, each copy the cap had
stopped was started once. If its box was offline at that moment, the start
failed and nothing tried again: the copy stayed stopped, marked as stopped
by the cap, until the next reset.
Each heartbeat now starts the copies of that box that are still marked and
not running, once the proxy is no longer capped. The same mark is set when a
stop lands, the cap is lifted meanwhile, and the start back up fails.
---
app/main.py | 45 +++++++++++++++++++++-
docs/proxies.md | 1 +
tests/test_proxies.py | 89 +++++++++++++++++++++++++++++++++++++++++++
3 files changed, 133 insertions(+), 2 deletions(-)
diff --git a/app/main.py b/app/main.py
index 8ef60fff..87d9ed53 100644
--- a/app/main.py
+++ b/app/main.py
@@ -4336,7 +4336,10 @@ async def _release_capped_copies(proxy: dict[str, Any]) -> list[str]:
started.append(f"{copy['slug']} on worker {copy['worker_id']}")
except HTTPException as exc:
logger.warning(
- "Could not restart %s through proxy %s after a reset: %s", copy["slug"], proxy["id"], exc.detail
+ "Could not restart %s through proxy %s after a reset (its box's next heartbeat tries again): %s",
+ copy["slug"],
+ proxy["id"],
+ exc.detail,
)
return started
@@ -4366,6 +4369,8 @@ async def _run_proxy_cap_resets() -> None:
_proxy_cap_unstopped: dict[int, dict[tuple[int, str], str]] = {}
# Stops already on their way, so a slow worker is not sent a second one.
_proxy_cap_stopping: set[tuple[int, int, str]] = set()
+# Starts of copies the cap released, already on their way.
+_proxy_cap_starting: set[tuple[int, int, str]] = set()
def _running(entry: dict[str, Any]) -> bool:
@@ -4392,7 +4397,9 @@ async def _account_proxy_traffic(worker_id: int, entries: list[Any]) -> None:
return
# Only traffic of copies the UI recorded on this worker counts, so a worker
# cannot spend, or cap, a proxy it runs nothing through.
- recorded = {(c["proxy_id"], c["slug"]) for c in await database.list_proxied_copies() if c["worker_id"] == worker_id}
+ mine = [c for c in await database.list_proxied_copies() if c["worker_id"] == worker_id]
+ recorded = {(c["proxy_id"], c["slug"]) for c in mine}
+ held = {(c["proxy_id"], c["slug"]) for c in mine if c.get("stopped_by_cap")}
now = time.time()
for proxy_id, items in by_proxy.items():
counted = [e for e in items if (proxy_id, egress.container_slug(e)) in recorded]
@@ -4403,6 +4410,14 @@ async def _account_proxy_traffic(worker_id: int, entries: list[Any]) -> None:
state, crossed = proxies.account(proxy["state"], proxy["definition"], worker_id, counted, now)
await database.update_proxy_state(proxy_id, state)
if not state.get("capped_at"):
+ # The cap was lifted, but a copy it stopped could not be started
+ # then (the box was offline, say). This box is answering now.
+ for e in items:
+ slug = egress.container_slug(e)
+ key = (proxy_id, worker_id, slug)
+ if (proxy_id, slug) in held and not _running(e) and key not in _proxy_cap_starting:
+ _proxy_cap_starting.add(key)
+ _spawn(_start_released_copy(proxy_id, worker_id, slug))
continue
targets = {(worker_id, egress.container_slug(e)) for e in items if _running(e)}
if crossed:
@@ -4416,6 +4431,30 @@ async def _account_proxy_traffic(worker_id: int, entries: list[Any]) -> None:
_spawn(_stop_capped_proxy({**proxy, "state": state}, targets))
+async def _start_released_copy(proxy_id: int, worker_id: int, slug: str) -> None:
+ """Start one copy the cap stopped, after its release could not start it."""
+ try:
+ async with _proxy_state_lock(proxy_id):
+ proxy = await database.get_proxy(proxy_id)
+ held = any(
+ c["worker_id"] == worker_id and c["slug"] == slug and c.get("stopped_by_cap")
+ for c in await database.list_proxied_copies(proxy_id)
+ )
+ # Capped again, or the owner started or stopped it by hand since.
+ if not proxy or proxy["state"].get("capped_at") or not held:
+ return
+ await _start_proxied_copy(proxy, worker_id, slug)
+ async with _proxy_state_lock(proxy_id):
+ fresh = await database.get_proxy(proxy_id)
+ if fresh and not fresh["state"].get("capped_at"):
+ await database.set_proxied_copy_capped(proxy_id, worker_id, slug, False)
+ logger.info("Started %s through proxy %s, which the cap had stopped", slug, proxy_id)
+ except HTTPException as exc:
+ logger.warning("Could not start %s again through proxy %s: %s", slug, proxy_id, exc.detail)
+ finally:
+ _proxy_cap_starting.discard((proxy_id, worker_id, slug))
+
+
async def _stop_capped_proxy(proxy: dict[str, Any], targets: set[tuple[int, str]]) -> None:
pid = proxy["id"]
unstopped = _proxy_cap_unstopped.setdefault(pid, {})
@@ -4440,6 +4479,8 @@ async def _stop_capped_proxy(proxy: dict[str, Any], targets: set[tuple[int, str]
try:
await _start_proxied_copy(fresh, worker_id, slug)
except HTTPException as exc:
+ # Marked, so the box's next heartbeat starts it.
+ await database.set_proxied_copy_capped(pid, worker_id, slug, True)
logger.warning("Could not start %s again through proxy %s: %s", slug, pid, exc.detail)
except HTTPException as exc:
if exc.status_code == 404 and any(
diff --git a/docs/proxies.md b/docs/proxies.md
index 3790fa5b..3bc3edd1 100644
--- a/docs/proxies.md
+++ b/docs/proxies.md
@@ -230,6 +230,7 @@ A reset, on the day or by hand, starts only the copies the cap stopped; a copy
you stopped yourself stays stopped. Turning a cap on starts a fresh period, so
traffic from before it existed does not count. Turning it off, or raising it
above what this period has used, lifts the stop and starts those copies again.
+A copy whose box was offline then is started when that box next reports.
Workers report every 60 seconds, so the copies can run up to a minute past the
threshold. That is why the default is 90%, not 100%.
diff --git a/tests/test_proxies.py b/tests/test_proxies.py
index f6280c76..d2dcecf1 100644
--- a/tests/test_proxies.py
+++ b/tests/test_proxies.py
@@ -797,6 +797,95 @@ async def go():
assert proxy["state"]["used_bytes"] == 950_001_100
+async def _heartbeat_awaiting_spawns(worker_id, entries):
+ spawned: list = []
+ with patch.object(main, "_spawn", side_effect=spawned.append):
+ await main._account_proxy_traffic(worker_id, entries)
+ while spawned:
+ await spawned.pop(0)
+
+
+def _capped_copy(pid):
+ return database.update_proxy_state(
+ pid, {"used_bytes": 950_000_000, "capped_at": "2026-11-04T01:00:00+00:00", "cap_reason": "P hit"}
+ )
+
+
+class TestReleasedCopiesThatDidNotStart:
+ """A lifted cap whose restart failed: the copy's box starts it when it next reports."""
+
+ def test_the_boxs_next_heartbeat_starts_it(self, db):
+ offline = AsyncMock(side_effect=HTTPException(status_code=503, detail="Worker offline"))
+ online = AsyncMock(return_value={})
+
+ async def go():
+ pid = (await main.api_create_proxy(_Req(_socks(cap_gb=1)), {}))["id"]
+ await database.save_proxied_copy(pid, 7, "honeygain")
+ await database.set_proxied_copy_capped(pid, 7, "honeygain", True)
+ await _capped_copy(pid)
+ with patch.object(main, "_proxy_to_worker", offline):
+ await main.api_update_proxy(_Req({"cap_gb": ""}), pid, {})
+ after_release = (await database.list_proxied_copies(pid))[0]["stopped_by_cap"]
+ hb = [_entry(0, 0, slug="honeygain", proxy_id=pid, status="exited", tunnel_status="exited")]
+ with patch.object(main, "_proxy_to_worker", online):
+ await _heartbeat_awaiting_spawns(7, hb)
+ return pid, after_release, (await database.list_proxied_copies(pid))[0]["stopped_by_cap"]
+
+ pid, after_release, after_heartbeat = _run(go)
+ assert after_release == 1, "the failed start keeps the copy marked"
+ assert online.call_args.args[:3] == (7, "POST", f"/api/proxied/honeygain/{pid}/start")
+ assert after_heartbeat == 0
+ assert not main._proxy_cap_starting
+
+ @pytest.mark.parametrize(
+ ("capped", "marked", "status"),
+ [
+ pytest.param(True, True, "exited", id="capped-again"),
+ pytest.param(False, False, "exited", id="started-or-stopped-by-hand"),
+ pytest.param(False, True, "running", id="already-running"),
+ ],
+ )
+ def test_nothing_is_started_when_it_should_not_be(self, db, capped, marked, status):
+ call = AsyncMock(return_value={})
+
+ async def go():
+ pid = (await main.api_create_proxy(_Req(_socks(cap_gb=1)), {}))["id"]
+ await database.save_proxied_copy(pid, 7, "honeygain")
+ await database.set_proxied_copy_capped(pid, 7, "honeygain", marked)
+ if capped:
+ await _capped_copy(pid)
+ hb = [_entry(0, 0, slug="honeygain", proxy_id=pid, status=status, tunnel_status=status)]
+ with patch.object(main, "_proxy_to_worker", call):
+ await _heartbeat_awaiting_spawns(7, hb)
+ return pid
+
+ pid = _run(go)
+ starts = [c for c in call.call_args_list if c.args[2] == f"/api/proxied/honeygain/{pid}/start"]
+ assert starts == []
+
+ def test_a_restart_that_fails_after_a_cap_lifted_mid_stop_is_marked(self, db):
+ """The stop landed, the cap was lifted meanwhile, and the start back up failed."""
+
+ async def worker_call(worker_id, method, path, definition, **kw):
+ if path.endswith("/start"):
+ raise HTTPException(status_code=503, detail="Worker offline")
+ return {"was_running": True}
+
+ async def go():
+ pid = (await main.api_create_proxy(_Req(_socks(cap_gb=1)), {}))["id"]
+ await database.save_proxied_copy(pid, 7, "honeygain")
+ proxy = await database.get_proxy(pid)
+ with (
+ patch.object(main, "_proxy_worker_call", side_effect=worker_call),
+ patch.object(main, "_spawn", side_effect=_discard_spawn),
+ ):
+ await main._stop_capped_proxy(proxy, {(7, "honeygain")})
+ return (await database.list_proxied_copies(pid))[0]["stopped_by_cap"]
+
+ assert _run(go) == 1
+ main._drop_proxy_cap_bell(1)
+
+
class TestValidationEcho:
def test_a_422_on_a_proxy_route_does_not_echo_the_body(self):
from contextlib import asynccontextmanager
From 4efbbbc9771b60524a147ab6aeea65667e63d72c Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 14:38:05 +0100
Subject: [PATCH 05/34] fix(worker): keep removing stale activity probes while
Activity is off
With Activity off, _activity_cycle returned before its sweep, so the only
sweep was the one at worker start. A finished probe another worker left
after that (one that died and came back under a new id) stayed on the
host, and its Docker log keeps raw addresses. The cycle now checks Docker
first and, when Activity is off, runs the sweep without a round.
---
app/worker_api.py | 16 +++++++++++++---
tests/test_activity_worker.py | 33 ++++++++++++++++++++++++++++++++-
2 files changed, 45 insertions(+), 4 deletions(-)
diff --git a/app/worker_api.py b/app/worker_api.py
index 5cea9c5d..4ab49742 100644
--- a/app/worker_api.py
+++ b/app/worker_api.py
@@ -945,12 +945,22 @@ def _seconds_to_next_window(now: float | None = None) -> float:
async def _activity_cycle() -> None:
- """One round, if the owner switched Activity on and Docker answers."""
+ """One round, if the owner switched Activity on and Docker answers.
+
+ With Activity off there is no round, but probes other workers left behind
+ are still swept: their logs hold raw addresses and nothing else removes them.
+ """
global _pending_activity, _activity_seq
- if not _activity_enabled:
- return
if not await asyncio.to_thread(orchestrator.docker_available):
return
+ if not _activity_enabled:
+ client = await asyncio.to_thread(activity._client)
+ try:
+ await asyncio.to_thread(activity.sweep, client)
+ finally:
+ with contextlib.suppress(Exception):
+ client.close()
+ return
switches = _activity_switches
result = await asyncio.to_thread(activity.run_round)
if not _activity_enabled or _activity_switches != switches:
diff --git a/tests/test_activity_worker.py b/tests/test_activity_worker.py
index 2c26b2ae..74219066 100644
--- a/tests/test_activity_worker.py
+++ b/tests/test_activity_worker.py
@@ -484,9 +484,11 @@ def wapi():
with (
patch("app.worker_api.orchestrator.docker_available", return_value=True) as docker_ok,
patch("app.worker_api.activity.run_round", return_value=None) as run_round,
+ patch("app.worker_api.activity._client", return_value=MagicMock()),
+ patch("app.worker_api.activity.sweep", return_value=[]),
):
asyncio.run(worker_api._activity_cycle())
- assert not docker_ok.called and not run_round.called, "a round ran before any reply"
+ assert not run_round.called, "a round ran before any reply"
reply = MagicMock()
reply.json.return_value = {"activity": {"enabled": True}}
worker_api._apply_activity_reply(reply, None)
@@ -523,10 +525,39 @@ def test_disabled_never_runs_a_round(self, wapi):
with (
patch("app.worker_api.orchestrator.docker_available", return_value=True),
patch("app.worker_api.activity.run_round") as run_round,
+ patch("app.worker_api.activity._client", return_value=MagicMock()),
+ patch("app.worker_api.activity.sweep", return_value=[]),
):
asyncio.run(wapi._activity_cycle())
assert not run_round.called
+ def test_disabled_still_sweeps_probes_other_workers_left(self, wapi):
+ """With Activity off the sweep at worker start is not the last one: a probe another
+ worker leaves later (it died and came back under a new id) holds raw addresses in
+ its log, and only a sweep removes it."""
+ wapi._activity_enabled = False
+ client = MagicMock()
+ with (
+ patch("app.worker_api.orchestrator.docker_available", return_value=True),
+ patch("app.worker_api.activity.run_round") as run_round,
+ patch("app.worker_api.activity._client", return_value=client),
+ patch("app.worker_api.activity.sweep", return_value=[]) as sweep,
+ ):
+ asyncio.run(wapi._activity_cycle())
+ sweep.assert_called_once_with(client)
+ client.close.assert_called_once()
+ assert not run_round.called
+
+ def test_disabled_without_docker_does_not_sweep(self, wapi):
+ wapi._activity_enabled = False
+ with (
+ patch("app.worker_api.orchestrator.docker_available", return_value=False),
+ patch("app.worker_api.activity._client") as make_client,
+ patch("app.worker_api.activity.sweep") as sweep,
+ ):
+ asyncio.run(wapi._activity_cycle())
+ assert not make_client.called and not sweep.called
+
def test_enabled_runs_a_round_negative_control(self, wapi):
"""The control for the test above: the same call with the switch on runs."""
wapi._activity_enabled = True
From f1102a9cca41d5d34da604aa372d9d48611b8e76 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 14:40:26 +0100
Subject: [PATCH 06/34] docs(proxies): say SOCKS5 tunnels carry TCP only, as
cp-tunnel enforces
docs/proxies.md said a SOCKS5 proxy carries UDP when the provider supports
UDP ASSOCIATE. cp-tunnel lets a socks5 or http tunnel out only over TCP to
the proxy's port, so UDP relays are dropped. The page now says SOCKS5 and
HTTP CONNECT carry TCP only and a Tailscale exit node carries UDP, matching
the comparison page. A test reads the tunnel's egress rule and fails if the
page promises UDP through SOCKS5 again.
---
docs/proxies.md | 9 ++++----
tests/test_docs_proxy_udp.py | 44 ++++++++++++++++++++++++++++++++++++
2 files changed, 49 insertions(+), 4 deletions(-)
create mode 100644 tests/test_docs_proxy_udp.py
diff --git a/docs/proxies.md b/docs/proxies.md
index 3790fa5b..beefaa40 100644
--- a/docs/proxies.md
+++ b/docs/proxies.md
@@ -12,9 +12,9 @@ under its own device name (`cashpilot-box-p3` for proxy 3).
| Type | What you give it | Notes |
|---|---|---|
-| SOCKS5 | host, port, optional user and password | Carries UDP only if the provider supports SOCKS5 UDP ASSOCIATE. |
+| SOCKS5 | host, port, optional user and password | TCP only: the tunnel lets out only TCP to the proxy's port, so SOCKS5 UDP relays are dropped. Name lookups still work (see [DNS](#dns)). |
| HTTP CONNECT | host, port, optional user and password | TCP only. Name lookups still work (see [DNS](#dns)). |
-| Tailscale exit node | an auth key and the exit node's name or 100.x address | Traffic leaves from the exit node's connection. See [Tailscale exit nodes](#tailscale-exit-nodes). |
+| Tailscale exit node | an auth key and the exit node's name or 100.x address | Traffic leaves from the exit node's connection, UDP included. See [Tailscale exit nodes](#tailscale-exit-nodes). |
Any provider works. Nothing in CashPilot is specific to one.
@@ -168,8 +168,9 @@ A service is refused when its catalog entry needs the host:
- terms that forbid proxies or VPNs: `requirements.proxy_prohibited` in the
catalog, with a link to the provider's own terms.
-Services that need UDP do not work through HTTP CONNECT, and through SOCKS5
-only when the provider supports UDP ASSOCIATE.
+Services that need UDP do not work through SOCKS5 or HTTP CONNECT: the
+tunnel lets out only TCP to the proxy's address and port. A Tailscale exit
+node carries UDP.
## Exit IPs and grouping
diff --git a/tests/test_docs_proxy_udp.py b/tests/test_docs_proxy_udp.py
new file mode 100644
index 00000000..d7e14dd9
--- /dev/null
+++ b/tests/test_docs_proxy_udp.py
@@ -0,0 +1,44 @@
+"""docs/proxies.md must not promise UDP through a SOCKS5 or HTTP proxy.
+
+tunnel/cp-tunnel lets a socks5 or http tunnel out only over TCP to the
+proxy's own address and port. A SOCKS5 UDP ASSOCIATE relay sends UDP to the
+proxy, so the firewall drops it. The page used to say SOCKS5 carries UDP when
+the provider supports UDP ASSOCIATE.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+TUNNEL = ROOT / "tunnel" / "cp-tunnel"
+PAGE = ROOT / "docs" / "proxies.md"
+
+UDP_PROMISE = re.compile(r"(carries|works?|through) [^.|]*udp associate", re.IGNORECASE)
+
+
+def _socks_egress_rule() -> str:
+ branch = TUNNEL.read_text(encoding="utf-8").split("socks5 | http)", 1)[1].split("tailscale)", 1)[0]
+ rules = [line.strip() for line in branch.splitlines() if line.strip().startswith("EGRESS_RULE=")]
+ assert len(rules) == 1, "expected one EGRESS_RULE in the socks5 | http branch of cp-tunnel"
+ return rules[0]
+
+
+def promises_udp(text: str) -> bool:
+ return bool(UDP_PROMISE.search(re.sub(r"\s+", " ", text)))
+
+
+def test_the_socks_tunnel_lets_out_only_tcp():
+ rule = _socks_egress_rule()
+ assert "tcp dport $PROXY_PORT" in rule
+ assert "udp" not in rule, "cp-tunnel now lets UDP out; this test's premise no longer holds"
+
+
+def test_the_page_does_not_promise_udp_through_socks5():
+ assert not promises_udp(PAGE.read_text(encoding="utf-8"))
+
+
+def test_the_check_catches_the_old_claims():
+ assert promises_udp("| SOCKS5 | host | Carries UDP only if the provider supports SOCKS5 UDP ASSOCIATE. |")
+ assert promises_udp("and through SOCKS5\nonly when the provider supports UDP ASSOCIATE.")
From a59014c533a562b9ecb9863c5d80a42c31aee39b Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 14:41:23 +0100
Subject: [PATCH 07/34] fix(isolation): let Podman workers deploy instead of
failing on a fence they cannot hold
Podman's Docker-compatible API drops the network options the fence relies
on (the interface name, masquerade off, inter-container traffic off), so on
a Podman worker every fenced deploy failed the network check, and a fresh
install defaults to fenced.
The worker now detects Podman from /version and resolves to a new
unsupported mode: earners deploy on Podman's default network, the fleet page
and the isolation check say they are not fenced, and turning isolation on is
refused with the reason. With CASHPILOT_ISOLATION=on a deploy is refused
before the running container is touched, since the fence was asked for.
---
app/isolation_check.py | 6 +++
app/lan_fence.py | 41 +++++++++++++++++++
app/templates/fleet.html | 4 +-
docs/isolation-check.md | 3 +-
docs/isolation.md | 6 ++-
tests/test_lan_fence.py | 86 ++++++++++++++++++++++++++++++++++++++++
6 files changed, 143 insertions(+), 3 deletions(-)
diff --git a/app/isolation_check.py b/app/isolation_check.py
index 029684dc..dcf59f65 100644
--- a/app/isolation_check.py
+++ b/app/isolation_check.py
@@ -413,6 +413,12 @@ def placement_findings(earners: list[Any], network: str, mode: str | None = None
"and can reach your LAN and this host. Turn it on with the button on this worker's row",
ISOLATION_PAGE,
)
+ elif mode == lan_fence.MODE_UNSUPPORTED:
+ why, docs = (
+ "This worker runs on Podman, which CashPilot cannot fence yet, so these earners sit on Podman's "
+ "default network and can reach your LAN and this host",
+ ISOLATION_PAGE,
+ )
elif mode == lan_fence.MODE_OFF:
why, docs = (
"Isolation is turned off on this worker with CASHPILOT_ISOLATION=off, so these earners sit on "
diff --git a/app/lan_fence.py b/app/lan_fence.py
index c7f8b31b..6365df12 100644
--- a/app/lan_fence.py
+++ b/app/lan_fence.py
@@ -42,6 +42,13 @@
``off`` deletes the file.
* ``legacy``: an upgraded install whose earners are on the default bridge.
Nothing changes until the owner turns isolation on from the fleet page.
+* ``unsupported``: a Podman engine, with the setting not ``off``. Podman's
+ Docker-compatible API keeps the interface name out of the network's options
+ and drops the masquerade and inter-container options, and netavark adds its
+ own masquerade, so none of what the fence relies on can be set or checked
+ there. Earners deploy on the default network and the fleet page says they
+ are not fenced; with CASHPILOT_ISOLATION=on, deploys are refused, since on
+ was asked for and cannot be given.
The top of this module is stdlib only: the helper container imports it to
render the rules, and the UI image imports its constants.
@@ -76,6 +83,12 @@
MODE_OFF = "off"
MODE_ON = "on"
MODE_LEGACY = "legacy"
+MODE_UNSUPPORTED = "unsupported"
+
+PODMAN_UNSUPPORTED = (
+ "CashPilot cannot fence earners off the LAN on Podman yet: its Docker-compatible API does not keep "
+ "the network settings the fence relies on"
+)
#: Seconds between checks that the table is still in place.
RECHECK_SECONDS = 300
@@ -627,6 +640,22 @@ def _remember(on: bool) -> None:
)
+def is_podman(client: Any) -> bool:
+ """A Podman engine behind the Docker-compatible API.
+
+ Podman names itself in /version ("Podman Engine"). An engine that does not
+ answer is taken for Docker: the fence's own checks still refuse a network
+ it cannot hold.
+ """
+ try:
+ version = client.version() or {}
+ except Exception: # noqa: BLE001 - taken for Docker, as above
+ return False
+ names = [str((version.get("Platform") or {}).get("Name") or "")]
+ names += [str((c or {}).get("Name") or "") for c in version.get("Components") or []]
+ return any("podman" in n.lower() for n in names)
+
+
def resolve_mode(client: Any) -> str:
if _orchestrator()._CONTAINER_NETWORK:
return MODE_OPERATOR
@@ -634,6 +663,8 @@ def resolve_mode(client: Any) -> str:
if chosen == "off":
_remember(False)
return MODE_OFF
+ if is_podman(client):
+ return MODE_UNSUPPORTED
if (
chosen == "on"
or _remembered_on()
@@ -906,6 +937,12 @@ def network_for_deploy() -> str | None:
client = orch._get_client()
mode = resolve_mode(client)
_state["mode"] = mode
+ if mode == MODE_UNSUPPORTED and setting() == "on":
+ raise orch.ContainerNetworkError(
+ f"CASHPILOT_ISOLATION=on, but {PODMAN_UNSUPPORTED}. The running container was not touched. "
+ "Set CASHPILOT_ISOLATION=auto or off on this worker to deploy on Podman's default network, "
+ "without the fence (docs/isolation.md)."
+ )
if mode != MODE_ON:
return None
try:
@@ -971,6 +1008,8 @@ def reconcile() -> dict[str, Any]:
if mode == MODE_ON:
with contextlib.suppress(RuntimeError, IsolationConflict):
apply_fence(client)
+ elif mode == MODE_UNSUPPORTED:
+ _set_fence("unsupported", error=PODMAN_UNSUPPORTED + ".")
elif _needs_reduced(client, mode, orch._CONTAINER_NETWORK):
try:
reduce_fence(client)
@@ -1045,6 +1084,8 @@ def enable() -> dict[str, Any]:
"This worker has CASHPILOT_ISOLATION=off. Remove that setting and restart the worker to turn isolation on."
)
client = orch._get_client()
+ if is_podman(client):
+ raise IsolationConflict(PODMAN_UNSUPPORTED + ".")
apply_fence(client)
_remember(True)
_state["mode"] = MODE_ON
diff --git a/app/templates/fleet.html b/app/templates/fleet.html
index e54ce7fd..94d12513 100644
--- a/app/templates/fleet.html
+++ b/app/templates/fleet.html
@@ -480,7 +480,9 @@
Proxies
? ` `
: '';
let line = '';
- if (iso.mode === 'legacy') {
+ if (iso.mode === 'unsupported') {
+ line = 'This worker runs on Podman, which CashPilot cannot fence yet, so its earners sit on Podman\'s default network and can reach your LAN and this host.';
+ } else if (iso.mode === 'legacy') {
line = `Earners on this worker can reach your LAN and this host: ${esc(outside.join(', ') || 'none running')}.${button('Turn on isolation')}${stoppedNote}`;
} else if (iso.mode === 'on' && fence.state === 'failed') {
line = `The LAN fence is not in place on this worker: ${esc(fence.error || 'unknown error')}. While it is missing, earners on its network can reach this host's own services, and they may still reach the internet if another masquerade rule on the host covers them. Deploys are refused until it is back. To run without the fence, set CASHPILOT_ISOLATION=off on the worker.`;
diff --git a/docs/isolation-check.md b/docs/isolation-check.md
index 76566393..90caec58 100644
--- a/docs/isolation-check.md
+++ b/docs/isolation-check.md
@@ -105,7 +105,8 @@ What CashPilot's own fence has been tested on:
| Linux Docker with nf_tables (tested on Ubuntu 24.04, Docker 29.5, inside colima) | tested: LAN, host, tailnet and metadata blocked, internet works, fails closed without the table |
| The same host forced onto `iptables-legacy` (older NAS kernels) | tested, same results |
| Docker Desktop on Windows and macOS | not tested. It uses the same VM model as colima, so it is expected to work, unless Enhanced Container Isolation blocks the helper |
-| Rootless Docker, Podman | not tested |
+| Rootless Docker | not tested |
+| Podman | not fenced: detected, earners stay on its default network, and the fleet page says so |
| Unraid, Synology, QNAP | not tested |
When the fence cannot be written on a host, the worker refuses deploys and says
diff --git a/docs/isolation.md b/docs/isolation.md
index fdd9e9ae..aaa3945e 100644
--- a/docs/isolation.md
+++ b/docs/isolation.md
@@ -114,6 +114,7 @@ otherwise (older NAS kernels). Both write the same rules.
|---|---|---|
| on | a new install, `CASHPILOT_ISOLATION=on`, the `cashpilot-isolated` network CashPilot created exists, an earner is still bound to it after Unraid removed it, or the worker recorded it in `.isolation_on` in its data folder (`CASHPILOT_DATA_DIR`, `/data` by default) | earners join the fenced network |
| legacy | an upgraded install whose earners already sit on Docker's default bridge | nothing changes on upgrade; the fleet page offers **Turn on isolation** |
+| unsupported | a Podman engine, with `CASHPILOT_ISOLATION` not `off` | earners deploy on Podman's default network, not fenced, and the fleet page says so; with `CASHPILOT_ISOLATION=on`, deploys are refused, since the fence was asked for and cannot be given |
| off | `CASHPILOT_ISOLATION=off` | Docker's default bridge, as before; earners left on the fenced network keep their internet until you redeploy them |
| your own network | `CASHPILOT_CONTAINER_NETWORK` is set | that network, with your own rules, as before; CashPilot writes no table, except the NAT alone while earners are still on `cashpilot-isolated`, so they keep their internet, without the fence's drops, until you redeploy them |
@@ -169,7 +170,10 @@ off button on purpose: one click should not take the fence down.
host) is not dropped. Use a VLAN or your own rules.
- Tested on Linux Docker 29 with nf_tables (colima on macOS, the same VM model
as Docker Desktop), including the `iptables-legacy` fallback. Docker Desktop
- itself, rootless Docker, Podman and Unraid are not tested yet. On Docker
+ itself, rootless Docker and Unraid are not tested yet. Podman is not fenced:
+ its Docker-compatible API drops the network settings the fence relies on
+ (the interface name, masquerade off, inter-container traffic off), so the
+ worker detects it and leaves earners on its default network. On Docker
Desktop, the Enhanced Container Isolation setting may block the helper; the
deploy is then refused with the opt-out in the message.
diff --git a/tests/test_lan_fence.py b/tests/test_lan_fence.py
index 9008b934..6590919d 100644
--- a/tests/test_lan_fence.py
+++ b/tests/test_lan_fence.py
@@ -443,10 +443,24 @@ def _get(name):
client.networks.get.side_effect = _get
client.networks.create.side_effect = lambda name, **kw: nets.setdefault(name, _network(name))
client.containers.list.return_value = list(earners)
+ client.version.return_value = {"Platform": {"Name": "Docker Engine - Community"}, "Components": []}
client._nets = nets
return client
+#: /version as Podman 5 answers it on its Docker-compatible socket.
+PODMAN_VERSION = {
+ "Platform": {"Name": "linux/arm64/fedora-42"},
+ "Components": [{"Name": "Podman Engine", "Version": "5.6.1"}, {"Name": "Conmon", "Version": "2.1.13"}],
+}
+
+
+def _podman(managed=None, earners=()):
+ client = _docker(managed, earners)
+ client.version.return_value = PODMAN_VERSION
+ return client
+
+
def _mode(client, setting="auto", operator=""):
with patch.object(lan_fence, "_SETTING", setting), patch.object(orchestrator, "_CONTAINER_NETWORK", operator):
return lan_fence.resolve_mode(client)
@@ -966,6 +980,78 @@ def test_it_returns_the_earners_to_move(self):
assert lan_fence.NETWORK_NAME in client._nets
+class TestPodman:
+ """Podman's Docker-compatible API cannot hold the fence's network, so it is never fenced."""
+
+ @pytest.mark.parametrize(
+ ("version", "podman"),
+ [
+ (PODMAN_VERSION, True),
+ ({"Platform": {"Name": "Podman Engine"}}, True),
+ ({"Platform": {"Name": "Docker Engine - Community"}, "Components": [{"Name": "Engine"}]}, False),
+ ({}, False),
+ ],
+ )
+ def test_it_is_detected_from_the_version(self, version, podman):
+ client = MagicMock()
+ client.version.return_value = version
+ assert lan_fence.is_podman(client) is podman
+
+ def test_an_engine_that_does_not_answer_is_taken_for_docker(self):
+ client = MagicMock()
+ client.version.side_effect = OSError("no answer")
+ assert lan_fence.is_podman(client) is False
+
+ @pytest.mark.parametrize("setting", ["auto", "on"])
+ def test_it_resolves_to_unsupported_and_records_nothing(self, setting):
+ assert _mode(_podman(), setting) == lan_fence.MODE_UNSUPPORTED
+ assert not lan_fence.ON_MARKER.exists()
+
+ def test_off_and_the_operators_network_still_win(self):
+ assert _mode(_podman(), "off") == lan_fence.MODE_OFF
+ assert _mode(_podman(), operator="mine") == lan_fence.MODE_OPERATOR
+
+ def test_a_deploy_goes_to_the_default_network_without_the_fence(self):
+ client = _podman()
+ kwargs, helper = _deploy(client, "auto")
+ assert kwargs["network"] is None
+ helper.assert_not_called()
+ client.networks.create.assert_not_called()
+
+ def test_with_isolation_on_a_deploy_is_refused_before_the_old_container_is_touched(self):
+ old = MagicMock()
+ old.labels = {"cashpilot.service": "honeygain"}
+ client = _podman()
+ with pytest.raises(orchestrator.ContainerNetworkError, match="Podman"):
+ _deploy(client, "on", old=old)
+ old.stop.assert_not_called()
+ old.remove.assert_not_called()
+ client.containers.run.assert_not_called()
+
+ def test_the_check_reports_it_and_runs_no_helper(self):
+ state, helper = _reconcile(_podman(), "auto")
+ helper.assert_not_called()
+ assert state["mode"] == "unsupported" and state["network"] is None
+ assert state["fence"]["state"] == "unsupported" and "Podman" in state["fence"]["error"]
+
+ def test_turning_it_on_is_refused(self):
+ client = _podman(earners=[_container("cashpilot-earnapp")])
+ with (
+ patch.object(lan_fence, "_SETTING", "auto"),
+ patch.object(orchestrator, "_CONTAINER_NETWORK", ""),
+ patch.object(orchestrator, "_get_client", return_value=client),
+ patch.object(lan_fence, "run_helper") as helper,
+ pytest.raises(lan_fence.IsolationConflict, match="Podman"),
+ ):
+ lan_fence.enable()
+ helper.assert_not_called()
+ assert not lan_fence.ON_MARKER.exists()
+
+ def test_the_isolation_check_says_why_the_earners_are_exposed(self):
+ found = isolation_check.placement_findings([_container("cashpilot-earnapp")], "", lan_fence.MODE_UNSUPPORTED)
+ assert found[0]["kind"] == "no_isolated_network" and "Podman" in found[0]["detail"]
+
+
class TestTheUiRoute:
def test_owner_only(self):
ui_app.router.lifespan_context = _noop_lifespan
From 141c62845e2dab4ce1c9013151f579c31f2a936b Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 14:43:05 +0100
Subject: [PATCH 08/34] docs: say the worker reaches Docker through the socket
proxy in CLAUDE.md and two pages
CLAUDE.md still said two containers, a worker that must have Docker socket
access and must be privileged. Since v1.71.0 the worker reaches Docker only
through cashpilot-socket-proxy, and since v1.73.0 owner isolation limits it
to what it created. faq.md and how-it-works.md said the same. The
socket-proxy docs test now covers CLAUDE.md and these phrasings.
---
CLAUDE.md | 8 ++++----
docs/faq.md | 2 +-
docs/how-it-works.md | 2 +-
.../test_docs_say_the_socket_proxy_holds_the_socket.py | 10 +++++++++-
4 files changed, 15 insertions(+), 7 deletions(-)
diff --git a/CLAUDE.md b/CLAUDE.md
index 17e652d9..de201453 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -41,7 +41,7 @@ python -m zensical build --clean --strict
python -I .github/scripts/site-check.py site
```
-Two containers: `cashpilot-ui` (port 8080, web dashboard + earnings collection) and `cashpilot-worker` (port 8081, Docker agent requiring socket access).
+Three long-running containers in `docker-compose.yml`: `cashpilot-ui` (port 8080, web dashboard + earnings collection), `cashpilot-worker` (port 8081, Docker agent) and `cashpilot-socket-proxy` (holds the host's Docker socket). Since v1.71.0 the worker has no Docker socket: it reaches Docker through the proxy's filtered socket (`DOCKER_HOST=unix:///run/cashpilot-docker/docker.sock`), and since v1.73.0 the proxy's owner isolation limits it to the containers, volumes and networks it created. See `docs/security-defaults.md`.
### Testing traps
@@ -114,7 +114,7 @@ CashPilot is split into two **always-separate** components. The UI never touches
| Component | Description |
|-----------|-------------|
| **CashPilot UI** | The single web dashboard. Collects all earnings centrally, shows global fleet view, manages workers. **Has NO Docker socket.** Can be hosted anywhere. Only ONE UI instance exists. |
-| **CashPilot Worker** | Agent running on each server that has Docker. **Must have Docker socket access**. Manages local containers, reports status to UI via heartbeats. Has a minimal config page. |
+| **CashPilot Worker** | Agent running on each server that has Docker. **Reaches Docker through the socket proxy** (`cashpilot-socket-proxy`), never the raw socket. Manages local containers, reports status to UI via heartbeats. Has a minimal config page. |
Two separate Docker images:
- **`drumsergio/cashpilot`** — UI image: FastAPI, Jinja2, templates, static assets, collectors, APScheduler. **No Docker SDK.**
@@ -125,7 +125,7 @@ Two separate Docker images:
### Core Principles
1. **Separation of concerns.** UI handles: dashboard, earnings collection, scheduling, user auth. Workers handle: Docker container lifecycle, health reporting.
-2. **Workers must be privileged.** A worker without Docker socket is useless.
+2. **Workers need Docker, through the socket proxy.** A worker without Docker access is useless, but it never gets the raw socket: the proxy refuses privileged containers, binds of `/` or the socket, and anything the worker did not create.
3. **Single source of truth.** The UI instance is the only one that collects earnings and stores historical data.
4. **Earnings are never duplicated.** Since only the UI collects, there is no risk of double-counting.
5. **Workers are stateless satellites.** A worker knows which containers to keep running and the UI URL to report to.
@@ -204,7 +204,7 @@ filtered on `service` silently matched nothing in production. Use
## Key Rules
- Never hardcode credentials; all secrets via environment variables
- Docker images: `drumsergio/cashpilot` with semver tags, never `:latest`
-- Worker requires Docker socket access (`/var/run/docker.sock`)
+- Worker reaches Docker only through the socket proxy (`DOCKER_HOST=unix:///run/cashpilot-docker/docker.sock`); the worker never mounts `/var/run/docker.sock` (the proxy does, and so does the one-shot `cashpilot-migrate`)
- Always use bind mounts (not named volumes) on Unraid
- License: GPL-3.0
diff --git a/docs/faq.md b/docs/faq.md
index 167fd13e..636e8f03 100644
--- a/docs/faq.md
+++ b/docs/faq.md
@@ -38,7 +38,7 @@ If CashPilot cannot write the key to disk at all, because `/data` is unwritable
**What about security?**
-Every service CashPilot deploys runs in its own container with every Linux capability dropped (a few services add back only the ones they declare) and `--security-opt no-new-privileges` set, so a process inside cannot gain privileges. Containers cannot see your host filesystem beyond the volumes a service declares. They CAN reach your local network by default, because Docker does not block that; [Protecting Your Home Network](home-network-security.md) shows how to put them on a firewalled bridge that reaches the internet but not your LAN, router or host. That bridge does not cover a service on host networking (Mysterium, today); the guide says what to do about those. Service credentials are encrypted at rest using Fernet symmetric encryption. Only the worker container requires Docker socket access; the UI container has no privileged access.
+Every service CashPilot deploys runs in its own container with every Linux capability dropped (a few services add back only the ones they declare) and `--security-opt no-new-privileges` set, so a process inside cannot gain privileges. Containers cannot see your host filesystem beyond the volumes a service declares. They CAN reach your local network by default, because Docker does not block that; [Protecting Your Home Network](home-network-security.md) shows how to put them on a firewalled bridge that reaches the internet but not your LAN, router or host. That bridge does not cover a service on host networking (Mysterium, today); the guide says what to do about those. Service credentials are encrypted at rest using Fernet symmetric encryption. The UI has no Docker access, and the worker is never given the raw Docker socket: it reaches Docker through `cashpilot-socket-proxy`, which refuses privileged containers and anything the worker did not create ([security defaults](security-defaults.md)).
That said, no setup is bulletproof. You are still running third-party software that routes external traffic through your network. Docker isolation significantly reduces the attack surface compared to running these services directly on your host, but it does not eliminate all risk. We recommend running CashPilot on a dedicated machine or VLAN, keeping Docker and your host OS up to date, and reviewing the open-source code of any service before deploying it.
diff --git a/docs/how-it-works.md b/docs/how-it-works.md
index 762aa049..8d087e78 100644
--- a/docs/how-it-works.md
+++ b/docs/how-it-works.md
@@ -14,7 +14,7 @@ There is **no standalone mode**. Every server that runs Docker containers needs
## Design Principles
1. **Separation of concerns.** UI handles: dashboard, earnings collection, scheduling, user auth. Workers handle: Docker container lifecycle, health reporting. They never overlap.
-2. **Workers must be privileged.** A worker without Docker socket is useless. If you don't need Docker management, just run the UI alone to track earnings.
+2. **Workers need Docker, through the socket proxy.** A worker without Docker access is useless, but it never gets the raw socket: it reaches Docker through `cashpilot-socket-proxy`. If you don't need Docker management, just run the UI alone to track earnings.
3. **Single source of truth.** The UI instance is the only one that collects earnings, stores historical data, and serves the dashboard. Workers never collect earnings.
4. **Earnings are never duplicated.** Since only the UI collects, there is no risk of the same account being counted twice.
5. **Workers are stateless satellites.** A worker knows which containers to keep running and the UI URL to report to. It persists only its own id and fleet key to local files (`.worker_id`, `.worker_key`; no database, no earnings data).
diff --git a/tests/test_docs_say_the_socket_proxy_holds_the_socket.py b/tests/test_docs_say_the_socket_proxy_holds_the_socket.py
index fe7ffd0f..d636aa99 100644
--- a/tests/test_docs_say_the_socket_proxy_holds_the_socket.py
+++ b/tests/test_docs_say_the_socket_proxy_holds_the_socket.py
@@ -20,7 +20,8 @@
# Pages describing what the shipped compose file installs today. Changelogs,
# upgrade notes and research notes describe the past on purpose.
-PAGES = [ROOT / "README.md", *sorted((ROOT / "docs").glob("*.md"))]
+# CLAUDE.md is read by every agent that changes the code, so it counts too.
+PAGES = [ROOT / "README.md", ROOT / "CLAUDE.md", *sorted((ROOT / "docs").glob("*.md"))]
STALE = [
r"\btwo (docker )?containers\b",
@@ -28,6 +29,9 @@
r"\bonly the worker (holds|mounts|touches) the docker socket\b",
r"\bboth can command the docker socket\b",
r"\bdoes not keep the containers off your lan by itself\b",
+ r"\brequiring (docker )?socket access\b",
+ r"\b(must|requires?) (have )?docker socket access\b",
+ r"\bworkers must be privileged\b",
]
# Each entry: a page and a pattern that is true there, with the reason.
@@ -64,6 +68,10 @@ def test_no_page_says_the_worker_holds_the_socket(page: Path):
"A worker holds the Docker socket on its server.",
"Only the worker\ntouches the Docker socket.",
"It does not keep the containers off your LAN by itself.",
+ "cashpilot-worker (port 8081, Docker agent requiring socket access)",
+ "Agent running on each server. **Must have Docker socket access**.",
+ "- Worker requires Docker socket access (`/var/run/docker.sock`)",
+ "2. **Workers must be privileged.** A worker without Docker socket is useless.",
],
)
def test_the_check_catches_each_old_claim(text: str):
From 63cb8f89ff5d74ac3769a612c102d337c2600d93 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 14:45:59 +0100
Subject: [PATCH 09/34] docs(storj): link the sources for the Chapter 11 and
held-back claims
The guide gave the filing date, the freeze on pre-filing node earnings, the
token drop, the exchange exits and the held-back schedule with no source.
Each now links Storj's own announcement, its payout update on the official
forum, the Storj docs, CoinGecko, Binance and Coinbase. The token drop reads
'more than 98%', which CoinGecko shows today.
---
docs/guides/storj.md | 4 ++--
tests/test_storj_guide_sources.py | 36 +++++++++++++++++++++++++++++++
2 files changed, 38 insertions(+), 2 deletions(-)
create mode 100644 tests/test_storj_guide_sources.py
diff --git a/docs/guides/storj.md b/docs/guides/storj.md
index 54f44ebe..b79fc161 100644
--- a/docs/guides/storj.md
+++ b/docs/guides/storj.md
@@ -9,7 +9,7 @@
Storj is a decentralized cloud storage network where you earn by renting out your unused disk space. Run the storage node via Docker and get paid $1.35/TB stored per month plus $1/TB egress (rates since 1 September 2026). Payouts are monthly in USDC, a US-dollar stablecoin, on Ethereum mainnet to a wallet you control, once the balance clears a fee-based minimum (STORJ token until the September 2026 payout). Requires at least 550GB of available disk space and a stable internet connection. No account or signup needed -- nodes are permissionless since mid-2025.
!!! warning "Storj Labs is in Chapter 11 (October 2026)"
- Storj Labs filed for Chapter 11 on 26 July 2026 (US Bankruptcy Court, Northern District of West Virginia, case 5:26-bk-00512) to restructure legacy debt. The network keeps running and Storj keeps paying active nodes, but node earnings from before the filing are frozen until the court rules on them. The STORJ token is down about 98% from its 2021 peak. Binance delisted it on 3 September 2026, and Coinbase suspended trading in it on 28 September 2026. A new node withholds part of its earnings during its first nine months and fills only as customer demand allows, so starting one now ties a long payback to a company in restructuring that could still wind down.
+ Storj Labs filed for Chapter 11 on 26 July 2026 (US Bankruptcy Court, Northern District of West Virginia, case 5:26-bk-00512) to restructure legacy debt ([Storj's announcement](https://www.storj.io/blog/storj-announces-voluntary-financial-restructuring-to-resolve-legacy-liabilities-and-position-the-business-for-growth/)). The network keeps running and Storj keeps paying active nodes, but node earnings from before the filing are frozen until the court rules on them ([Storj's payout update](https://forum.storj.io/t/storagenode-payout-update-august-prepayments-and-july-2026-status/32266)). The STORJ token is down more than 98% from its March 2021 peak ([CoinGecko](https://www.coingecko.com/en/coins/storj), November 2026). Binance delisted it on 3 September 2026 ([Binance](https://www.binance.com/en/support/announcement/detail/d72915ed7a60473b92f0818d959a227a)), and Coinbase suspended trading in it on 28 September 2026 ([Coinbase status](https://status.exchange.coinbase.com/incidents/bczrbdtb1fd5)). A new node withholds part of its earnings during its first nine months and fills only as customer demand allows, so starting one now ties a long payback to a company in restructuring that could still wind down.
## Earning Estimates
@@ -98,7 +98,7 @@ Pick the boxes under **Deploy to** and click **Deploy Storj**. CashPilot creates
- **Use a DDNS hostname for `ADDRESS`, never a literal IP.** Satellites dial your node back at exactly the address it advertises. Residential ISPs re-provision public IPs without notice, and when that happens a node advertising the old literal IP is counted offline around the clock (container healthy, dashboard green, earnings from new data at zero) until someone notices the provider's offline emails. A dynamic-DNS hostname (DuckDNS, Cloudflare DDNS, your router's DDNS client) self-heals through an IP change. If satellites cannot reach the node, CashPilot's producer state for Storj reports it with the exact reason; you can also check yourself with `curl -s http://:14002/api/sno/` (`lastPinged` stale or `quicStatus: "Misconfigured"` means unreachable).
- **No signup required**: Storj nodes are permissionless since mid-2025. No auth token, no account creation.
-- **Held-back earnings**: a new node's satellites withhold 75% of its earnings in months 1 to 3, 50% in months 4 to 6 and 25% in months 7 to 9. Half of the held amount comes back in month 16; the rest only when the node exits gracefully. Earnings from before Storj Labs' 26 July 2026 Chapter 11 filing are frozen until the bankruptcy court rules on them.
+- **Held-back earnings**: a new node's satellites withhold 75% of its earnings in months 1 to 3, 50% in months 4 to 6 and 25% in months 7 to 9. Half of the held amount comes back in month 16; the rest only when the node exits gracefully ([Storj docs](https://storj.dev/node/faq/held-back-amount)). Earnings from before Storj Labs' 26 July 2026 Chapter 11 filing are frozen until the bankruptcy court rules on them ([Storj's payout update](https://forum.storj.io/t/storagenode-payout-update-august-prepayments-and-july-2026-status/32266)).
- **One node per public IP**: Storj recommends one node per public IP for optimal satellite allocation. Nodes on the same /24 subnet share data allocation.
- **Uptime matters**: Nodes with poor uptime get less data. Aim for 99.5%+ uptime.
- **Disk selection**: Always use spinning disks (HDD). Storj data is cold storage: IOPS don't matter, capacity does. SSDs offer no advantage and cost more per TB.
diff --git a/tests/test_storj_guide_sources.py b/tests/test_storj_guide_sources.py
new file mode 100644
index 00000000..12d9b6ce
--- /dev/null
+++ b/tests/test_storj_guide_sources.py
@@ -0,0 +1,36 @@
+"""The Storj guide's Chapter 11 and held-back claims each name their source.
+
+The guide stated the filing date, the freeze on pre-filing node earnings and
+the held-back schedule with no link, so a reader could not check them and a
+reviewer could not tell when one went stale.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+GUIDE = Path(__file__).resolve().parents[1] / "docs" / "guides" / "storj.md"
+
+FILING = "https://www.storj.io/blog/storj-announces-voluntary-financial-restructuring"
+FREEZE = "https://forum.storj.io/t/storagenode-payout-update-august-prepayments-and-july-2026-status/32266"
+HELD = "https://storj.dev/node/faq/held-back-amount"
+
+
+def _line(marker: str) -> str:
+ lines = [line for line in GUIDE.read_text(encoding="utf-8").splitlines() if marker in line]
+ assert len(lines) == 1, f"expected one line with {marker!r} in the Storj guide"
+ return lines[0]
+
+
+@pytest.mark.parametrize(
+ ("marker", "sources"),
+ [
+ ("5:26-bk-00512", [FILING, FREEZE]),
+ ("**Held-back earnings**", [HELD, FREEZE]),
+ ],
+)
+def test_each_claim_links_its_source(marker: str, sources: list[str]):
+ line = _line(marker)
+ assert [s for s in sources if f"]({s}" not in line] == []
From d47c1b425dac53c005d91c887969136561f88b9c Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 14:46:21 +0100
Subject: [PATCH 10/34] fix(proxies): stop the tunnel before the earner, so a
half-failed stop is not undone
proxy_tunnel.stop stopped the earner first. When the tunnel stop then failed,
the next heartbeat found a ready tunnel beside a stopped earner and started the
earner again, losing the stop. The tunnel now stops first; an earner left
running without its tunnel has no way out and the heartbeat leaves it alone.
Same fix as CashPilot-Desktop's stopProxied.
---
app/proxy_tunnel.py | 11 ++++++++---
tests/test_proxy_tunnel.py | 23 +++++++++++++++++++++--
2 files changed, 29 insertions(+), 5 deletions(-)
diff --git a/app/proxy_tunnel.py b/app/proxy_tunnel.py
index 23a6331b..08f5a3d6 100644
--- a/app/proxy_tunnel.py
+++ b/app/proxy_tunnel.py
@@ -435,19 +435,24 @@ def _is_ours(container: Any | None, slug: str, proxy_id: int) -> bool:
def stop(slug: str, proxy_id: int) -> bool:
- """Stop the earner, then its tunnel. Returns whether either was running.
+ """Stop the tunnel, then its earner. Returns whether either was running.
Docker answers a stop of a stopped container with success, so the cap asks
this to tell a copy it stopped from one the owner had already stopped.
+
+ The tunnel goes first so a stop that fails halfway never leaves a ready
+ tunnel beside a stopped earner: the heartbeat starts an earner in that
+ state (earner_action), which would undo the stop. An earner left running
+ after its tunnel stopped has no way out, and earner_action leaves it alone.
"""
earner, sidecar = _pair(slug, proxy_id)
if not _is_ours(earner, slug, proxy_id) and not _is_ours(sidecar, slug, proxy_id):
raise ValueError(f"No proxied copy of {slug} through proxy {proxy_id} on this worker")
was_running = any(_is_ours(c, slug, proxy_id) and getattr(c, "status", "") in _LIVE for c in (earner, sidecar))
- if _is_ours(earner, slug, proxy_id):
- earner.stop(timeout=orchestrator._get_stop_timeout(slug))
if _is_ours(sidecar, slug, proxy_id):
sidecar.stop(timeout=10)
+ if _is_ours(earner, slug, proxy_id):
+ earner.stop(timeout=orchestrator._get_stop_timeout(slug))
return was_running
diff --git a/tests/test_proxy_tunnel.py b/tests/test_proxy_tunnel.py
index 7fea50a9..b0b7647f 100644
--- a/tests/test_proxy_tunnel.py
+++ b/tests/test_proxy_tunnel.py
@@ -700,14 +700,33 @@ def _pair(self, earner_status="running", tunnel_status="running"):
tunnel = _container("cashpilot-tunnel-x-p1", "x", proxy=1, role="tunnel", status=tunnel_status)
return earner, tunnel
- def test_stop_takes_the_earner_before_its_tunnel(self):
+ def test_stop_takes_the_tunnel_before_its_earner(self):
earner, tunnel = self._pair()
order = MagicMock()
order.attach_mock(earner.stop, "earner")
order.attach_mock(tunnel.stop, "tunnel")
with patch.object(orchestrator, "_get_client", return_value=_client_with(earner, tunnel)):
proxy_tunnel.stop("x", 1)
- assert [c[0] for c in order.mock_calls] == ["earner", "tunnel"]
+ assert [c[0] for c in order.mock_calls] == ["tunnel", "earner"]
+
+ def test_a_stop_that_fails_halfway_is_not_undone_by_the_heartbeat(self):
+ """A ready tunnel beside a stopped earner is what the heartbeat starts
+ again, so a failed stop must never leave that state."""
+ for failing in ("tunnel", "earner"):
+ earner, tunnel = self._pair()
+
+ def stopped(c):
+ return lambda **_kw: setattr(c, "status", "exited")
+
+ earner.stop.side_effect = stopped(earner)
+ tunnel.stop.side_effect = stopped(tunnel)
+ (tunnel if failing == "tunnel" else earner).stop.side_effect = RuntimeError("daemon timeout")
+ with (
+ patch.object(orchestrator, "_get_client", return_value=_client_with(earner, tunnel)),
+ pytest.raises(RuntimeError),
+ ):
+ proxy_tunnel.stop("x", 1)
+ assert proxy_tunnel.earner_action(earner, tunnel, ready=True) is None, failing
def test_stop_and_start_refuse_a_copy_that_is_not_ours(self):
stranger = _container("cashpilot-x-p1", "y", proxy=1)
From ce89a8d143cc9933d95a37e8bad15ad60e2502bb Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 14:49:51 +0100
Subject: [PATCH 11/34] fix(catalog): set UpRock's minimum to the 250 UPT its
help centre states
uprock.yml said the minimum was $5 (payment.minimum_payout) and 5 USD
(cashout.min_amount), while its own rating con quoted UpRock's help centre:
withdrawals are UPT to a Solana wallet from 250 UPT. The payout card counted
down to a threshold UpRock does not have. Both fields and the guide now say
250 UPT, with the help centre linked. A catalog test fails when a minimum
quoted in a con disagrees with cashout.min_amount and its currency.
---
docs/guides/index.md | 2 +-
docs/guides/uprock.md | 4 +-
services/depin/uprock.yml | 12 +++--
.../test_catalog_minimum_matches_its_cons.py | 54 +++++++++++++++++++
4 files changed, 65 insertions(+), 7 deletions(-)
create mode 100644 tests/test_catalog_minimum_matches_its_cons.py
diff --git a/docs/guides/index.md b/docs/guides/index.md
index dc0706c0..443e6261 100644
--- a/docs/guides/index.md
+++ b/docs/guides/index.md
@@ -83,7 +83,7 @@ Setup and configuration for every service CashPilot supports.
| [Teneo Protocol](teneo.md) | Residential IP | App only | — | active |
| [Theta Edge Node](theta-edge.md) | — | App only | — | active |
| [Titan Network](titan.md) | Residential IP, Storage 50GB | App only | — | active |
-| [Uprock](uprock.md) | Residential IP | App only | $5 | active |
+| [Uprock](uprock.md) | Residential IP | App only | 250 UPT | active |
| [Wipter](wipter.md) | Residential IP | App only | $20 | dead |
## GPU Compute
diff --git a/docs/guides/uprock.md b/docs/guides/uprock.md
index a4bd071e..4de0a7ce 100644
--- a/docs/guides/uprock.md
+++ b/docs/guides/uprock.md
@@ -14,7 +14,7 @@ Uprock is a DePIN (Decentralized Physical Infrastructure Network) platform that
|--------|-------|
| Monthly range | $0 - $5 (estimate) |
| Per | device |
-| Minimum payout | $5 |
+| Minimum payout | 250 UPT |
| Payout frequency | On request |
| Payment methods | Crypto |
@@ -56,6 +56,8 @@ No Docker image available. Uprock is a browser extension and mobile app only.
## Important Notes
+- **Withdrawals**: rewards are UPT tokens, withdrawn to a Solana wallet once you hold 250 UPT, with two-factor authentication on. A withdrawal fee applies ([UpRock help centre](https://help.uprock.com/en/articles/2057635-security-and-user-asset-protection-2fa-and-withdrawals)).
+
- **Android works well**: The Android app runs reliably in the background with proper battery optimization settings.
- **No IP conflicts**: Can run alongside other bandwidth-sharing services on the same device/network.
- **Extension + App**: You can run both the browser extension and mobile app simultaneously on different devices for additional earnings.
diff --git a/services/depin/uprock.yml b/services/depin/uprock.yml
index 02a1bd5e..a8fea42e 100644
--- a/services/depin/uprock.yml
+++ b/services/depin/uprock.yml
@@ -31,8 +31,10 @@ requirements:
payment:
methods: [crypto]
- minimum_payout: "$5"
- currency: USD
+ # help.uprock.com/en/articles/2057635: "The minimum withdrawal is 250 UPT",
+ # to a Solana wallet (checked 2026-11-07).
+ minimum_payout: "250 UPT"
+ currency: UPT
frequency: "On request"
earnings:
@@ -45,9 +47,9 @@ earnings:
cashout:
method: redirect
dashboard_url: "https://uprock.com"
- min_amount: 5.0
- currency: USD
- notes: "Crypto token-based rewards. Claim from dashboard."
+ min_amount: 250.0
+ currency: UPT
+ notes: "UPT tokens, withdrawn to a Solana wallet from 250 UPT, with two-factor authentication on. A withdrawal fee applies."
platforms: [browser-extension, android]
diff --git a/tests/test_catalog_minimum_matches_its_cons.py b/tests/test_catalog_minimum_matches_its_cons.py
new file mode 100644
index 00000000..970c5b36
--- /dev/null
+++ b/tests/test_catalog_minimum_matches_its_cons.py
@@ -0,0 +1,54 @@
+"""A minimum quoted in a rating con must be the minimum the catalog uses.
+
+UpRock's payout_terms con quoted its help centre ("a 250 UPT minimum") while
+``payment.minimum_payout`` said "$5" and ``cashout.min_amount`` said 5 USD, so
+the payout card counted down to a threshold the provider does not have.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import pytest
+import yaml
+
+ROOT = Path(__file__).resolve().parents[1]
+QUOTED = re.compile(r"(?P\$)?(?P\d[\d,]*(?:\.\d+)?) ?(?P[A-Z]{2,6})? minimum")
+
+
+def quoted_minimums(entry: dict) -> list[tuple[float, str]]:
+ found = []
+ for con in (entry.get("rating") or {}).get("cons") or []:
+ for m in QUOTED.finditer(str(con.get("text") or "")):
+ unit = "USD" if m.group("usd") else (m.group("unit") or "")
+ if unit:
+ found.append((float(m.group("amount").replace(",", "")), unit))
+ return found
+
+
+def mismatches(entry: dict) -> list[tuple[float, str]]:
+ cashout = entry.get("cashout") or {}
+ declared = (cashout.get("min_amount"), str(cashout.get("currency") or "").upper())
+ return [q for q in quoted_minimums(entry) if (q[0], q[1]) != declared]
+
+
+ENTRIES = {p.stem: yaml.safe_load(p.read_text(encoding="utf-8")) or {} for p in sorted(ROOT.glob("services/*/*.yml"))}
+
+
+@pytest.mark.parametrize("slug", [s for s, e in ENTRIES.items() if quoted_minimums(e)])
+def test_the_quoted_minimum_is_the_catalog_minimum(slug: str):
+ assert mismatches(ENTRIES[slug]) == []
+
+
+def test_some_entry_quotes_a_minimum():
+ """Otherwise the test above has nothing to check."""
+ assert "uprock" in [s for s, e in ENTRIES.items() if quoted_minimums(e)]
+
+
+def test_the_check_catches_the_old_uprock_entry():
+ old = {
+ "cashout": {"min_amount": 5.0, "currency": "USD"},
+ "rating": {"cons": [{"text": "withdrawals are UPT tokens to a Solana wallet, with a 250 UPT minimum."}]},
+ }
+ assert mismatches(old) == [(250.0, "UPT")]
From 28c568f414b6178af7a2a867e39cf5e3f5cb6fd4 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 14:54:08 +0100
Subject: [PATCH 12/34] fix(isolation): log why /version could not be read
before taking the engine for Docker
---
app/lan_fence.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/app/lan_fence.py b/app/lan_fence.py
index 6365df12..c5c4e2a5 100644
--- a/app/lan_fence.py
+++ b/app/lan_fence.py
@@ -649,7 +649,8 @@ def is_podman(client: Any) -> bool:
"""
try:
version = client.version() or {}
- except Exception: # noqa: BLE001 - taken for Docker, as above
+ except Exception as exc: # noqa: BLE001 - taken for Docker, as above
+ logger.debug("Could not read the engine's /version, so it is taken for Docker: %s", exc)
return False
names = [str((version.get("Platform") or {}).get("Name") or "")]
names += [str((c or {}).get("Name") or "") for c in version.get("Components") or []]
From 606b22b1d467f7a2094fa1d79d73fb5292df7438 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 14:54:21 +0100
Subject: [PATCH 13/34] docs(isolation): describe the helper's create and spec
functions
---
app/lan_fence.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/app/lan_fence.py b/app/lan_fence.py
index 8d9917b6..b9ca274b 100644
--- a/app/lan_fence.py
+++ b/app/lan_fence.py
@@ -884,6 +884,7 @@ def run_helper(client: Any, action: str, subnet: str, filtered: bool = True) ->
def _create_helper(client: Any, action: str, subnet: str, filtered: bool) -> Any:
+ """Create the helper container, not started, pulling its image once if it is missing."""
from docker.errors import ImageNotFound
image = _helper_image(client)
@@ -895,6 +896,7 @@ def _create_helper(client: Any, action: str, subnet: str, filtered: bool) -> Any
def _helper_spec(action: str, subnet: str, filtered: bool) -> dict[str, Any]:
+ """The helper's container settings: host networking, NET_ADMIN and NET_RAW, every other capability dropped."""
return dict(
name=HELPER_NAME,
entrypoint=["python", "-c", HELPER_SCRIPT],
From be0932248701b20c3ffdac0fddb7d5ed22e32118 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 15:00:54 +0100
Subject: [PATCH 14/34] fix(update): pull the update helper image first, so the
in-app update works behind the socket proxy
containers.run pulls a missing image only when Docker answers "No such
image". Behind the socket proxy the create is refused first with the
proxy's own 404 ("owner policy could not resolve image"), so on any host
that did not already hold docker:27.5.1-cli the Update button failed with
"Docker refused to start the update helper". The worker now checks for
the image and pulls it before the preflight.
---
app/orchestrator.py | 29 +++++++++++++++
tests/test_update_feature.py | 72 ++++++++++++++++++++++++++++++++++++
2 files changed, 101 insertions(+)
diff --git a/app/orchestrator.py b/app/orchestrator.py
index 4c096f99..81d5296f 100644
--- a/app/orchestrator.py
+++ b/app/orchestrator.py
@@ -1738,6 +1738,34 @@ def _read_file_from_container(container: Any, path: str) -> bytes:
raise ValueError(f"{path} is not a file in the archive")
+def _ensure_updater_image(client: docker.DockerClient) -> None:
+ """Pull the helper image if this host does not have it yet.
+
+ ``containers.run`` pulls a missing image by itself only when the daemon
+ answers "No such image". Behind the socket proxy the create is refused
+ before it reaches the daemon, with the proxy's own 404 ("owner policy
+ could not resolve image"), so ``run`` raised instead of pulling, and the
+ first in-app update on a proxied install failed with "Docker refused to
+ start the update helper". Pulling first works the same with or without
+ the proxy.
+ """
+ try:
+ client.images.get(UPDATER_IMAGE)
+ return
+ except NotFound:
+ pass
+ except APIError as exc:
+ logger.warning("Could not check for the update helper image: %s", exc)
+ try:
+ client.images.pull(UPDATER_IMAGE)
+ except APIError as exc:
+ logger.warning("Could not pull the update helper image %s: %s", UPDATER_IMAGE, exc)
+ refusal = proxy_refusal(exc)
+ if refusal:
+ raise SelfUpdateUnavailable(f"Not updating. {refusal}") from exc
+ raise RuntimeError(f"Docker could not pull the update helper image {UPDATER_IMAGE}.") from exc
+
+
def _resolve_compose_config(
client: docker.DockerClient,
working_dir: str,
@@ -2031,6 +2059,7 @@ def spawn_self_update() -> dict[str, Any]:
logger.warning("Self-update lock check failed: %s", exc)
raise RuntimeError("Docker refused to start the update helper.") from exc
+ _ensure_updater_image(client)
cfg = _resolve_compose_config(client, working_dir, environment, services, {**project_mount, **env_mounts})
if proxied():
refused = _proxy_compose_refusals(cfg, services)
diff --git a/tests/test_update_feature.py b/tests/test_update_feature.py
index e8d58a80..a84b73fd 100644
--- a/tests/test_update_feature.py
+++ b/tests/test_update_feature.py
@@ -476,6 +476,78 @@ def test_other_daemon_errors_surface_as_outage_not_500(self):
orchestrator.spawn_self_update()
+class TestUpdaterImage:
+ """The helper image has to be on the host before the preflight starts. Behind
+ the socket proxy, a create naming a missing image gets the proxy's 404
+ ("owner policy could not resolve image"), not Docker's "No such image", so
+ containers.run never pulls it by itself."""
+
+ @staticmethod
+ def _proxy_404():
+ from docker.errors import NotFound
+
+ return NotFound(
+ "404 Client Error: Not Found",
+ response=MagicMock(status_code=404),
+ explanation=f'owner policy could not resolve image "{orchestrator.UPDATER_IMAGE}" referenced by container Image',
+ )
+
+ def test_a_missing_helper_image_is_pulled_before_the_preflight(self):
+ me = _fake_container(labels=_COMPOSE_LABELS)
+ client = _fake_docker(me=me)
+ _wire_runs(client)
+ order = []
+ missing = self._proxy_404()
+ run = client.containers.run.side_effect
+
+ def get(ref):
+ order.append(("get", ref))
+ raise missing
+
+ def pull(ref, *args, **kwargs):
+ order.append(("pull", ref))
+
+ def tracked_run(image, *args, **kwargs):
+ order.append(("run", image))
+ return run(image, *args, **kwargs)
+
+ client.images.get.side_effect = get
+ client.images.pull.side_effect = pull
+ client.containers.run.side_effect = tracked_run
+ with patch.object(orchestrator, "_get_client", return_value=client):
+ orchestrator.spawn_self_update()
+ assert order[:3] == [
+ ("get", orchestrator.UPDATER_IMAGE),
+ ("pull", orchestrator.UPDATER_IMAGE),
+ ("run", orchestrator.UPDATER_IMAGE),
+ ]
+ assert _updater_started(client)
+
+ def test_a_present_helper_image_is_not_pulled_again(self):
+ me = _fake_container(labels=_COMPOSE_LABELS)
+ client = _fake_docker(me=me)
+ _wire_runs(client)
+ with patch.object(orchestrator, "_get_client", return_value=client):
+ orchestrator.spawn_self_update()
+ client.images.get.assert_called_once_with(orchestrator.UPDATER_IMAGE)
+ client.images.pull.assert_not_called()
+
+ def test_a_pull_the_daemon_refuses_is_an_outage_and_starts_nothing(self):
+ from docker.errors import APIError
+
+ me = _fake_container(labels=_COMPOSE_LABELS)
+ client = _fake_docker(me=me)
+ _wire_runs(client)
+ client.images.get.side_effect = self._proxy_404()
+ client.images.pull.side_effect = APIError("pull access denied", response=MagicMock(status_code=500))
+ with (
+ patch.object(orchestrator, "_get_client", return_value=client),
+ pytest.raises(RuntimeError, match="could not pull the update helper image"),
+ ):
+ orchestrator.spawn_self_update()
+ client.containers.run.assert_not_called()
+
+
_UI_LABELS = {
"com.docker.compose.project": "cashpilot",
"com.docker.compose.service": "cashpilot-ui",
From e5a7cf7a23173711e412ddb26e4388ab679b0adb Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 15:04:25 +0100
Subject: [PATCH 15/34] docs: scope the socket-proxy claims to the shipped
compose files and owner labels
Owner isolation limits the worker to resources carrying its owner label,
not only what it created, and images are not scoped. The Unraid templates
and the TrueNAS app still mount the raw socket in the worker; say so where
CLAUDE.md, the FAQ and how-it-works describe the proxy.
---
CLAUDE.md | 8 ++++----
docs/faq.md | 2 +-
docs/how-it-works.md | 2 +-
3 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/CLAUDE.md b/CLAUDE.md
index de201453..d4c0e310 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -41,7 +41,7 @@ python -m zensical build --clean --strict
python -I .github/scripts/site-check.py site
```
-Three long-running containers in `docker-compose.yml`: `cashpilot-ui` (port 8080, web dashboard + earnings collection), `cashpilot-worker` (port 8081, Docker agent) and `cashpilot-socket-proxy` (holds the host's Docker socket). Since v1.71.0 the worker has no Docker socket: it reaches Docker through the proxy's filtered socket (`DOCKER_HOST=unix:///run/cashpilot-docker/docker.sock`), and since v1.73.0 the proxy's owner isolation limits it to the containers, volumes and networks it created. See `docs/security-defaults.md`.
+Three long-running containers in `docker-compose.yml`: `cashpilot-ui` (port 8080, web dashboard + earnings collection), `cashpilot-worker` (port 8081, Docker agent) and `cashpilot-socket-proxy` (holds the host's Docker socket). Since v1.71.0 the worker has no Docker socket: it reaches Docker through the proxy's filtered socket (`DOCKER_HOST=unix:///run/cashpilot-docker/docker.sock`), and since v1.73.0 the proxy's owner isolation limits it to containers, volumes and networks that carry the worker's owner label (`cashpilot.deployed-by=worker`); images are not scoped. See `docs/security-defaults.md`. The Unraid templates and the TrueNAS app still mount the raw socket in the worker (`docs/app-stores.md`).
### Testing traps
@@ -114,7 +114,7 @@ CashPilot is split into two **always-separate** components. The UI never touches
| Component | Description |
|-----------|-------------|
| **CashPilot UI** | The single web dashboard. Collects all earnings centrally, shows global fleet view, manages workers. **Has NO Docker socket.** Can be hosted anywhere. Only ONE UI instance exists. |
-| **CashPilot Worker** | Agent running on each server that has Docker. **Reaches Docker through the socket proxy** (`cashpilot-socket-proxy`), never the raw socket. Manages local containers, reports status to UI via heartbeats. Has a minimal config page. |
+| **CashPilot Worker** | Agent running on each server that has Docker. **Reaches Docker through the socket proxy** (`cashpilot-socket-proxy`) in the shipped compose files; the Unraid and TrueNAS listings still mount the raw socket. Manages local containers, reports status to UI via heartbeats. Has a minimal config page. |
Two separate Docker images:
- **`drumsergio/cashpilot`** — UI image: FastAPI, Jinja2, templates, static assets, collectors, APScheduler. **No Docker SDK.**
@@ -125,7 +125,7 @@ Two separate Docker images:
### Core Principles
1. **Separation of concerns.** UI handles: dashboard, earnings collection, scheduling, user auth. Workers handle: Docker container lifecycle, health reporting.
-2. **Workers need Docker, through the socket proxy.** A worker without Docker access is useless, but it never gets the raw socket: the proxy refuses privileged containers, binds of `/` or the socket, and anything the worker did not create.
+2. **Workers need Docker, through the socket proxy.** A worker without Docker access is useless, but in the shipped compose files it never gets the raw socket: the proxy refuses privileged containers and binds of `/` or the socket, and limits the worker to containers, volumes and networks with its owner label. The Unraid and TrueNAS listings still give it the raw socket, which is root on the host (`docs/app-stores.md`).
3. **Single source of truth.** The UI instance is the only one that collects earnings and stores historical data.
4. **Earnings are never duplicated.** Since only the UI collects, there is no risk of double-counting.
5. **Workers are stateless satellites.** A worker knows which containers to keep running and the UI URL to report to.
@@ -204,7 +204,7 @@ filtered on `service` silently matched nothing in production. Use
## Key Rules
- Never hardcode credentials; all secrets via environment variables
- Docker images: `drumsergio/cashpilot` with semver tags, never `:latest`
-- Worker reaches Docker only through the socket proxy (`DOCKER_HOST=unix:///run/cashpilot-docker/docker.sock`); the worker never mounts `/var/run/docker.sock` (the proxy does, and so does the one-shot `cashpilot-migrate`)
+- In the shipped compose files the worker reaches Docker only through the socket proxy (`DOCKER_HOST=unix:///run/cashpilot-docker/docker.sock`) and never mounts `/var/run/docker.sock` (the proxy does, and so does the one-shot `cashpilot-migrate`)
- Always use bind mounts (not named volumes) on Unraid
- License: GPL-3.0
diff --git a/docs/faq.md b/docs/faq.md
index 636e8f03..d7f5574e 100644
--- a/docs/faq.md
+++ b/docs/faq.md
@@ -38,7 +38,7 @@ If CashPilot cannot write the key to disk at all, because `/data` is unwritable
**What about security?**
-Every service CashPilot deploys runs in its own container with every Linux capability dropped (a few services add back only the ones they declare) and `--security-opt no-new-privileges` set, so a process inside cannot gain privileges. Containers cannot see your host filesystem beyond the volumes a service declares. They CAN reach your local network by default, because Docker does not block that; [Protecting Your Home Network](home-network-security.md) shows how to put them on a firewalled bridge that reaches the internet but not your LAN, router or host. That bridge does not cover a service on host networking (Mysterium, today); the guide says what to do about those. Service credentials are encrypted at rest using Fernet symmetric encryption. The UI has no Docker access, and the worker is never given the raw Docker socket: it reaches Docker through `cashpilot-socket-proxy`, which refuses privileged containers and anything the worker did not create ([security defaults](security-defaults.md)).
+Every service CashPilot deploys runs in its own container with every Linux capability dropped (a few services add back only the ones they declare) and `--security-opt no-new-privileges` set, so a process inside cannot gain privileges. Containers cannot see your host filesystem beyond the volumes a service declares. They CAN reach your local network by default, because Docker does not block that; [Protecting Your Home Network](home-network-security.md) shows how to put them on a firewalled bridge that reaches the internet but not your LAN, router or host. That bridge does not cover a service on host networking (Mysterium, today); the guide says what to do about those. Service credentials are encrypted at rest using Fernet symmetric encryption. The UI has no Docker access. With the shipped compose files the worker is never given the raw Docker socket: it reaches Docker through `cashpilot-socket-proxy`, which refuses privileged containers and binds of `/` or the socket, and limits the worker to containers, volumes and networks with its owner label ([security defaults](security-defaults.md)). The Unraid templates and the TrueNAS app still mount the raw socket in the worker, which is root on the host ([app stores](app-stores.md)).
That said, no setup is bulletproof. You are still running third-party software that routes external traffic through your network. Docker isolation significantly reduces the attack surface compared to running these services directly on your host, but it does not eliminate all risk. We recommend running CashPilot on a dedicated machine or VLAN, keeping Docker and your host OS up to date, and reviewing the open-source code of any service before deploying it.
diff --git a/docs/how-it-works.md b/docs/how-it-works.md
index 8d087e78..44a56fbd 100644
--- a/docs/how-it-works.md
+++ b/docs/how-it-works.md
@@ -14,7 +14,7 @@ There is **no standalone mode**. Every server that runs Docker containers needs
## Design Principles
1. **Separation of concerns.** UI handles: dashboard, earnings collection, scheduling, user auth. Workers handle: Docker container lifecycle, health reporting. They never overlap.
-2. **Workers need Docker, through the socket proxy.** A worker without Docker access is useless, but it never gets the raw socket: it reaches Docker through `cashpilot-socket-proxy`. If you don't need Docker management, just run the UI alone to track earnings.
+2. **Workers need Docker, through the socket proxy.** A worker without Docker access is useless. With the shipped compose files it never gets the raw socket: it reaches Docker through `cashpilot-socket-proxy`. The Unraid and TrueNAS listings still mount the raw socket ([app stores](app-stores.md)). If you don't need Docker management, just run the UI alone to track earnings.
3. **Single source of truth.** The UI instance is the only one that collects earnings, stores historical data, and serves the dashboard. Workers never collect earnings.
4. **Earnings are never duplicated.** Since only the UI collects, there is no risk of the same account being counted twice.
5. **Workers are stateless satellites.** A worker knows which containers to keep running and the UI URL to report to. It persists only its own id and fleet key to local files (`.worker_id`, `.worker_key`; no database, no earnings data).
From a69b29dd3912ffc2c378b9df13e1b6b0f698594c Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 15:04:57 +0100
Subject: [PATCH 16/34] test(storj): check each source link in full, up to its
closing parenthesis
---
tests/test_storj_guide_sources.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/tests/test_storj_guide_sources.py b/tests/test_storj_guide_sources.py
index 12d9b6ce..a3d6136f 100644
--- a/tests/test_storj_guide_sources.py
+++ b/tests/test_storj_guide_sources.py
@@ -13,7 +13,10 @@
GUIDE = Path(__file__).resolve().parents[1] / "docs" / "guides" / "storj.md"
-FILING = "https://www.storj.io/blog/storj-announces-voluntary-financial-restructuring"
+FILING = (
+ "https://www.storj.io/blog/"
+ "storj-announces-voluntary-financial-restructuring-to-resolve-legacy-liabilities-and-position-the-business-for-growth/"
+)
FREEZE = "https://forum.storj.io/t/storagenode-payout-update-august-prepayments-and-july-2026-status/32266"
HELD = "https://storj.dev/node/faq/held-back-amount"
@@ -33,4 +36,4 @@ def _line(marker: str) -> str:
)
def test_each_claim_links_its_source(marker: str, sources: list[str]):
line = _line(marker)
- assert [s for s in sources if f"]({s}" not in line] == []
+ assert [s for s in sources if f"]({s})" not in line] == []
From 83dbe64aaf4a09bc43f5eae7667070c521bb1263 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 15:05:59 +0100
Subject: [PATCH 17/34] docs(uprock): say Android earnings count toward 250 UPT
only after device verification; check payment.minimum_payout too
---
docs/guides/uprock.md | 2 +-
.../test_catalog_minimum_matches_its_cons.py | 26 +++++++++++++++++--
2 files changed, 25 insertions(+), 3 deletions(-)
diff --git a/docs/guides/uprock.md b/docs/guides/uprock.md
index 4de0a7ce..e7eacdd8 100644
--- a/docs/guides/uprock.md
+++ b/docs/guides/uprock.md
@@ -56,7 +56,7 @@ No Docker image available. Uprock is a browser extension and mobile app only.
## Important Notes
-- **Withdrawals**: rewards are UPT tokens, withdrawn to a Solana wallet once you hold 250 UPT, with two-factor authentication on. A withdrawal fee applies ([UpRock help centre](https://help.uprock.com/en/articles/2057635-security-and-user-asset-protection-2fa-and-withdrawals)).
+- **Withdrawals**: rewards are UPT tokens, withdrawn to a Solana wallet once 250 UPT are available to withdraw, with two-factor authentication on. A withdrawal fee applies ([UpRock help centre](https://help.uprock.com/en/articles/2057635-security-and-user-asset-protection-2fa-and-withdrawals)). Earnings from an Android device stay locked until UpRock's device verification completes, so they do not count toward the 250 UPT until then ([device verification](https://help.uprock.com/en/help/articles/8124029-what-is-device-verification)).
- **Android works well**: The Android app runs reliably in the background with proper battery optimization settings.
- **No IP conflicts**: Can run alongside other bandwidth-sharing services on the same device/network.
diff --git a/tests/test_catalog_minimum_matches_its_cons.py b/tests/test_catalog_minimum_matches_its_cons.py
index 970c5b36..3e4fa0eb 100644
--- a/tests/test_catalog_minimum_matches_its_cons.py
+++ b/tests/test_catalog_minimum_matches_its_cons.py
@@ -27,10 +27,21 @@ def quoted_minimums(entry: dict) -> list[tuple[float, str]]:
return found
+def _stated(text: object) -> tuple[float, str] | None:
+ """payment.minimum_payout as (amount, unit): "$5" is (5.0, "USD"), "250 UPT" is (250.0, "UPT")."""
+ m = re.fullmatch(r"(?P\$)?(?P\d[\d,]*(?:\.\d+)?) ?(?P[A-Z]{2,6})?", str(text or "").strip())
+ if not m:
+ return None
+ unit = "USD" if m.group("usd") else (m.group("unit") or "")
+ return float(m.group("amount").replace(",", "")), unit
+
+
def mismatches(entry: dict) -> list[tuple[float, str]]:
+ """Quoted minimums that disagree with cashout.min_amount or payment.minimum_payout."""
cashout = entry.get("cashout") or {}
- declared = (cashout.get("min_amount"), str(cashout.get("currency") or "").upper())
- return [q for q in quoted_minimums(entry) if (q[0], q[1]) != declared]
+ derived = (cashout.get("min_amount"), str(cashout.get("currency") or "").upper())
+ stated = _stated((entry.get("payment") or {}).get("minimum_payout"))
+ return [q for q in quoted_minimums(entry) if q != derived or q != stated]
ENTRIES = {p.stem: yaml.safe_load(p.read_text(encoding="utf-8")) or {} for p in sorted(ROOT.glob("services/*/*.yml"))}
@@ -48,7 +59,18 @@ def test_some_entry_quotes_a_minimum():
def test_the_check_catches_the_old_uprock_entry():
old = {
+ "payment": {"minimum_payout": "$5"},
"cashout": {"min_amount": 5.0, "currency": "USD"},
"rating": {"cons": [{"text": "withdrawals are UPT tokens to a Solana wallet, with a 250 UPT minimum."}]},
}
assert mismatches(old) == [(250.0, "UPT")]
+
+
+def test_the_check_catches_a_stale_payment_minimum():
+ """cashout.min_amount is derived from payment.minimum_payout; a stale source is caught too."""
+ entry = {
+ "payment": {"minimum_payout": "$5"},
+ "cashout": {"min_amount": 250.0, "currency": "UPT"},
+ "rating": {"cons": [{"text": "with a 250 UPT minimum."}]},
+ }
+ assert mismatches(entry) == [(250.0, "UPT")]
From e6a0189f09588a6b14c75bec111d8334d5dcc695 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 15:06:55 +0100
Subject: [PATCH 18/34] test(proxies): also catch a plain 'SOCKS5 carries UDP'
claim
---
tests/test_docs_proxy_udp.py | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/tests/test_docs_proxy_udp.py b/tests/test_docs_proxy_udp.py
index d7e14dd9..81701ce6 100644
--- a/tests/test_docs_proxy_udp.py
+++ b/tests/test_docs_proxy_udp.py
@@ -15,7 +15,10 @@
TUNNEL = ROOT / "tunnel" / "cp-tunnel"
PAGE = ROOT / "docs" / "proxies.md"
-UDP_PROMISE = re.compile(r"(carries|works?|through) [^.|]*udp associate", re.IGNORECASE)
+UDP_PROMISE = re.compile(
+ r"(carries|works?|through) [^.|]*udp associate|(socks5|http connect)[^.|]*\bcarr(y|ies) udp",
+ re.IGNORECASE,
+)
def _socks_egress_rule() -> str:
@@ -42,3 +45,9 @@ def test_the_page_does_not_promise_udp_through_socks5():
def test_the_check_catches_the_old_claims():
assert promises_udp("| SOCKS5 | host | Carries UDP only if the provider supports SOCKS5 UDP ASSOCIATE. |")
assert promises_udp("and through SOCKS5\nonly when the provider supports UDP ASSOCIATE.")
+ assert promises_udp("SOCKS5 carries UDP.")
+ assert promises_udp("An HTTP CONNECT proxy carries UDP too.")
+
+
+def test_the_check_accepts_the_true_claims():
+ assert not promises_udp("SOCKS5 or HTTP CONNECT: the tunnel lets out only TCP. A Tailscale exit node carries UDP.")
From b23857aca2ddbb760c158f79be5d2d4176335efc Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 15:10:36 +0100
Subject: [PATCH 19/34] fix(proxies): let the owner's decision and a returning
cap win over a pending retry
A retry now rechecks the copy's cap mark before it starts (an owner start or
stop clears it), keeps the mark when the cap came back during the start, and
shares the in-flight set with the reset's starts so the two never double up.
---
app/main.py | 25 ++++++++++++++++++++++---
tests/test_proxies.py | 42 ++++++++++++++++++++++++++++++++++++++++++
2 files changed, 64 insertions(+), 3 deletions(-)
diff --git a/app/main.py b/app/main.py
index fa18a011..89ceb75f 100644
--- a/app/main.py
+++ b/app/main.py
@@ -4328,8 +4328,10 @@ async def _release_capped_copies(proxy: dict[str, Any]) -> list[str]:
_drop_proxy_cap_bell(proxy["id"])
started: list[str] = []
for copy in await database.list_proxied_copies(proxy["id"]):
- if not copy.get("stopped_by_cap"):
+ key = (proxy["id"], copy["worker_id"], copy["slug"])
+ if not copy.get("stopped_by_cap") or key in _proxy_cap_starting:
continue
+ _proxy_cap_starting.add(key)
try:
await _start_proxied_copy(proxy, copy["worker_id"], copy["slug"])
await database.set_proxied_copy_capped(proxy["id"], copy["worker_id"], copy["slug"], False)
@@ -4338,6 +4340,8 @@ async def _release_capped_copies(proxy: dict[str, Any]) -> list[str]:
logger.warning(
"Could not restart %s through proxy %s after a reset: %s", copy["slug"], proxy["id"], exc.detail
)
+ finally:
+ _proxy_cap_starting.discard(key)
return started
@@ -4366,7 +4370,8 @@ async def _run_proxy_cap_resets() -> None:
_proxy_cap_unstopped: dict[int, dict[tuple[int, str], str]] = {}
# Stops already on their way, so a slow worker is not sent a second one.
_proxy_cap_stopping: set[tuple[int, int, str]] = set()
-# Starts of copies the cap stopped, retried after a reset, on their way.
+# Starts of copies the cap stopped (by a reset or a heartbeat retry) on their way,
+# so the two paths never send a second start for the same copy.
_proxy_cap_starting: set[tuple[int, int, str]] = set()
@@ -4438,13 +4443,27 @@ async def _restart_capped_copies(proxy_id: int, worker_id: int, slugs: set[str])
for slug in sorted(slugs):
async with _proxy_state_lock(proxy_id):
fresh = await database.get_proxy(proxy_id)
+ # The owner may have started or stopped it since the heartbeat; that
+ # clears the mark, and the owner's decision stands.
+ marked = any(
+ c["worker_id"] == worker_id and c["slug"] == slug and c.get("stopped_by_cap")
+ for c in await database.list_proxied_copies(proxy_id)
+ )
if not fresh or fresh["state"].get("capped_at"):
return
+ if not marked:
+ continue
try:
await _start_proxied_copy(fresh, worker_id, slug)
- await database.set_proxied_copy_capped(proxy_id, worker_id, slug, False)
except HTTPException as exc:
logger.warning("Could not restart %s through proxy %s after a reset: %s", slug, proxy_id, exc.detail)
+ continue
+ # The cap may have come back while the start ran. Then the mark stays, so
+ # the next heartbeat stops the copy again and the next reset starts it.
+ async with _proxy_state_lock(proxy_id):
+ again = await database.get_proxy(proxy_id)
+ if again and not again["state"].get("capped_at"):
+ await database.set_proxied_copy_capped(proxy_id, worker_id, slug, False)
finally:
_proxy_cap_starting.difference_update((proxy_id, worker_id, slug) for slug in slugs)
diff --git a/tests/test_proxies.py b/tests/test_proxies.py
index e8808565..592aef02 100644
--- a/tests/test_proxies.py
+++ b/tests/test_proxies.py
@@ -440,6 +440,48 @@ async def go():
]
assert [c.args for c in rig.mark.call_args_list] == [(1, 7, "a", False)]
+ def test_a_retry_respects_the_owner_and_a_cap_that_came_back(self):
+ uncapped = {"id": 1, "definition": dict(CAPPED), "state": {}}
+ capped = {**uncapped, "state": {"capped_at": "x", "cap_reason": "r"}}
+
+ def run(copies, proxies_seen):
+ call = AsyncMock(return_value={})
+ mark = AsyncMock()
+ with (
+ patch.object(main.database, "get_proxy", new_callable=AsyncMock, side_effect=proxies_seen),
+ patch.object(main.database, "list_proxied_copies", new_callable=AsyncMock, return_value=copies),
+ patch.object(main.database, "set_proxied_copy_capped", mark),
+ patch.object(main, "_proxy_worker_call", call),
+ ):
+ asyncio.run(main._restart_capped_copies(1, 7, {"a"}))
+ return call, mark
+
+ # The owner started or stopped it since the heartbeat: the mark is gone, no start.
+ call, mark = run([{"proxy_id": 1, "worker_id": 7, "slug": "a", "stopped_by_cap": 0}], [uncapped])
+ assert call.call_count == 0 and mark.call_count == 0
+ # The cap came back while the start ran: the mark stays for the next stop and reset.
+ call, mark = run([{"proxy_id": 1, "worker_id": 7, "slug": "a", "stopped_by_cap": 1}], [uncapped, capped])
+ assert call.call_count == 1 and mark.call_count == 0
+ assert not main._proxy_cap_starting
+
+ def test_a_reset_skips_a_copy_whose_start_is_already_on_its_way(self):
+ proxy = {"id": 1, "definition": dict(CAPPED), "state": {}}
+ call = AsyncMock(return_value={})
+ main._proxy_cap_starting.add((1, 7, "a"))
+ with (
+ patch.object(main.database, "clear_alerts", new_callable=AsyncMock),
+ patch.object(
+ main.database,
+ "list_proxied_copies",
+ new_callable=AsyncMock,
+ return_value=[{"proxy_id": 1, "worker_id": 7, "slug": "a", "stopped_by_cap": 1}],
+ ),
+ patch.object(main.database, "set_proxied_copy_capped", new_callable=AsyncMock),
+ patch.object(main, "_proxy_worker_call", call),
+ ):
+ assert asyncio.run(main._release_capped_copies(proxy)) == []
+ assert call.call_count == 0
+
def test_a_copy_the_cap_stopped_stays_down_while_the_cap_holds(self):
call = AsyncMock(return_value={})
rig = _CapRig([{"proxy_id": 1, "worker_id": 7, "slug": "a", "stopped_by_cap": 1}], call)
From 27b24563995181d1c5a919879afb0987a0283481 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 15:11:31 +0100
Subject: [PATCH 20/34] fix(proxies): sweep only stale staged sidecars, and run
one deploy per copy at a time
---
app/proxy_tunnel.py | 35 ++++++++++++++++++++++++++++++++---
tests/test_proxy_tunnel.py | 33 +++++++++++++++++++++++++++++++++
2 files changed, 65 insertions(+), 3 deletions(-)
diff --git a/app/proxy_tunnel.py b/app/proxy_tunnel.py
index 54930c71..60eadea5 100644
--- a/app/proxy_tunnel.py
+++ b/app/proxy_tunnel.py
@@ -34,6 +34,7 @@
import threading
import time
import uuid
+from datetime import UTC, datetime
from typing import Any
from urllib.parse import quote
@@ -69,6 +70,28 @@
# deploy() swapping the old pair for the staged one, and status(), take turns: a
# heartbeat must not start an earner that is being removed, or read a half swap.
_swap_lock = threading.Lock()
+# One deploy per copy at a time: a second one would take the first one's staged
+# sidecar for a leftover and remove it.
+_deploy_locks: dict[tuple[str, int], threading.Lock] = {}
+_deploy_locks_guard = threading.Lock()
+#: A staged sidecar older than this is a leftover: a deploy is past its ready
+#: timeout by then. Younger ones may belong to a deploy still running.
+STAGED_STALE_SECONDS = 300
+
+
+def _deploy_lock(slug: str, proxy_id: int) -> threading.Lock:
+ with _deploy_locks_guard:
+ return _deploy_locks.setdefault((slug, proxy_id), threading.Lock())
+
+
+def _age_seconds(container: Any) -> float:
+ """Seconds since the container was created; 0 when Docker does not say."""
+ raw = str((getattr(container, "attrs", None) or {}).get("Created") or "")
+ try:
+ created = datetime.fromisoformat(re.sub(r"(\.\d{6})\d+", r"\1", raw).replace("Z", "+00:00"))
+ except ValueError:
+ return 0.0
+ return (datetime.now(UTC) - created).total_seconds()
class ProxyTunnelError(RuntimeError):
@@ -410,6 +433,11 @@ def deploy(slug: str, proxy_id: int, earner: dict[str, Any], proxy: dict[str, An
sidecar takes the final name, and only then is the earner created inside
its namespace. A sidecar that cannot start therefore costs nothing.
"""
+ with _deploy_lock(slug, proxy_id):
+ return _deploy(slug, proxy_id, earner, proxy, own_ip)
+
+
+def _deploy(slug: str, proxy_id: int, earner: dict[str, Any], proxy: dict[str, Any], own_ip: str | None) -> dict:
client = orchestrator._get_client()
# Checked first: settings that cannot work (a host that does not resolve,
# no tunnel image) fail fast, before a sidecar is created.
@@ -578,13 +606,14 @@ def sweep_test_tunnels() -> list[str]:
continue
_remove_quietly(container, 2)
swept.append(container.name)
- # And staged sidecars of a deploy the worker died in: they hold the proxy
- # login, and the heartbeat never reports them.
+ # And staged sidecars of a deploy that died: they hold the proxy login, and
+ # the heartbeat never reports them. Only stale ones, so a deploy another
+ # worker on this daemon is running keeps its sidecar.
staged = client.containers.list(
all=True, filters={"label": [f"{LABEL_MANAGED}=true", f"{LABEL_ROLE}={ROLE_TUNNEL}", LABEL_PROXY]}
)
for container in staged:
- if str(container.name).endswith(STAGED_SUFFIX):
+ if str(container.name).endswith(STAGED_SUFFIX) and _age_seconds(container) >= STAGED_STALE_SECONDS:
_remove_quietly(container, 2)
swept.append(container.name)
return swept
diff --git a/tests/test_proxy_tunnel.py b/tests/test_proxy_tunnel.py
index 6fa558b7..e76907dd 100644
--- a/tests/test_proxy_tunnel.py
+++ b/tests/test_proxy_tunnel.py
@@ -964,6 +964,39 @@ def test_a_staged_sidecar_of_a_deploy_that_died_is_swept(self):
assert proxy_tunnel.sweep_test_tunnels() == ["cashpilot-tunnel-x-p1-next"]
live.remove.assert_not_called()
+ def test_a_young_staged_sidecar_is_left_to_its_deploy(self):
+ from datetime import UTC, datetime
+
+ young = _container("cashpilot-tunnel-x-p1-next", "x", proxy=1, role="tunnel")
+ young.attrs["Created"] = datetime.now(UTC).isoformat().replace("+00:00", "123Z")
+ client = MagicMock()
+ client.containers.list.side_effect = [[], [young]]
+ with patch.object(orchestrator, "_get_client", return_value=client):
+ assert proxy_tunnel.sweep_test_tunnels() == []
+ young.remove.assert_not_called()
+
+ def test_two_deploys_of_one_copy_take_turns(self):
+ import threading
+
+ inside, overlap = [], []
+ gate = threading.Event()
+
+ def slow(*_a, **_kw):
+ if inside:
+ overlap.append(True)
+ inside.append(True)
+ gate.wait(0.2)
+ inside.pop()
+ return {}
+
+ with patch.object(proxy_tunnel, "_deploy", side_effect=slow):
+ threads = [threading.Thread(target=proxy_tunnel.deploy, args=("x", 1, {}, {}, None)) for _ in range(2)]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join()
+ assert not overlap
+
def test_a_probe_answer_that_is_not_an_ip_is_not_ready(self):
sidecar = MagicMock(status="running")
sidecar.exec_run.return_value = (0, b'{"ip":"e","country":""}\n')
From a63ef8435f4de9fc503e01c1eb2a5aa43ecb1511 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 15:17:31 +0100
Subject: [PATCH 21/34] fix(migrate): keep confirmed identity backups confirmed
through the owner migration
The backup fingerprint includes a volume's creation time, so a volume made
again reads as new state. The owner migration makes every earner volume
again with the same data, and every confirmed Mysterium or ProxyBase wallet
backup turned to "changed" on the Payouts page. The migration now keeps the
replaced volume's creation time in a label on the new one, and the
fingerprint reads it.
---
UPGRADING.md | 4 +++-
app/constants.py | 7 +++++++
app/orchestrator.py | 8 ++++++--
app/owner_migrate.py | 9 ++++++++-
tests/test_backup_confirmations.py | 17 +++++++++++++++++
tests/test_owner_migrate.py | 24 ++++++++++++++++++++++++
6 files changed, 65 insertions(+), 4 deletions(-)
diff --git a/UPGRADING.md b/UPGRADING.md
index 8775df88..6c7123ac 100644
--- a/UPGRADING.md
+++ b/UPGRADING.md
@@ -54,7 +54,9 @@ ones:
- every volume of an earner the worker started is copied to a backup named
`-pre-isolation`, checked, recreated with the label, filled back from
the backup and checked again, and the earner is recreated as it was. A
- Mysterium identity keeps its address. Each backup carries the label
+ Mysterium identity keeps its address, and a backup you confirmed on the
+ Payouts page stays confirmed (if you migrated with 1.73.0 or 1.73.1, the
+ page shows those boxes as changed: confirm them again). Each backup carries the label
`cashpilot.backup-of=`, and the log line
*Backup … holds the data of …* says which earner it belonged to;
- the worker's own volumes are copied into new ones with `_owned` names
diff --git a/app/constants.py b/app/constants.py
index 04239962..091872f1 100644
--- a/app/constants.py
+++ b/app/constants.py
@@ -40,6 +40,13 @@
UNDEPLOYABLE_STATUSES = frozenset({"broken", "dead", "dropped"})
+#: On a volume the owner migration made again: the creation time of the volume
+#: it replaced. The backup fingerprint (orchestrator.state_id) reads it instead of
+#: the new volume's own, so moving the same data onto a labelled volume does not
+#: void a confirmed backup of it.
+LABEL_STATE_CREATED = "cashpilot.state-created"
+
+
#: deployed-by values only CashPilot itself writes: "worker" by deploy_raw, and
#: "direct" by the deploy path before workers (removed in #85), whose containers
#: still run on old installs. "compose" is an exported file the user runs by hand.
diff --git a/app/orchestrator.py b/app/orchestrator.py
index 4c096f99..1264c58e 100644
--- a/app/orchestrator.py
+++ b/app/orchestrator.py
@@ -52,6 +52,7 @@
LABEL_PROXY,
LABEL_ROLE,
LABEL_SERVICE,
+ LABEL_STATE_CREATED,
LABEL_VERSION,
drift_warnings_enabled,
)
@@ -1216,7 +1217,7 @@ def state_id(container: Any, targets: dict[str, str] | list[str] | None, client:
"""A short fingerprint of WHERE a service's irreplaceable state lives.
Built from each critical mount's type and source, plus a named volume's
- creation time. It changes when the state can have changed under the user:
+ creation time (the replaced volume's, for one the owner migration moved). It changes when the state can have changed under the user:
the volume was removed and made again (same name, new creation time), the
service now points at another volume or host path, or the state sits in the
container's own layer and the container was replaced. It does NOT change
@@ -1248,7 +1249,10 @@ def state_id(container: Any, targets: dict[str, str] | list[str] | None, client:
name = str(mount.get("Name") or "")
created = ""
try:
- created = str(((client or _get_client()).volumes.get(name).attrs or {}).get("CreatedAt") or "")
+ attrs = (client or _get_client()).volumes.get(name).attrs or {}
+ # The owner migration copies a volume's data onto a new one of the
+ # same name and keeps the old creation time in this label.
+ created = str((attrs.get("Labels") or {}).get(LABEL_STATE_CREATED) or attrs.get("CreatedAt") or "")
except Exception as exc: # noqa: BLE001 - unknown, not a change
logger.debug("Could not read volume %s for the state fingerprint: %s", name, exc)
return None
diff --git a/app/owner_migrate.py b/app/owner_migrate.py
index c8a6c6c9..cbdc1ddb 100644
--- a/app/owner_migrate.py
+++ b/app/owner_migrate.py
@@ -34,6 +34,7 @@
from pathlib import Path
from typing import Any
+from app.constants import LABEL_STATE_CREATED
from app.socket_policy import OWNER, OWNER_LABEL
logger = logging.getLogger("cashpilot.migrate")
@@ -690,6 +691,8 @@ def back_up(self, vol: str, used_by: list[str]) -> None:
"backup": backup,
"used_by": used_by,
"labels": attrs.get("Labels") or {},
+ # Kept on the new volume, so a confirmed backup of this data stays confirmed.
+ "created": attrs.get("CreatedAt") or "",
"digest": result,
"backed_up": True,
"restored": False,
@@ -722,7 +725,11 @@ def restore(self, vol: str) -> None:
current.remove()
current = None
if current is None:
- self.client.volumes.create(vol, driver="local", labels={**record["labels"], OWNER_LABEL: OWNER})
+ labels = {**record["labels"], OWNER_LABEL: OWNER}
+ if record.get("created"):
+ # A label the old volume already had is the older, truer time.
+ labels.setdefault(LABEL_STATE_CREATED, record["created"])
+ self.client.volumes.create(vol, driver="local", labels=labels)
result = self.copy(record["backup"], vol)
if result["digest"] != record["digest"]["digest"]:
raise MigrationError(
diff --git a/tests/test_backup_confirmations.py b/tests/test_backup_confirmations.py
index ca6aeb40..861e2e59 100644
--- a/tests/test_backup_confirmations.py
+++ b/tests/test_backup_confirmations.py
@@ -31,6 +31,7 @@
from app import main, orchestrator, payout_registry
from app import state_backup as sb
+from app.constants import LABEL_STATE_CREATED
ROOT = Path(__file__).resolve().parent.parent
NOW = datetime(2026, 10, 31, 12, 0, tzinfo=UTC)
@@ -207,6 +208,22 @@ def test_a_volume_removed_and_made_again_changes_it(self):
b = orchestrator.state_id(_container(VOLUME), TARGETS, _client({"mysterium-data": "2026-10-30T13:00:36Z"}))
assert a != b
+ def test_a_volume_the_owner_migration_made_again_keeps_it(self):
+ """The migration copies the data onto a new volume of the same name and
+ keeps the old creation time in a label; the fingerprint reads that."""
+ before = orchestrator.state_id(_container(VOLUME), TARGETS, _client({"mysterium-data": "2026-03-26T11:41:03Z"}))
+ migrated = SimpleNamespace(
+ volumes=SimpleNamespace(
+ get=lambda name: SimpleNamespace(
+ attrs={
+ "CreatedAt": "2026-11-07T13:44:20Z",
+ "Labels": {LABEL_STATE_CREATED: "2026-03-26T11:41:03Z", "cashpilot.deployed-by": "worker"},
+ }
+ )
+ )
+ )
+ assert orchestrator.state_id(_container(VOLUME), TARGETS, migrated) == before
+
def test_a_bind_mount_moved_to_another_folder_changes_it(self):
mount = {"Type": "bind", "Source": "/mnt/a", "Destination": "/app/identity"}
a = orchestrator.state_id(_container([mount]), {"/app/identity": "x"})
diff --git a/tests/test_owner_migrate.py b/tests/test_owner_migrate.py
index 8f72894b..3f31d255 100644
--- a/tests/test_owner_migrate.py
+++ b/tests/test_owner_migrate.py
@@ -429,6 +429,30 @@ def test_the_identity_moves_onto_a_labelled_volume_of_the_same_name(install, tmp
assert d.vols["hand-data"].attrs["Labels"] is None
+def test_a_migrated_volume_keeps_the_backup_fingerprint_it_had(install, tmp_path):
+ """A confirmed backup of an identity is tied to its volume's creation time.
+ The migration makes the volume again with the same data, so it carries the
+ old time over, and the confirmation stays valid."""
+ from types import SimpleNamespace
+
+ from app import orchestrator
+ from app.constants import LABEL_STATE_CREATED
+
+ d, myst, _hand = install
+ d.vols["mysterium-data"].attrs["CreatedAt"] = "2026-03-26T11:41:03Z"
+ targets = {"/var/lib/mysterium-node": "identity"}
+ box = SimpleNamespace(
+ id="c",
+ attrs={"Mounts": [{"Type": "volume", "Name": "mysterium-data", "Destination": "/var/lib/mysterium-node"}]},
+ )
+ before = orchestrator.state_id(box, targets, d)
+ assert FakeMigrator(d, om.State(tmp_path)).run([]) == 0
+ # Docker gives the volume it made again a creation time of its own.
+ d.vols["mysterium-data"].attrs["CreatedAt"] = "2026-11-07T13:44:20Z"
+ assert d.vols["mysterium-data"].attrs["Labels"] == {**OWNER, LABEL_STATE_CREATED: "2026-03-26T11:41:03Z"}
+ assert orchestrator.state_id(box, targets, d) == before
+
+
def test_a_second_run_finds_nothing_to_do(install, tmp_path):
d, _, _ = install
assert FakeMigrator(d, om.State(tmp_path)).run([]) == 0
From 8370895d155001fa78e76f70c5221e8e55ecfc25 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 15:20:20 +0100
Subject: [PATCH 22/34] test(update): cover a failed image check and a pull the
proxy refuses
---
tests/test_update_feature.py | 32 ++++++++++++++++++++++++++++++++
1 file changed, 32 insertions(+)
diff --git a/tests/test_update_feature.py b/tests/test_update_feature.py
index a84b73fd..b09a35e8 100644
--- a/tests/test_update_feature.py
+++ b/tests/test_update_feature.py
@@ -547,6 +547,38 @@ def test_a_pull_the_daemon_refuses_is_an_outage_and_starts_nothing(self):
orchestrator.spawn_self_update()
client.containers.run.assert_not_called()
+ def test_a_failed_image_check_still_pulls(self):
+ from docker.errors import APIError
+
+ me = _fake_container(labels=_COMPOSE_LABELS)
+ client = _fake_docker(me=me)
+ _wire_runs(client)
+ client.images.get.side_effect = APIError("daemon busy", response=MagicMock(status_code=500))
+ with patch.object(orchestrator, "_get_client", return_value=client):
+ orchestrator.spawn_self_update()
+ client.images.pull.assert_called_once_with(orchestrator.UPDATER_IMAGE)
+ assert _updater_started(client)
+
+ def test_a_pull_the_proxy_refuses_says_why_and_starts_nothing(self):
+ from docker.errors import APIError
+
+ me = _fake_container(labels=_COMPOSE_LABELS)
+ client = _fake_docker(me=me)
+ _wire_runs(client)
+ client.images.get.side_effect = self._proxy_404()
+ response = MagicMock(status_code=403)
+ response.json.return_value = {
+ "message": "request denied by sockguard policy",
+ "reason": "image pull denied: registry docker.io is not allowlisted",
+ }
+ client.images.pull.side_effect = APIError("denied", response=response)
+ with (
+ patch.object(orchestrator, "_get_client", return_value=client),
+ pytest.raises(orchestrator.SelfUpdateUnavailable, match="registry docker.io is not allowlisted"),
+ ):
+ orchestrator.spawn_self_update()
+ client.containers.run.assert_not_called()
+
_UI_LABELS = {
"com.docker.compose.project": "cashpilot",
From 66f1f3623f251cd01ec4c64c01552109b8071464 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 15:27:40 +0100
Subject: [PATCH 23/34] fix(update): report a stalled helper-image pull as an
outage, and a refused image check as the proxy's reason
---
app/orchestrator.py | 8 ++++++++
tests/test_update_feature.py | 15 +++++++++++++++
2 files changed, 23 insertions(+)
diff --git a/app/orchestrator.py b/app/orchestrator.py
index 81d5296f..935e02e9 100644
--- a/app/orchestrator.py
+++ b/app/orchestrator.py
@@ -1755,8 +1755,13 @@ def _ensure_updater_image(client: docker.DockerClient) -> None:
except NotFound:
pass
except APIError as exc:
+ refusal = proxy_refusal(exc)
+ if refusal:
+ raise SelfUpdateUnavailable(f"Not updating. {refusal}") from exc
logger.warning("Could not check for the update helper image: %s", exc)
try:
+ # The client's read timeout (60 s by default) ends a pull that stalls;
+ # a slow pull that keeps sending progress goes on.
client.images.pull(UPDATER_IMAGE)
except APIError as exc:
logger.warning("Could not pull the update helper image %s: %s", UPDATER_IMAGE, exc)
@@ -1764,6 +1769,9 @@ def _ensure_updater_image(client: docker.DockerClient) -> None:
if refusal:
raise SelfUpdateUnavailable(f"Not updating. {refusal}") from exc
raise RuntimeError(f"Docker could not pull the update helper image {UPDATER_IMAGE}.") from exc
+ except Exception as exc: # noqa: BLE001 - a read timeout or a dropped socket, not a daemon answer
+ logger.warning("Pulling the update helper image %s did not finish: %s", UPDATER_IMAGE, type(exc).__name__)
+ raise RuntimeError(f"Docker could not pull the update helper image {UPDATER_IMAGE}.") from exc
def _resolve_compose_config(
diff --git a/tests/test_update_feature.py b/tests/test_update_feature.py
index b09a35e8..b7271927 100644
--- a/tests/test_update_feature.py
+++ b/tests/test_update_feature.py
@@ -559,6 +559,21 @@ def test_a_failed_image_check_still_pulls(self):
client.images.pull.assert_called_once_with(orchestrator.UPDATER_IMAGE)
assert _updater_started(client)
+ def test_a_pull_that_stalls_is_an_outage_and_starts_nothing(self):
+ import requests
+
+ me = _fake_container(labels=_COMPOSE_LABELS)
+ client = _fake_docker(me=me)
+ _wire_runs(client)
+ client.images.get.side_effect = self._proxy_404()
+ client.images.pull.side_effect = requests.exceptions.ReadTimeout("read timed out")
+ with (
+ patch.object(orchestrator, "_get_client", return_value=client),
+ pytest.raises(RuntimeError, match="could not pull the update helper image"),
+ ):
+ orchestrator.spawn_self_update()
+ client.containers.run.assert_not_called()
+
def test_a_pull_the_proxy_refuses_says_why_and_starts_nothing(self):
from docker.errors import APIError
From bbf9a95f4562b21ddb43e164532297783a8361fb Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 15:54:07 +0100
Subject: [PATCH 24/34] fix(worker): open, sweep and close the probe client in
one thread
A cancelled await on the client factory could drop a client created after
the cancel, leaving its connection open. _sweep_probes does all three in
one thread; the start-up sweep uses it too.
---
app/worker_api.py | 24 ++++++++++++------------
1 file changed, 12 insertions(+), 12 deletions(-)
diff --git a/app/worker_api.py b/app/worker_api.py
index 4ab49742..2b3ad9d8 100644
--- a/app/worker_api.py
+++ b/app/worker_api.py
@@ -944,6 +944,16 @@ def _seconds_to_next_window(now: float | None = None) -> float:
return activity.WINDOW_SECONDS - (now % activity.WINDOW_SECONDS) + 1
+def _sweep_probes() -> None:
+ """Open a client, sweep, close it, all in one thread: a cancelled await cannot drop the client."""
+ client = activity._client()
+ try:
+ activity.sweep(client)
+ finally:
+ with contextlib.suppress(Exception):
+ client.close()
+
+
async def _activity_cycle() -> None:
"""One round, if the owner switched Activity on and Docker answers.
@@ -954,12 +964,7 @@ async def _activity_cycle() -> None:
if not await asyncio.to_thread(orchestrator.docker_available):
return
if not _activity_enabled:
- client = await asyncio.to_thread(activity._client)
- try:
- await asyncio.to_thread(activity.sweep, client)
- finally:
- with contextlib.suppress(Exception):
- client.close()
+ await asyncio.to_thread(_sweep_probes)
return
switches = _activity_switches
result = await asyncio.to_thread(activity.run_round)
@@ -983,12 +988,7 @@ async def _activity_loop() -> None:
"""
with contextlib.suppress(Exception):
if await asyncio.to_thread(orchestrator.docker_available):
- client = await asyncio.to_thread(activity._client)
- try:
- await asyncio.to_thread(activity.sweep, client)
- finally:
- with contextlib.suppress(Exception):
- client.close()
+ await asyncio.to_thread(_sweep_probes)
while True:
await asyncio.sleep(_seconds_to_next_window())
try:
From 17b7dbecbdb758613bab330047c9e948fc6bb722 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 16:37:04 +0100
Subject: [PATCH 25/34] fix(migrate): adopt pre-worker earners so they stay
visible behind the proxy
The owner migrator only took containers labelled deployed-by=worker, but
CashPilot also counts deployed-by=direct as its own. On a host with such
earners the fence network was left unlabelled and the earners dropped out
of the worker's view. They now count as ours, and each is recreated with
the worker label (with its volumes moved when they need it).
---
app/owner_migrate.py | 38 +++++++++++++++++++++++++++++--------
tests/test_owner_migrate.py | 33 ++++++++++++++++++++++++++++++++
2 files changed, 63 insertions(+), 8 deletions(-)
diff --git a/app/owner_migrate.py b/app/owner_migrate.py
index c8a6c6c9..bd40104b 100644
--- a/app/owner_migrate.py
+++ b/app/owner_migrate.py
@@ -34,6 +34,7 @@
from pathlib import Path
from typing import Any
+from app.constants import CASHPILOT_DEPLOYED
from app.socket_policy import OWNER, OWNER_LABEL
logger = logging.getLogger("cashpilot.migrate")
@@ -106,6 +107,14 @@ def owned(labels: dict[str, str] | None) -> bool:
return (labels or {}).get(OWNER_LABEL) == OWNER
+def ours(labels: dict[str, str] | None) -> bool:
+ """A container CashPilot deployed: by the worker, or by the deploy path
+ before workers ("direct"), whose earners still run on old installs. The
+ proxy shows the worker neither kind until it carries the worker label, so
+ the migration moves both and recreates each "direct" one with it."""
+ return (labels or {}).get(OWNER_LABEL) in CASHPILOT_DEPLOYED
+
+
def backup_name(volume: str) -> str:
return volume + BACKUP_SUFFIX
@@ -185,6 +194,9 @@ def _remove_helper(self) -> None:
with contextlib.suppress(NotFound):
self.client.containers.get(HELPER_NAME).remove(force=True)
+ def _ours(self) -> list[Any]:
+ return [c for c in self.client.containers.list(all=True, filters={"label": OWNER_LABEL}) if ours(c.labels)]
+
# -- the run ------------------------------------------------------------
def run(self, compose_volumes: list[tuple[str, str]]) -> int:
@@ -289,7 +301,7 @@ def migrate_networks(self) -> None:
names = {r["name"] for r in self.state.all("networks") if not r.get("done")}
for net in self.client.networks.list(filters={"label": FENCE_LABEL}):
names.add((net.attrs or {}).get("Name") or net.name)
- for c in self.client.containers.list(all=True, filters={"label": f"{OWNER_LABEL}={OWNER}"}):
+ for c in self._ours():
names.update(((c.attrs or {}).get("NetworkSettings") or {}).get("Networks") or {})
for name in sorted(names - BUILTIN_NETWORKS):
try:
@@ -317,7 +329,7 @@ def migrate_network(self, name: str) -> None:
attrs = net.attrs or {}
members = []
for c in self._on_network(name):
- if not owned(c.labels):
+ if not ours(c.labels):
raise LeftAlone(
f"{c.name} is on it and was not deployed by the worker, so the network was left as it "
f"was, and the worker cannot put services on it. To move it, disconnect that container "
@@ -350,7 +362,7 @@ def migrate_network(self, name: str) -> None:
for c in self._on_network(name):
if c.name in known:
continue
- if not owned(c.labels):
+ if not ours(c.labels):
# Leave it as it is, with the members an earlier run took off back on.
self._reconnect(net, record)
self.state.drop("networks", name)
@@ -465,10 +477,12 @@ def _run_plan_logged(self, plan: dict[str, Any]) -> None:
self.fail(f"{', '.join(c['Name'].lstrip('/') for c in plan['containers'])}: {exc}")
def _new_plans(self) -> list[dict[str, Any]]:
- """Groups of worker containers that share unlabelled volumes, one plan each."""
+ """Groups of worker containers that share unlabelled volumes, one plan each.
+ A "direct" container gets a plan even with no such volume: recreating
+ it is the only way to give it the worker label."""
from docker.errors import NotFound
- workers = self.client.containers.list(all=True, filters={"label": f"{OWNER_LABEL}={OWNER}"})
+ workers = self._ours()
# A stored plan that did not finish already holds these, stopped by it;
# a new plan would record them as stopped and bring them back stopped.
held = {c["Id"] for p in self.state.all("plans") if not p.get("done") for c in p["containers"]}
@@ -488,6 +502,7 @@ def _new_plans(self) -> list[dict[str, Any]]:
continue
users.setdefault(mount["Name"], set()).add(c.id)
attrs[c.id] = c.attrs
+ legacy = [c for c in workers if c.id not in held and not owned(c.labels)]
# Join containers that share a volume: they stop and come back together.
groups: list[tuple[set[str], set[str]]] = []
for vol, cids in users.items():
@@ -497,13 +512,17 @@ def _new_plans(self) -> list[dict[str, Any]]:
merged[0].update(group[0])
merged[1].update(group[1])
groups.append(merged)
+ for c in legacy:
+ if not any(c.id in g[0] for g in groups):
+ attrs[c.id] = c.attrs
+ groups.append(({c.id}, set()))
groups = self._with_joiners(groups, attrs, held)
plans = []
for cids, vols in groups:
strangers = sorted(
attrs[cid]["Name"].lstrip("/")
for cid in cids
- if not owned((attrs[cid].get("Config") or {}).get("Labels"))
+ if not ours((attrs[cid].get("Config") or {}).get("Labels"))
)
if strangers:
self.warn(
@@ -603,7 +622,10 @@ def _current(self, info: dict[str, Any]) -> Any | None:
def run_plan(self, plan: dict[str, Any]) -> None:
names = [c["Name"].lstrip("/") for c in plan["containers"]]
- logger.info("Moving %s onto labelled volumes: %s", ", ".join(names), ", ".join(plan["volumes"]))
+ if plan["volumes"]:
+ logger.info("Moving %s onto labelled volumes: %s", ", ".join(names), ", ".join(plan["volumes"]))
+ else:
+ logger.info("Recreating %s with %s=%s, as the worker deploys it", ", ".join(names), OWNER_LABEL, OWNER)
# A container in another's namespace goes first and comes back last.
in_plan = {c["Id"]: c["Name"].lstrip("/") for c in plan["containers"]}
joined = [c for c in plan["containers"] if _namespace_of(c) in in_plan]
@@ -653,7 +675,7 @@ def run_plan(self, plan: dict[str, Any]) -> None:
self._left_for_the_worker(info)
plan["done"] = True
self.state.put("plans", plan["key"], plan)
- logger.info("%s back as before, on labelled volumes.", ", ".join(names))
+ logger.info("%s back as before, labelled for the worker.", ", ".join(names))
@staticmethod
def _left_for_the_worker(info: dict[str, Any]) -> None:
diff --git a/tests/test_owner_migrate.py b/tests/test_owner_migrate.py
index 8f72894b..812f2b45 100644
--- a/tests/test_owner_migrate.py
+++ b/tests/test_owner_migrate.py
@@ -872,3 +872,36 @@ def test_a_proxied_earner_with_its_own_volume_moves_in_the_same_plan_as_its_tunn
new_earner = d.containers.get("cashpilot-honeygain-p1")
assert new_earner.attrs["HostConfig"]["NetworkMode"] == f"container:{new_tunnel.id}"
assert d.data["honeygain-data"] == {"device": "id"}
+
+
+def test_earners_the_pre_worker_deploy_path_made_are_adopted_with_the_fence(install, tmp_path):
+ """deployed-by=direct counts as CashPilot's (constants.CASHPILOT_DEPLOYED).
+ Left alone, the fence would stay unlabelled and the earners would vanish
+ from the worker behind the proxy."""
+ d, myst, _ = install
+ direct = {"cashpilot.managed": "true", "cashpilot.service": "honeygain", om.OWNER_LABEL: "direct"}
+ old = d.add(Container(d, "cashpilot-honeygain", direct, networks=("cashpilot-isolated",)))
+ d.volume("earnapp-data", {"uuid": "sdk-node-1"})
+ old_vol = d.add(
+ Container(
+ d,
+ "cashpilot-earnapp",
+ {**direct, "cashpilot.service": "earnapp"},
+ [("earnapp-data", "/etc/earnapp")],
+ networks=("cashpilot-isolated",),
+ running=False,
+ )
+ )
+ m = FakeMigrator(d, om.State(tmp_path))
+ assert m.run([]) == 0 and m.warnings == 0
+ assert om.owned(d.nets["cashpilot-isolated"].attrs["Labels"])
+ hg = d.containers.get("cashpilot-honeygain")
+ assert hg.id != old.id and hg.status == "running"
+ assert hg.labels == {**direct, **OWNER, om.RECREATED_LABEL: old.id}
+ assert hg.networks() == ["cashpilot-isolated"]
+ ea = d.containers.get("cashpilot-earnapp")
+ assert ea.id != old_vol.id and ea.status == "exited" and om.owned(ea.labels)
+ assert om.owned(d.vols["earnapp-data"].attrs["Labels"]) and d.data["earnapp-data"] == {"uuid": "sdk-node-1"}
+ again = FakeMigrator(d, om.State(tmp_path))
+ assert again.run([]) == 0 and again.copies == []
+ assert d.containers.get("cashpilot-honeygain").id == hg.id
From 327ad50f8bc79f97486d8f9765ac3f6949f47063 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 16:39:15 +0100
Subject: [PATCH 26/34] fix(update): stop the refusal sending deployer-run
stacks to a path that is not on the host
When Compose runs inside a webhook or deployer container, the working_dir
label holds the deployer's own path. The refusal told the operator to set
CASHPILOT_COMPOSE_DIR to it as if it were the host folder. It now names the
path as what Compose recorded and says to update from the deployer in that
case.
---
app/orchestrator.py | 13 ++++++++++---
tests/test_socket_proxy.py | 19 +++++++++++++++++++
2 files changed, 29 insertions(+), 3 deletions(-)
diff --git a/app/orchestrator.py b/app/orchestrator.py
index 4c096f99..c1e9b6c4 100644
--- a/app/orchestrator.py
+++ b/app/orchestrator.py
@@ -1907,10 +1907,17 @@ def _check_proxy_binds(working_dir: str, extra_paths: list[str]) -> None:
"""Refuse, with the line to paste, an update the proxy would not let bind its folders."""
compose_dir = os.getenv("CASHPILOT_COMPOSE_DIR", "").strip().rstrip("/")
if compose_dir != working_dir.rstrip("/"):
+ # working_dir is the path Compose itself ran in. Under a webhook or a
+ # deployer container (Portainer and the like) that path is inside the
+ # deployer and may not exist on this host; the helper binds it as is,
+ # so no other value would work either: the deployer has to update.
raise SelfUpdateUnavailable(
- "The update helper needs your compose folder, and the Docker socket proxy lets "
- "containers bind only the folders it was told about. Add this line to the .env "
- f"next to your compose file, then run docker compose up -d once: "
+ f"The update helper needs your compose folder, which Docker Compose recorded as "
+ f"{working_dir}, and the Docker socket proxy lets containers bind only the folders "
+ "it was told about. If Compose runs inside another container for you (a webhook, "
+ "Portainer or another deployer), that path is inside it, not on this host: update "
+ "the stack from your deployer instead. Otherwise add this line to the .env next to "
+ "your compose file, then run docker compose up -d once: "
f"{COMPOSE_DIR_ENV}={working_dir}"
)
roots = _proxy_bind_roots()
diff --git a/tests/test_socket_proxy.py b/tests/test_socket_proxy.py
index b0e24499..ba45cd62 100644
--- a/tests/test_socket_proxy.py
+++ b/tests/test_socket_proxy.py
@@ -611,6 +611,25 @@ def test_proxied_a_compose_folder_the_proxy_does_not_know_is_refused_with_the_li
client.containers.run.assert_not_called()
+def test_proxied_under_a_deployer_container_the_refusal_does_not_pass_its_path_off_as_the_hosts(monkeypatch):
+ """Compose run by a webhook or Portainer records the deployer's own path
+ (/data/compose/12), which need not exist on the host. The advice must say
+ so and send the operator to the deployer, not only to that path."""
+ _proxied_env(monkeypatch, compose_dir="")
+ labels = {**tuf._COMPOSE_LABELS, "com.docker.compose.project.working_dir": "/data/compose/12"}
+ client = tuf._fake_docker(me=tuf._fake_container(labels=labels))
+ tuf._wire_runs(client)
+ with (
+ patch.object(orchestrator, "_get_client", return_value=client),
+ pytest.raises(orchestrator.SelfUpdateUnavailable) as refused,
+ ):
+ orchestrator.spawn_self_update()
+ message = str(refused.value)
+ assert "recorded as /data/compose/12" in message
+ assert "not on this host" in message and "update the stack from your deployer" in message
+ client.containers.run.assert_not_called()
+
+
def test_proxied_a_trailing_slash_is_the_same_folder(monkeypatch):
_proxied_env(monkeypatch, compose_dir="/srv/cashpilot/")
client, result = _run_update()
From 13cd3e7c5c6359021e40dae72a7e108bb302b70c Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 16:43:33 +0100
Subject: [PATCH 27/34] fix(proxies): give a tunnel on the isolated bridge the
bridge's resolvers
Earners on the isolated bridge get container_dns(), because its rules refuse
the daemon's LAN resolver. The proxy tunnel joined the same bridge without
them. Same gap Desktop has in sidecarCreate.
---
app/proxy_tunnel.py | 3 +++
tests/test_proxy_tunnel.py | 9 +++++++++
2 files changed, 12 insertions(+)
diff --git a/app/proxy_tunnel.py b/app/proxy_tunnel.py
index 23a6331b..02331eee 100644
--- a/app/proxy_tunnel.py
+++ b/app/proxy_tunnel.py
@@ -277,6 +277,9 @@ def sidecar_kwargs(name: str, slug: str, proxy_id: int | None, network: str | No
"name": name,
"labels": labels,
"network": network,
+ # The isolated bridge's resolvers, as every earner on it gets them: its
+ # rules refuse the daemon's LAN resolver. None on the default bridge.
+ "dns": orchestrator.container_dns(network),
"hostname": hostname,
"cap_drop": ["ALL"],
"cap_add": ["NET_ADMIN"],
diff --git a/tests/test_proxy_tunnel.py b/tests/test_proxy_tunnel.py
index 7fea50a9..ac08be81 100644
--- a/tests/test_proxy_tunnel.py
+++ b/tests/test_proxy_tunnel.py
@@ -172,6 +172,15 @@ def test_hardening(self):
assert "environment" not in kw and "command" not in kw
assert kw["log_config"] == {"type": "json-file", "config": {"max-size": "1m", "max-file": "2"}}
+ def test_a_tunnel_on_the_isolated_bridge_gets_its_resolvers(self):
+ # The bridge's rules refuse the daemon's LAN resolver, so the tunnel
+ # needs the same resolvers every earner on it gets (container_dns).
+ with patch.dict(os.environ, {"CASHPILOT_TUNNEL_IMAGE": "drumsergio/cashpilot-tunnel:1.64.0"}):
+ fenced = proxy_tunnel.sidecar_kwargs("cashpilot-tunnel-x-p1", "x", 1, "cp-isolated", "x-p1")
+ plain = proxy_tunnel.sidecar_kwargs("cashpilot-tunnel-x-p1", "x", 1, None, "x-p1")
+ assert fenced["dns"] == orchestrator.container_dns("cp-isolated") == ["1.1.1.1", "9.9.9.9"]
+ assert plain["dns"] is None
+
def test_secret_goes_only_into_the_archive(self):
client = MagicMock()
created = MagicMock(id="sidecar-id", status="running")
From d130f8ee184f7514767a9323f7b46e8af7a4795c Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 16:53:44 +0100
Subject: [PATCH 28/34] docs(faq): stop saying earners reach the LAN by default
Since 1.70 a new install fences its earners off the LAN, host, tailnet and
cloud metadata. The FAQ still said they reach the LAN by default.
---
docs/faq.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/faq.md b/docs/faq.md
index 167fd13e..fe37750b 100644
--- a/docs/faq.md
+++ b/docs/faq.md
@@ -38,7 +38,7 @@ If CashPilot cannot write the key to disk at all, because `/data` is unwritable
**What about security?**
-Every service CashPilot deploys runs in its own container with every Linux capability dropped (a few services add back only the ones they declare) and `--security-opt no-new-privileges` set, so a process inside cannot gain privileges. Containers cannot see your host filesystem beyond the volumes a service declares. They CAN reach your local network by default, because Docker does not block that; [Protecting Your Home Network](home-network-security.md) shows how to put them on a firewalled bridge that reaches the internet but not your LAN, router or host. That bridge does not cover a service on host networking (Mysterium, today); the guide says what to do about those. Service credentials are encrypted at rest using Fernet symmetric encryption. Only the worker container requires Docker socket access; the UI container has no privileged access.
+Every service CashPilot deploys runs in its own container with every Linux capability dropped (a few services add back only the ones they declare) and `--security-opt no-new-privileges` set, so a process inside cannot gain privileges. Containers cannot see your host filesystem beyond the volumes a service declares. On a new install the worker also fences them off your LAN, the host, your tailnet and cloud metadata, so they reach the internet and nothing local ([LAN isolation](isolation.md)). An install upgraded from a release before 1.70 keeps its earners on Docker's default bridge, which can reach your LAN, until you click **Turn on isolation** on the Fleet page. The fence does not cover a service on host networking (Mysterium, today); [the isolation page](isolation.md) says what to do about those, and [Protecting Your Home Network](home-network-security.md) covers firewalling your own bridge instead. Service credentials are encrypted at rest using Fernet symmetric encryption. Only the worker container requires Docker socket access; the UI container has no privileged access.
That said, no setup is bulletproof. You are still running third-party software that routes external traffic through your network. Docker isolation significantly reduces the attack surface compared to running these services directly on your host, but it does not eliminate all risk. We recommend running CashPilot on a dedicated machine or VLAN, keeping Docker and your host OS up to date, and reviewing the open-source code of any service before deploying it.
From 36ba03ba0e3b1a67f52df3c683e2b676e2620c87 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 20:41:09 +0100
Subject: [PATCH 29/34] fix(proxies): never let a cap restart land after the
owner stopped the copy
The heartbeat retry and the reset read the cap mark, then sent the start; an
owner stop in between cleared the mark too late, and the start that followed
left the copy running. The owner's start or stop and both cap starts now hold
one lock per copy from the mark check to the mark update.
The isolated-bridge tunnel test now pins the resolvers it expects, so a
CASHPILOT_CONTAINER_DNS set in the environment cannot fail it.
---
app/main.py | 98 ++++++++++++++++++++++++--------------
tests/test_proxies.py | 30 ++++++++++++
tests/test_proxy_tunnel.py | 1 +
3 files changed, 93 insertions(+), 36 deletions(-)
diff --git a/app/main.py b/app/main.py
index e66e3e1d..d233ccee 100644
--- a/app/main.py
+++ b/app/main.py
@@ -4197,6 +4197,26 @@ def _proxy_state_lock(proxy_id: int) -> asyncio.Lock:
return locks.setdefault(proxy_id, asyncio.Lock())
+# One lock per copy, held by the owner's start or stop and by the cap's own starts
+# from the mark check to the mark update, so a start the cap decided on can never
+# land after the owner's stop.
+_proxy_copy_locks: weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, dict[tuple[int, int, str], asyncio.Lock]] = (
+ weakref.WeakKeyDictionary()
+)
+
+
+def _proxy_copy_lock(proxy_id: int, worker_id: int, slug: str) -> asyncio.Lock:
+ locks = _proxy_copy_locks.setdefault(asyncio.get_running_loop(), {})
+ return locks.setdefault((proxy_id, worker_id, slug), asyncio.Lock())
+
+
+async def _copy_marked_by_cap(proxy_id: int, worker_id: int, slug: str) -> bool:
+ return any(
+ c["worker_id"] == worker_id and c["slug"] == slug and c.get("stopped_by_cap")
+ for c in await database.list_proxied_copies(proxy_id)
+ )
+
+
@app.post("/api/proxies/{proxy_id}/copies")
async def api_add_proxied_copy(
proxy_id: int, body: ProxyCopyRequest, _auth: dict[str, Any] = Depends(_require_owner)
@@ -4291,17 +4311,18 @@ async def api_proxied_copy_command(
proxy = await _proxy_or_404(proxy_id)
if command == "start" and proxy["state"].get("capped_at"):
raise HTTPException(status_code=409, detail=proxy["state"].get("cap_reason") or "This proxy reached its cap")
- result = await _proxy_worker_call(
- worker_id,
- "POST",
- f"/api/proxied/{slug}/{proxy_id}/{command}",
- proxy["definition"],
- # The proxy as stored now, so an edit since the deploy reaches the copy.
- json={"proxy": proxies.worker_proxy_spec(proxy["definition"])} if command == "start" else None,
- timeout=90,
- )
- # A start or stop by hand is the owner's decision; a later reset must not undo it.
- await database.set_proxied_copy_capped(proxy_id, worker_id, slug, False)
+ async with _proxy_copy_lock(proxy_id, worker_id, slug):
+ result = await _proxy_worker_call(
+ worker_id,
+ "POST",
+ f"/api/proxied/{slug}/{proxy_id}/{command}",
+ proxy["definition"],
+ # The proxy as stored now, so an edit since the deploy reaches the copy.
+ json={"proxy": proxies.worker_proxy_spec(proxy["definition"])} if command == "start" else None,
+ timeout=90,
+ )
+ # A start or stop by hand is the owner's decision; a later reset must not undo it.
+ await database.set_proxied_copy_capped(proxy_id, worker_id, slug, False)
return result
@@ -4351,8 +4372,12 @@ async def _release_capped_copies(proxy: dict[str, Any]) -> list[str]:
continue
_proxy_cap_starting.add(key)
try:
- await _start_proxied_copy(proxy, copy["worker_id"], copy["slug"])
- await database.set_proxied_copy_capped(proxy["id"], copy["worker_id"], copy["slug"], False)
+ async with _proxy_copy_lock(*key):
+ # The owner may have started or stopped it since the list was read.
+ if not await _copy_marked_by_cap(*key):
+ continue
+ await _start_proxied_copy(proxy, copy["worker_id"], copy["slug"])
+ await database.set_proxied_copy_capped(proxy["id"], copy["worker_id"], copy["slug"], False)
started.append(f"{copy['slug']} on worker {copy['worker_id']}")
except HTTPException as exc:
logger.warning(
@@ -4462,29 +4487,30 @@ async def _restart_capped_copies(proxy_id: int, worker_id: int, slugs: set[str])
"""Start copies the cap stopped and a reset could not start, unless the cap is back."""
try:
for slug in sorted(slugs):
- async with _proxy_state_lock(proxy_id):
- fresh = await database.get_proxy(proxy_id)
- # The owner may have started or stopped it since the heartbeat; that
- # clears the mark, and the owner's decision stands.
- marked = any(
- c["worker_id"] == worker_id and c["slug"] == slug and c.get("stopped_by_cap")
- for c in await database.list_proxied_copies(proxy_id)
- )
- if not fresh or fresh["state"].get("capped_at"):
- return
- if not marked:
- continue
- try:
- await _start_proxied_copy(fresh, worker_id, slug)
- except HTTPException as exc:
- logger.warning("Could not restart %s through proxy %s after a reset: %s", slug, proxy_id, exc.detail)
- continue
- # The cap may have come back while the start ran. Then the mark stays, so
- # the next heartbeat stops the copy again and the next reset starts it.
- async with _proxy_state_lock(proxy_id):
- again = await database.get_proxy(proxy_id)
- if again and not again["state"].get("capped_at"):
- await database.set_proxied_copy_capped(proxy_id, worker_id, slug, False)
+ async with _proxy_copy_lock(proxy_id, worker_id, slug):
+ async with _proxy_state_lock(proxy_id):
+ fresh = await database.get_proxy(proxy_id)
+ # The owner may have started or stopped it since the heartbeat; that
+ # clears the mark, and the owner's decision stands. The copy lock
+ # keeps that true until this start has landed.
+ marked = await _copy_marked_by_cap(proxy_id, worker_id, slug)
+ if not fresh or fresh["state"].get("capped_at"):
+ return
+ if not marked:
+ continue
+ try:
+ await _start_proxied_copy(fresh, worker_id, slug)
+ except HTTPException as exc:
+ logger.warning(
+ "Could not restart %s through proxy %s after a reset: %s", slug, proxy_id, exc.detail
+ )
+ continue
+ # The cap may have come back while the start ran. Then the mark stays, so
+ # the next heartbeat stops the copy again and the next reset starts it.
+ async with _proxy_state_lock(proxy_id):
+ again = await database.get_proxy(proxy_id)
+ if again and not again["state"].get("capped_at"):
+ await database.set_proxied_copy_capped(proxy_id, worker_id, slug, False)
finally:
_proxy_cap_starting.difference_update((proxy_id, worker_id, slug) for slug in slugs)
diff --git a/tests/test_proxies.py b/tests/test_proxies.py
index 9fd114d3..03b1a1c9 100644
--- a/tests/test_proxies.py
+++ b/tests/test_proxies.py
@@ -926,6 +926,36 @@ async def go():
assert after_heartbeat == 0
assert not main._proxy_cap_starting
+ def test_a_retry_never_starts_a_copy_after_the_owner_stopped_it(self, db):
+ """The owner's stop is in flight when the retry fires: the retry must not undo it."""
+ calls = []
+
+ async def go():
+ gate = asyncio.Event()
+
+ async def worker_call(worker_id, method, path, definition, **kw):
+ calls.append(path)
+ if path.endswith("/stop"):
+ await gate.wait()
+ return {}
+
+ pid = (await main.api_create_proxy(_Req(_socks(cap_gb=1)), {}))["id"]
+ await database.save_proxied_copy(pid, 7, "honeygain")
+ await database.set_proxied_copy_capped(pid, 7, "honeygain", True)
+ with patch.object(main, "_proxy_worker_call", side_effect=worker_call):
+ stop = asyncio.create_task(main.api_proxied_copy_command(pid, 7, "honeygain", "stop", {}))
+ while not calls:
+ await asyncio.sleep(0)
+ retry = asyncio.create_task(main._restart_capped_copies(pid, 7, {"honeygain"}))
+ for _ in range(20):
+ await asyncio.sleep(0)
+ gate.set()
+ await asyncio.gather(stop, retry)
+ return pid
+
+ pid = _run(go)
+ assert calls == [f"/api/proxied/honeygain/{pid}/stop"], calls
+
@pytest.mark.parametrize(
("capped", "marked", "status"),
[
diff --git a/tests/test_proxy_tunnel.py b/tests/test_proxy_tunnel.py
index bc4f9d5b..4753adb9 100644
--- a/tests/test_proxy_tunnel.py
+++ b/tests/test_proxy_tunnel.py
@@ -172,6 +172,7 @@ def test_hardening(self):
assert "environment" not in kw and "command" not in kw
assert kw["log_config"] == {"type": "json-file", "config": {"max-size": "1m", "max-file": "2"}}
+ @patch.object(orchestrator, "_CONTAINER_DNS", "1.1.1.1,9.9.9.9")
def test_a_tunnel_on_the_isolated_bridge_gets_its_resolvers(self):
# The bridge's rules refuse the daemon's LAN resolver, so the tunnel
# needs the same resolvers every earner on it gets (container_dns).
From 9f8f7c8a69fec0057fcf446b074bbc973a1232a0 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 22:26:57 +0100
Subject: [PATCH 30/34] fix(isolation): refuse to fence an engine that cannot
be identified
An engine whose /version fails was taken for Docker, so a Podman that
answered nothing else could get a network the fence's rules do not name.
Fail closed instead, and give each fence helper of another process its
own wait deadline so a replaced helper is not killed at the old one's.
---
app/lan_fence.py | 23 ++++++++++++++++-------
tests/test_lan_fence.py | 40 ++++++++++++++++++++++++++++++++++++++--
2 files changed, 54 insertions(+), 9 deletions(-)
diff --git a/app/lan_fence.py b/app/lan_fence.py
index 5830e364..4b053e6c 100644
--- a/app/lan_fence.py
+++ b/app/lan_fence.py
@@ -645,14 +645,17 @@ def is_podman(client: Any) -> bool:
"""A Podman engine behind the Docker-compatible API.
Podman names itself in /version ("Podman Engine"). An engine that does not
- answer is taken for Docker: the fence's own checks still refuse a network
- it cannot hold.
+ answer cannot be told apart, and a Podman taken for Docker gets a network
+ whose bridge is not the one the fence's rules name, so this raises
+ RuntimeError instead of guessing: isolation fails closed.
"""
try:
version = client.version() or {}
- except Exception as exc: # noqa: BLE001 - taken for Docker, as above
- logger.debug("Could not read the engine's /version, so it is taken for Docker: %s", exc)
- return False
+ except Exception as exc: # noqa: BLE001 - re-raised as the documented RuntimeError
+ raise RuntimeError(
+ f"Could not read the container engine's /version ({exc}), so it cannot tell Docker from Podman "
+ "and the LAN fence was not applied."
+ ) from exc
names = [str((version.get("Platform") or {}).get("Name") or "")]
names += [str((c or {}).get("Name") or "") for c in version.get("Components") or []]
return any("podman" in n.lower() for n in names)
@@ -854,12 +857,15 @@ def _clear_helper(client: Any) -> None:
"""
from docker.errors import NotFound
- deadline = time.monotonic() + HELPER_TIMEOUT + HELPER_GRACE
+ # Per helper: one another process created after the last poll gets its own wait.
+ waiting_on, deadline = None, 0.0
while True:
try:
found = client.containers.get(HELPER_NAME)
if (getattr(found, "labels", None) or {}).get(PROBE_LABEL) != "fence":
return
+ if getattr(found, "id", None) != waiting_on:
+ waiting_on, deadline = getattr(found, "id", None), time.monotonic() + HELPER_TIMEOUT + HELPER_GRACE
if not _helper_busy(found.attrs or {}, time.time()) or time.monotonic() > deadline:
found.remove(force=True, v=True)
return
@@ -1028,7 +1034,10 @@ def network_for_deploy() -> str | None:
"""
orch = _orchestrator()
client = orch._get_client()
- mode = resolve_mode(client)
+ try:
+ mode = resolve_mode(client)
+ except RuntimeError as exc:
+ raise orch.ContainerNetworkError(f"{exc} The running container was not touched. {_off_hint()}") from None
_state["mode"] = mode
if mode == MODE_UNSUPPORTED and setting() == "on":
raise orch.ContainerNetworkError(
diff --git a/tests/test_lan_fence.py b/tests/test_lan_fence.py
index eae5373d..363c65d7 100644
--- a/tests/test_lan_fence.py
+++ b/tests/test_lan_fence.py
@@ -1010,6 +1010,23 @@ def test_one_still_there_after_the_helper_timeout_is_removed(self):
_run_helper(client)
stuck.remove.assert_called_once_with(force=True, v=True)
+ def test_a_helper_that_replaced_the_one_waited_for_gets_its_own_wait(self):
+ first, second = _helper_container("running"), _helper_container("running")
+ first.id, second.id = "first", "second"
+ client = MagicMock()
+ client.containers.get.side_effect = [first, second, NotFound("gone")]
+ # first is seen at t=0; second only at t=11, past first's deadline of 10.
+ clock = iter([0.0, 0.0, 11.0, 11.0])
+ with (
+ patch.object(lan_fence, "HELPER_TIMEOUT", 10),
+ patch.object(lan_fence, "HELPER_GRACE", 0),
+ patch.object(lan_fence, "HELPER_POLL", 0),
+ patch.object(lan_fence.time, "monotonic", side_effect=lambda: next(clock, 11.0)),
+ ):
+ lan_fence._clear_helper(client)
+ second.remove.assert_not_called()
+ first.remove.assert_not_called()
+
def test_only_its_own_helper_is_removed_afterwards(self):
"""By its own object: a name lookup after the run could find the other side's new helper."""
client = MagicMock()
@@ -1101,10 +1118,29 @@ def test_it_is_detected_from_the_version(self, version, podman):
client.version.return_value = version
assert lan_fence.is_podman(client) is podman
- def test_an_engine_that_does_not_answer_is_taken_for_docker(self):
+ def test_an_engine_that_does_not_answer_fails_closed(self):
client = MagicMock()
client.version.side_effect = OSError("no answer")
- assert lan_fence.is_podman(client) is False
+ with pytest.raises(RuntimeError, match="cannot tell Docker from Podman"):
+ lan_fence.is_podman(client)
+
+ @pytest.mark.parametrize("setting", ["auto", "on"])
+ def test_a_deploy_is_refused_when_the_engine_cannot_be_identified(self, setting):
+ old = MagicMock()
+ old.labels = {"cashpilot.service": "honeygain"}
+ client = _docker()
+ client.version.side_effect = OSError("no answer")
+ with pytest.raises(orchestrator.ContainerNetworkError, match="cannot tell Docker from Podman"):
+ _deploy(client, setting, old=old)
+ old.stop.assert_not_called()
+ old.remove.assert_not_called()
+ client.containers.run.assert_not_called()
+ client.networks.create.assert_not_called()
+
+ def test_off_never_asks_for_the_version(self):
+ client = _docker()
+ client.version.side_effect = OSError("no answer")
+ assert _mode(client, "off") == lan_fence.MODE_OFF
@pytest.mark.parametrize("setting", ["auto", "on"])
def test_it_resolves_to_unsupported_and_records_nothing(self, setting):
From dbfc68a889b391672e6e585f54648e4bb8744d43 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 22:27:04 +0100
Subject: [PATCH 31/34] fix(proxies): keep stop, start and removal from racing
a redeploy
A removal during a redeploy could succeed before the staged sidecar was
promoted, leaving a copy running with no record. Stop, start and remove
now wait for the copy's deploy. A young staged sidecar is never taken
for a leftover, and the heartbeat clears staged sidecars past their
deploy's timeout.
---
app/proxy_tunnel.py | 37 ++++++++++++++++++---
tests/test_proxy_tunnel.py | 66 +++++++++++++++++++++++++++++++++-----
2 files changed, 90 insertions(+), 13 deletions(-)
diff --git a/app/proxy_tunnel.py b/app/proxy_tunnel.py
index 1b340958..5c61c01e 100644
--- a/app/proxy_tunnel.py
+++ b/app/proxy_tunnel.py
@@ -70,8 +70,9 @@
# deploy() swapping the old pair for the staged one, and status(), take turns: a
# heartbeat must not start an earner that is being removed, or read a half swap.
_swap_lock = threading.Lock()
-# One deploy per copy at a time: a second one would take the first one's staged
-# sidecar for a leftover and remove it.
+# One operation per copy at a time: a second deploy would take the first one's
+# staged sidecar for a leftover, and a stop, start or removal that ran during a
+# deploy would be undone when the deploy promotes its sidecar.
_deploy_locks: dict[tuple[str, int], threading.Lock] = {}
_deploy_locks_guard = threading.Lock()
#: A staged sidecar older than this is a leftover: a deploy is past its ready
@@ -448,15 +449,21 @@ def _deploy(slug: str, proxy_id: int, earner: dict[str, Any], proxy: dict[str, A
tunnel_image()
final = tunnel_name(slug, proxy_id)
staged = final + STAGED_SUFFIX
- # Left by a deploy that died midway, on this name only if it is ours.
+ # Left by a deploy that died midway, on this name only if it is ours. A young
+ # one can belong to a deploy another worker on this daemon is running.
if (leftover := _get(client, staged)) is not None:
if not _is_ours(leftover, slug, proxy_id):
raise ProxyTunnelError(f"A container named {staged} that CashPilot did not create is in the way")
+ if _age_seconds(leftover) < STAGED_STALE_SECONDS:
+ raise ProxyTunnelError(
+ f"Another deploy of this copy started {int(_age_seconds(leftover))} s ago and may still be running. "
+ f"Try again in a few minutes."
+ )
_remove_quietly(leftover, 2)
sidecar, exit_info = start_sidecar(client, staged, slug, proxy_id, proxy, own_ip, hostname=_hostname(final))
try:
with _swap_lock:
- remove(slug, proxy_id)
+ _remove(slug, proxy_id)
sidecar.rename(final)
except Exception:
_remove_quietly(sidecar, 2)
@@ -509,6 +516,11 @@ def stop(slug: str, proxy_id: int) -> bool:
state (earner_action), which would undo the stop. An earner left running
after its tunnel stopped has no way out, and earner_action leaves it alone.
"""
+ with _deploy_lock(slug, proxy_id):
+ return _stop(slug, proxy_id)
+
+
+def _stop(slug: str, proxy_id: int) -> bool:
earner, sidecar = _pair(slug, proxy_id)
if not _is_ours(earner, slug, proxy_id) and not _is_ours(sidecar, slug, proxy_id):
raise ValueError(f"No proxied copy of {slug} through proxy {proxy_id} on this worker")
@@ -533,6 +545,11 @@ def start(slug: str, proxy_id: int, proxy: dict[str, Any] | None = None, own_ip:
The earner is restarted rather than started when it is still running in a
namespace the old tunnel left behind.
"""
+ with _deploy_lock(slug, proxy_id):
+ return _start(slug, proxy_id, proxy, own_ip)
+
+
+def _start(slug: str, proxy_id: int, proxy: dict[str, Any] | None, own_ip: str | None) -> dict[str, Any]:
earner, sidecar = _pair(slug, proxy_id)
if not _is_ours(earner, slug, proxy_id) or not _is_ours(sidecar, slug, proxy_id):
raise ValueError(f"No proxied copy of {slug} through proxy {proxy_id} on this worker")
@@ -564,6 +581,11 @@ def start(slug: str, proxy_id: int, proxy: dict[str, Any] | None = None, own_ip:
def remove(slug: str, proxy_id: int) -> list[str]:
"""Remove the earner, then its tunnel and the tunnel's secret volume."""
+ with _deploy_lock(slug, proxy_id):
+ return _remove(slug, proxy_id)
+
+
+def _remove(slug: str, proxy_id: int) -> list[str]:
removed: list[str] = []
earner, sidecar = _pair(slug, proxy_id)
for container, timeout in ((earner, orchestrator._get_stop_timeout(slug)), (sidecar, 10)):
@@ -694,7 +716,12 @@ def _status(collect_stats: bool) -> list[dict[str, Any]]:
pairs: dict[tuple[str, str], dict[str, Any]] = {}
for container in found:
if str(getattr(container, "name", "")).endswith(STAGED_SUFFIX):
- continue # a redeploy's sidecar, not promoted yet
+ # A redeploy's sidecar, not promoted yet. One past any deploy's
+ # timeout is a leftover holding the proxy login: removed here, since
+ # the startup sweep skips those still young when the worker starts.
+ if _age_seconds(container) >= STAGED_STALE_SECONDS:
+ _remove_quietly(container, 2)
+ continue
labels = container.labels or {}
key = (labels.get(LABEL_SERVICE, ""), labels.get(LABEL_PROXY, ""))
role = "tunnel" if labels.get(LABEL_ROLE) == ROLE_TUNNEL else "earner"
diff --git a/tests/test_proxy_tunnel.py b/tests/test_proxy_tunnel.py
index 4753adb9..ee011d88 100644
--- a/tests/test_proxy_tunnel.py
+++ b/tests/test_proxy_tunnel.py
@@ -9,6 +9,7 @@
import io
import os
import tarfile
+from datetime import UTC, datetime, timedelta
from unittest.mock import MagicMock, patch
os.environ.setdefault("CASHPILOT_API_KEY", "test-fleet-key")
@@ -42,7 +43,9 @@ def _container(name, slug, *, proxy=None, role=None, instance=None, status="runn
if instance:
labels[LABEL_INSTANCE] = str(instance)
c.labels = labels
- c.attrs = {"Created": "2026-11-04T00:00:00Z", "State": {"StartedAt": "2026-11-04T10:00:00Z"}}
+ # A day old, so a staged sidecar built here is past STAGED_STALE_SECONDS whatever the clock says.
+ created = (datetime.now(UTC) - timedelta(days=1)).isoformat().replace("+00:00", "Z")
+ c.attrs = {"Created": created, "State": {"StartedAt": "2026-11-04T10:00:00Z"}}
c.image.tags = [f"{slug}:1"]
return c
@@ -314,7 +317,7 @@ def fake_deploy_raw(**kw):
with (
patch.object(orchestrator, "_get_client", return_value=client),
patch.object(proxy_tunnel, "tunnel_image", return_value="drumsergio/cashpilot-tunnel:1.64.0"),
- patch.object(proxy_tunnel, "remove", side_effect=lambda *a: calls.append("old-removed") or []),
+ patch.object(proxy_tunnel, "_remove", side_effect=lambda *a: calls.append("old-removed") or []),
patch.object(proxy_tunnel, "start_sidecar", side_effect=fake_start_sidecar) as start,
patch.object(orchestrator, "deploy_raw", side_effect=fake_deploy_raw),
):
@@ -331,7 +334,7 @@ def test_a_sidecar_that_cannot_start_keeps_the_running_copy(self):
with (
patch.object(orchestrator, "_get_client", return_value=client),
patch.object(proxy_tunnel, "tunnel_image", return_value="drumsergio/cashpilot-tunnel:1.64.0"),
- patch.object(proxy_tunnel, "remove") as remove,
+ patch.object(proxy_tunnel, "_remove") as remove,
patch.object(proxy_tunnel, "start_sidecar", side_effect=proxy_tunnel.ProxyTunnelError("No such image")),
pytest.raises(proxy_tunnel.ProxyTunnelError),
):
@@ -346,7 +349,7 @@ def test_a_failed_promotion_removes_the_staged_sidecar(self):
with (
patch.object(orchestrator, "_get_client", return_value=client),
patch.object(proxy_tunnel, "tunnel_image", return_value="drumsergio/cashpilot-tunnel:1.64.0"),
- patch.object(proxy_tunnel, "remove", return_value=[]),
+ patch.object(proxy_tunnel, "_remove", return_value=[]),
patch.object(proxy_tunnel, "start_sidecar", return_value=(sidecar, {"ip": "93.184.215.14"})),
patch.object(orchestrator, "deploy_raw") as deploy_raw,
pytest.raises(RuntimeError),
@@ -365,7 +368,7 @@ def test_a_staged_sidecar_left_behind_is_replaced_but_never_a_stranger(self):
with (
patch.object(orchestrator, "_get_client", return_value=client),
patch.object(proxy_tunnel, "tunnel_image", return_value="drumsergio/cashpilot-tunnel:1.64.0"),
- patch.object(proxy_tunnel, "remove", return_value=[]),
+ patch.object(proxy_tunnel, "_remove", return_value=[]),
patch.object(proxy_tunnel, "start_sidecar", return_value=(sidecar, {"ip": "93.184.215.14"})),
patch.object(orchestrator, "deploy_raw", return_value="earner-id"),
):
@@ -376,6 +379,54 @@ def test_a_staged_sidecar_left_behind_is_replaced_but_never_a_stranger(self):
proxy_tunnel.deploy("honeygain", 3, {"image": "honeygain/honeygain:1"}, _socks(), None)
assert found.remove.called is ok
+ def test_a_young_staged_sidecar_is_never_removed_by_another_deploy(self):
+ young = _container("cashpilot-tunnel-honeygain-p3-next", "honeygain", proxy=3, role="tunnel")
+ young.attrs["Created"] = datetime.now(UTC).isoformat().replace("+00:00", "Z")
+ client = MagicMock()
+ client.containers.get.return_value = young
+ with (
+ patch.object(orchestrator, "_get_client", return_value=client),
+ patch.object(proxy_tunnel, "tunnel_image", return_value="drumsergio/cashpilot-tunnel:1.64.0"),
+ patch.object(proxy_tunnel, "start_sidecar") as start,
+ pytest.raises(proxy_tunnel.ProxyTunnelError, match="may still be running"),
+ ):
+ proxy_tunnel.deploy("honeygain", 3, {"image": "honeygain/honeygain:1"}, _socks(), None)
+ young.remove.assert_not_called()
+ start.assert_not_called()
+
+ def test_the_heartbeat_removes_a_stale_staged_sidecar_and_keeps_a_young_one(self):
+ stale = _container("cashpilot-tunnel-x-p1-next", "x", proxy=1, role="tunnel")
+ young = _container("cashpilot-tunnel-y-p1-next", "y", proxy=1, role="tunnel")
+ young.attrs["Created"] = datetime.now(UTC).isoformat().replace("+00:00", "Z")
+ client = MagicMock()
+ client.containers.list.return_value = [stale, young]
+ with (
+ patch.object(orchestrator, "_get_client", return_value=client),
+ patch.object(proxy_tunnel, "_read_exit", return_value=None),
+ ):
+ assert proxy_tunnel.status(collect_stats=False) == []
+ stale.remove.assert_called_once_with(force=True, v=True)
+ young.remove.assert_not_called()
+
+ @pytest.mark.parametrize("op", ["stop", "start", "remove"])
+ def test_stop_start_and_remove_wait_for_a_deploy_of_the_same_copy(self, op):
+ import threading
+
+ done = threading.Event()
+ lock = proxy_tunnel._deploy_lock("x", 1)
+ lock.acquire()
+ try:
+ with patch.object(proxy_tunnel, f"_{op}", side_effect=lambda *a: done.set()):
+ t = threading.Thread(target=getattr(proxy_tunnel, op), args=("x", 1))
+ t.start()
+ assert not done.wait(0.2), f"{op} ran while a deploy of the copy held its lock"
+ lock.release()
+ t.join(2)
+ assert done.is_set()
+ finally:
+ if lock.locked():
+ lock.release()
+
def test_the_heartbeat_never_reports_a_staged_sidecar(self):
earner = _container("cashpilot-x-p1", "x", proxy=1)
tunnel = _container("cashpilot-tunnel-x-p1", "x", proxy=1, role="tunnel")
@@ -397,7 +448,7 @@ def test_earner_failure_removes_the_sidecar(self):
with (
patch.object(orchestrator, "_get_client", return_value=client),
patch.object(proxy_tunnel, "tunnel_image", return_value="drumsergio/cashpilot-tunnel:1.64.0"),
- patch.object(proxy_tunnel, "remove", return_value=[]),
+ patch.object(proxy_tunnel, "_remove", return_value=[]),
patch.object(proxy_tunnel, "start_sidecar", return_value=(sidecar, {"ip": "93.184.215.14"})),
patch.object(orchestrator, "deploy_raw", side_effect=RuntimeError("boom")),
pytest.raises(RuntimeError),
@@ -433,7 +484,7 @@ def test_a_deploy_with_settings_that_cannot_work_keeps_the_running_copy(self):
patch.object(orchestrator, "_get_client", return_value=MagicMock()),
patch.object(proxy_tunnel, "tunnel_image", return_value="drumsergio/cashpilot-tunnel:1.64.0"),
patch.object(proxy_tunnel, "resolve_proxy_ip", side_effect=proxy_tunnel.ProxyTunnelError("no such host")),
- patch.object(proxy_tunnel, "remove") as remove,
+ patch.object(proxy_tunnel, "_remove") as remove,
pytest.raises(proxy_tunnel.ProxyTunnelError),
):
proxy_tunnel.deploy("x", 1, {"image": "x:1"}, _socks(host="gone.example"), None)
@@ -994,7 +1045,6 @@ def test_a_staged_sidecar_of_a_deploy_that_died_is_swept(self):
live.remove.assert_not_called()
def test_a_young_staged_sidecar_is_left_to_its_deploy(self):
- from datetime import UTC, datetime
young = _container("cashpilot-tunnel-x-p1-next", "x", proxy=1, role="tunnel")
young.attrs["Created"] = datetime.now(UTC).isoformat().replace("+00:00", "123Z")
From 13cd23834224e61df4813a5f3c45865163c64ed7 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 22:27:11 +0100
Subject: [PATCH 32/34] fix(proxies): let a cap that comes back hold against
starts in flight
An owner start that waited for the copy, or a reset that started capped
copies, could land after the cap stopped them again. Both recheck the
cap once they hold the copy, a reset keeps the cap's mark when the cap
returns during its start, and a cap stop holds the copy while it runs.
---
app/main.py | 88 ++++++++++++++++++++++++++-----------------
tests/test_proxies.py | 77 ++++++++++++++++++++++++++++++++++++-
2 files changed, 129 insertions(+), 36 deletions(-)
diff --git a/app/main.py b/app/main.py
index d233ccee..eb070639 100644
--- a/app/main.py
+++ b/app/main.py
@@ -4312,6 +4312,13 @@ async def api_proxied_copy_command(
if command == "start" and proxy["state"].get("capped_at"):
raise HTTPException(status_code=409, detail=proxy["state"].get("cap_reason") or "This proxy reached its cap")
async with _proxy_copy_lock(proxy_id, worker_id, slug):
+ if command == "start":
+ # Again under the lock: the cap may have stopped the copy while this waited.
+ proxy = await _proxy_or_404(proxy_id)
+ if proxy["state"].get("capped_at"):
+ raise HTTPException(
+ status_code=409, detail=proxy["state"].get("cap_reason") or "This proxy reached its cap"
+ )
result = await _proxy_worker_call(
worker_id,
"POST",
@@ -4373,11 +4380,22 @@ async def _release_capped_copies(proxy: dict[str, Any]) -> list[str]:
_proxy_cap_starting.add(key)
try:
async with _proxy_copy_lock(*key):
- # The owner may have started or stopped it since the list was read.
- if not await _copy_marked_by_cap(*key):
+ # The owner may have started or stopped it since the list was read,
+ # and a heartbeat may have capped the proxy again.
+ async with _proxy_state_lock(proxy["id"]):
+ fresh = await database.get_proxy(proxy["id"])
+ marked = await _copy_marked_by_cap(*key)
+ if not fresh or fresh["state"].get("capped_at"):
+ return started
+ if not marked:
continue
- await _start_proxied_copy(proxy, copy["worker_id"], copy["slug"])
- await database.set_proxied_copy_capped(proxy["id"], copy["worker_id"], copy["slug"], False)
+ await _start_proxied_copy(fresh, copy["worker_id"], copy["slug"])
+ # The cap may have come back while the start ran: then the mark
+ # stays, so the next heartbeat stops it and the next reset starts it.
+ async with _proxy_state_lock(proxy["id"]):
+ again = await database.get_proxy(proxy["id"])
+ if again and not again["state"].get("capped_at"):
+ await database.set_proxied_copy_capped(proxy["id"], copy["worker_id"], copy["slug"], False)
started.append(f"{copy['slug']} on worker {copy['worker_id']}")
except HTTPException as exc:
logger.warning(
@@ -4520,37 +4538,39 @@ async def _stop_capped_proxy(proxy: dict[str, Any], targets: set[tuple[int, str]
unstopped = _proxy_cap_unstopped.setdefault(pid, {})
try:
for worker_id, slug in sorted(targets):
- try:
- answer = await _proxy_worker_call(
- worker_id, "POST", f"/api/proxied/{slug}/{pid}/stop", proxy["definition"], timeout=60
- )
- unstopped.pop((worker_id, slug), None)
- # An older worker does not say; it is taken as running, as before.
- if isinstance(answer, dict) and answer.get("was_running") is False:
- continue
- # A reset or a cap change may have lifted the cap while the stop
- # was on its way; then the copy goes back up instead.
- async with _proxy_state_lock(pid):
- fresh = await database.get_proxy(pid)
- still_capped = bool(fresh and fresh["state"].get("capped_at"))
- if still_capped:
- await database.set_proxied_copy_capped(pid, worker_id, slug, True)
- if not still_capped and fresh:
- try:
- await _start_proxied_copy(fresh, worker_id, slug)
- except HTTPException as exc:
- # Marked, so the box's next heartbeat starts it.
- await database.set_proxied_copy_capped(pid, worker_id, slug, True)
- logger.warning("Could not start %s again through proxy %s: %s", slug, pid, exc.detail)
- except HTTPException as exc:
- if exc.status_code == 404 and any(
- t in str(exc.detail) for t in ("No proxied copy", "Worker not found")
- ):
- # Already gone from that box, or the box itself is gone: nothing left to stop.
+ # The owner's start or stop of this copy waits, so its mark is never overwritten.
+ async with _proxy_copy_lock(pid, worker_id, slug):
+ try:
+ answer = await _proxy_worker_call(
+ worker_id, "POST", f"/api/proxied/{slug}/{pid}/stop", proxy["definition"], timeout=60
+ )
unstopped.pop((worker_id, slug), None)
- continue
- unstopped[(worker_id, slug)] = str(exc.detail)[:200]
- logger.warning("Could not stop %s at proxy %s's cap: %s", slug, pid, exc.detail)
+ # An older worker does not say; it is taken as running, as before.
+ if isinstance(answer, dict) and answer.get("was_running") is False:
+ continue
+ # A reset or a cap change may have lifted the cap while the stop
+ # was on its way; then the copy goes back up instead.
+ async with _proxy_state_lock(pid):
+ fresh = await database.get_proxy(pid)
+ still_capped = bool(fresh and fresh["state"].get("capped_at"))
+ if still_capped:
+ await database.set_proxied_copy_capped(pid, worker_id, slug, True)
+ if not still_capped and fresh:
+ try:
+ await _start_proxied_copy(fresh, worker_id, slug)
+ except HTTPException as exc:
+ # Marked, so the box's next heartbeat starts it.
+ await database.set_proxied_copy_capped(pid, worker_id, slug, True)
+ logger.warning("Could not start %s again through proxy %s: %s", slug, pid, exc.detail)
+ except HTTPException as exc:
+ if exc.status_code == 404 and any(
+ t in str(exc.detail) for t in ("No proxied copy", "Worker not found")
+ ):
+ # Already gone from that box, or the box itself is gone: nothing left to stop.
+ unstopped.pop((worker_id, slug), None)
+ continue
+ unstopped[(worker_id, slug)] = str(exc.detail)[:200]
+ logger.warning("Could not stop %s at proxy %s's cap: %s", slug, pid, exc.detail)
finally:
_proxy_cap_stopping.difference_update((pid, *t) for t in targets)
message = await _proxy_cap_message(proxy)
diff --git a/tests/test_proxies.py b/tests/test_proxies.py
index 03b1a1c9..e64cc0ce 100644
--- a/tests/test_proxies.py
+++ b/tests/test_proxies.py
@@ -245,9 +245,14 @@ def test_a_reset_starts_only_copies_the_cap_stopped(self):
{"proxy_id": 1, "worker_id": 7, "slug": "b", "stopped_by_cap": 0},
]
call = AsyncMock(return_value={})
+ stored = dict(proxy)
+
+ async def update(_pid, state):
+ stored["state"] = state
+
with (
- patch.object(main.database, "get_proxy", new_callable=AsyncMock, return_value=proxy),
- patch.object(main.database, "update_proxy_state", new_callable=AsyncMock) as upd,
+ patch.object(main.database, "get_proxy", new_callable=AsyncMock, side_effect=lambda _pid: dict(stored)),
+ patch.object(main.database, "update_proxy_state", new_callable=AsyncMock, side_effect=update) as upd,
patch.object(main.database, "clear_alerts", new_callable=AsyncMock),
patch.object(main.database, "list_proxied_copies", new_callable=AsyncMock, return_value=copies),
patch.object(main.database, "set_proxied_copy_capped", new_callable=AsyncMock),
@@ -464,6 +469,74 @@ def run(copies, proxies_seen):
assert call.call_count == 1 and mark.call_count == 0
assert not main._proxy_cap_starting
+ def test_a_reset_respects_a_cap_that_came_back(self):
+ uncapped = {"id": 1, "definition": dict(CAPPED), "state": {}}
+ capped = {**uncapped, "state": {"capped_at": "x", "cap_reason": "r"}}
+ copies = [{"proxy_id": 1, "worker_id": 7, "slug": "a", "stopped_by_cap": 1}]
+
+ def run(proxies_seen):
+ call = AsyncMock(return_value={})
+ mark = AsyncMock()
+ with (
+ patch.object(main.database, "clear_alerts", new_callable=AsyncMock),
+ patch.object(main.database, "get_proxy", new_callable=AsyncMock, side_effect=proxies_seen),
+ patch.object(main.database, "list_proxied_copies", new_callable=AsyncMock, return_value=copies),
+ patch.object(main.database, "set_proxied_copy_capped", mark),
+ patch.object(main, "_proxy_worker_call", call),
+ ):
+ started = asyncio.run(main._release_capped_copies(uncapped))
+ return started, call, mark
+
+ # A heartbeat capped it again before the start: nothing is started.
+ started, call, mark = run([capped])
+ assert started == [] and call.call_count == 0 and mark.call_count == 0
+ # The cap came back while the start ran: the mark stays for the next stop and reset.
+ started, call, mark = run([uncapped, capped])
+ assert call.call_count == 1 and mark.call_count == 0
+ assert not main._proxy_cap_starting
+
+ def test_an_owner_start_rechecks_the_cap_once_it_holds_the_copy(self):
+ uncapped = {"id": 1, "definition": dict(CAPPED), "state": {}}
+ capped = {**uncapped, "state": {"capped_at": "x", "cap_reason": "P reached its cap"}}
+ call = AsyncMock(return_value={})
+ with (
+ patch.object(main, "_proxy_or_404", new_callable=AsyncMock, side_effect=[uncapped, capped]),
+ patch.object(main.database, "set_proxied_copy_capped", new_callable=AsyncMock) as mark,
+ patch.object(main, "_proxy_worker_call", call),
+ pytest.raises(HTTPException) as exc,
+ ):
+ asyncio.run(main.api_proxied_copy_command(1, 7, "a", "start", {}))
+ assert exc.value.status_code == 409 and exc.value.detail == "P reached its cap"
+ call.assert_not_called()
+ mark.assert_not_called()
+
+ def test_a_cap_stop_waits_for_an_owner_command_on_the_same_copy(self):
+ proxy = {"id": 1, "definition": dict(CAPPED), "state": {"capped_at": "x", "cap_reason": "r"}}
+ order: list[str] = []
+
+ async def worker_call(worker_id, method, path, definition, **kw):
+ order.append(path)
+ return {"was_running": True}
+
+ async def go():
+ lock = main._proxy_copy_lock(1, 7, "a")
+ await lock.acquire()
+ stop = asyncio.create_task(main._stop_capped_proxy(proxy, {(7, "a")}))
+ await asyncio.sleep(0.05)
+ order.append("owner done")
+ lock.release()
+ await stop
+
+ with (
+ patch.object(main.database, "get_proxy", new_callable=AsyncMock, return_value=proxy),
+ patch.object(main.database, "set_proxied_copy_capped", new_callable=AsyncMock),
+ patch.object(main.database, "record_alert", new_callable=AsyncMock, return_value=None),
+ patch.object(main, "_proxy_worker_call", AsyncMock(side_effect=worker_call)),
+ ):
+ asyncio.run(go())
+ assert order == ["owner done", "/api/proxied/a/1/stop"]
+ main._drop_proxy_cap_bell(1)
+
def test_a_reset_skips_a_copy_whose_start_is_already_on_its_way(self):
proxy = {"id": 1, "definition": dict(CAPPED), "state": {}}
call = AsyncMock(return_value={})
From eb8350bdeb02f95eec975d29e76fb4b2e7541809 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 22:27:18 +0100
Subject: [PATCH 33/34] fix: price UPT, name blocked containers, and report
image-check timeouts
UpRock's minimum is in UPT, so a balance in USD could never be compared
with it. The migration warning for a stranger in a worker container's
namespace now names that container. A timeout reading the update helper
image ends as an outage, not a bare 500.
---
app/exchange_rates.py | 2 ++
app/orchestrator.py | 2 ++
app/owner_migrate.py | 10 ++++++++--
tests/test_catalog_is_the_source_of_truth.py | 1 +
tests/test_currency_correctness.py | 2 ++
tests/test_owner_migrate.py | 15 +++++++++++++++
tests/test_update_feature.py | 15 +++++++++++++++
7 files changed, 45 insertions(+), 2 deletions(-)
diff --git a/app/exchange_rates.py b/app/exchange_rates.py
index ceeaacfe..7e4c0f84 100644
--- a/app/exchange_rates.py
+++ b/app/exchange_rates.py
@@ -35,6 +35,8 @@
# renamed from ATOR and CoinGecko kept the original slug. Taken from the
# collector, which has been querying that id for the price all along.
"ANYONE": "airtor-protocol",
+ # UpRock's token; its payout minimum is set in UPT (services/depin/uprock.yml).
+ "UPT": "uprock",
}
CACHE_TTL = 900 # 15 minutes
diff --git a/app/orchestrator.py b/app/orchestrator.py
index a07a0de7..223d8dbc 100644
--- a/app/orchestrator.py
+++ b/app/orchestrator.py
@@ -1763,6 +1763,8 @@ def _ensure_updater_image(client: docker.DockerClient) -> None:
if refusal:
raise SelfUpdateUnavailable(f"Not updating. {refusal}") from exc
logger.warning("Could not check for the update helper image: %s", exc)
+ except Exception as exc: # noqa: BLE001 - a read timeout or a dropped socket: the pull below reports it
+ logger.warning("Could not check for the update helper image: %s", type(exc).__name__)
try:
# The client's read timeout (60 s by default) ends a pull that stalls;
# a slow pull that keeps sending progress goes on.
diff --git a/app/owner_migrate.py b/app/owner_migrate.py
index 0cc2daa2..35e47d12 100644
--- a/app/owner_migrate.py
+++ b/app/owner_migrate.py
@@ -525,9 +525,15 @@ def _new_plans(self) -> list[dict[str, Any]]:
if not ours((attrs[cid].get("Config") or {}).get("Labels"))
)
if strangers:
+ mine = sorted(
+ attrs[cid]["Name"].lstrip("/")
+ for cid in cids
+ if ours((attrs[cid].get("Config") or {}).get("Labels"))
+ )
+ uses = f", which uses {', '.join(sorted(vols))}," if vols else ""
self.warn(
- f"{', '.join(strangers)} shares the network of a container on {', '.join(sorted(vols))} and was "
- "not deployed by the worker, so both were left alone. Move them by hand."
+ f"{', '.join(strangers)} shares a network namespace with {', '.join(mine) or 'a worker container'}"
+ f"{uses} and was not deployed by the worker, so both were left alone. Move them by hand."
)
continue
outsiders = self._outside_users(vols, cids)
diff --git a/tests/test_catalog_is_the_source_of_truth.py b/tests/test_catalog_is_the_source_of_truth.py
index a5ec8260..25293722 100644
--- a/tests/test_catalog_is_the_source_of_truth.py
+++ b/tests/test_catalog_is_the_source_of_truth.py
@@ -41,6 +41,7 @@ def catalogued_slugs() -> set[str]:
# on a third-party price API, not a CashPilot service, and the mapping it
# lives in is keyed by CURRENCY code.
("app/exchange_rates.py", "mysterium"),
+ ("app/exchange_rates.py", "uprock"),
}
diff --git a/tests/test_currency_correctness.py b/tests/test_currency_correctness.py
index bdb6095b..ebad34e1 100644
--- a/tests/test_currency_correctness.py
+++ b/tests/test_currency_correctness.py
@@ -139,6 +139,8 @@ def test_anyone_is_priceable(self):
assert "ANYONE" in exchange_rates.CRYPTO_IDS
assert exchange_rates.CRYPTO_IDS["ANYONE"] == "airtor-protocol"
+ # UpRock's payout minimum is in UPT, so its balance needs a UPT price to compare.
+ assert exchange_rates.CRYPTO_IDS["UPT"] == "uprock"
class TestNetEarningsSubtractLikeFromLike:
diff --git a/tests/test_owner_migrate.py b/tests/test_owner_migrate.py
index a13438a9..d1130c1c 100644
--- a/tests/test_owner_migrate.py
+++ b/tests/test_owner_migrate.py
@@ -929,3 +929,18 @@ def test_earners_the_pre_worker_deploy_path_made_are_adopted_with_the_fence(inst
again = FakeMigrator(d, om.State(tmp_path))
assert again.run([]) == 0 and again.copies == []
assert d.containers.get("cashpilot-honeygain").id == hg.id
+
+
+def test_a_stranger_in_a_direct_containers_namespace_is_named_with_it(tmp_path, caplog):
+ """No volume to name: the warning says which CashPilot container was left alone, and why."""
+ d = FakeDocker()
+ direct = {"cashpilot.managed": "true", "cashpilot.service": "honeygain", om.OWNER_LABEL: "direct"}
+ hg = d.add(Container(d, "cashpilot-honeygain", direct))
+ stranger = d.add(Container(d, "my-sidecar", {}, networks=()))
+ stranger.attrs["HostConfig"]["NetworkMode"] = f"container:{hg.id}"
+ m = FakeMigrator(d, om.State(tmp_path))
+ with caplog.at_level("WARNING", logger=om.logger.name):
+ assert m.run([]) == 0 and m.warnings == 1
+ (message,) = [r.getMessage() for r in caplog.records if "my-sidecar" in r.getMessage()]
+ assert "shares a network namespace with cashpilot-honeygain and was not deployed" in message
+ assert d.containers.get("cashpilot-honeygain") is hg
diff --git a/tests/test_update_feature.py b/tests/test_update_feature.py
index b7271927..8c5b57a5 100644
--- a/tests/test_update_feature.py
+++ b/tests/test_update_feature.py
@@ -574,6 +574,21 @@ def test_a_pull_that_stalls_is_an_outage_and_starts_nothing(self):
orchestrator.spawn_self_update()
client.containers.run.assert_not_called()
+ def test_an_image_check_that_times_out_is_an_outage_not_a_crash(self):
+ import requests
+
+ me = _fake_container(labels=_COMPOSE_LABELS)
+ client = _fake_docker(me=me)
+ _wire_runs(client)
+ client.images.get.side_effect = requests.exceptions.ReadTimeout("read timed out")
+ client.images.pull.side_effect = requests.exceptions.ConnectionError("connection reset")
+ with (
+ patch.object(orchestrator, "_get_client", return_value=client),
+ pytest.raises(RuntimeError, match="could not pull the update helper image"),
+ ):
+ orchestrator.spawn_self_update()
+ client.containers.run.assert_not_called()
+
def test_a_pull_the_proxy_refuses_says_why_and_starts_nothing(self):
from docker.errors import APIError
From af2f725cadce7184fb153104880194d64b790627 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sat, 7 Nov 2026 22:27:25 +0100
Subject: [PATCH 34/34] docs: say where the LAN fence and the proxy's bind
refusal stop
Podman gets no fence, and the proxy refuses binds of / or the socket
only while the operator's bind lists leave them out.
---
CLAUDE.md | 2 +-
docs/faq.md | 2 +-
docs/security-defaults.md | 3 ++-
tests/test_docs_say_the_socket_proxy_holds_the_socket.py | 8 ++++++++
4 files changed, 12 insertions(+), 3 deletions(-)
diff --git a/CLAUDE.md b/CLAUDE.md
index d4c0e310..85abdcbc 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -125,7 +125,7 @@ Two separate Docker images:
### Core Principles
1. **Separation of concerns.** UI handles: dashboard, earnings collection, scheduling, user auth. Workers handle: Docker container lifecycle, health reporting.
-2. **Workers need Docker, through the socket proxy.** A worker without Docker access is useless, but in the shipped compose files it never gets the raw socket: the proxy refuses privileged containers and binds of `/` or the socket, and limits the worker to containers, volumes and networks with its owner label. The Unraid and TrueNAS listings still give it the raw socket, which is root on the host (`docs/app-stores.md`).
+2. **Workers need Docker, through the socket proxy.** A worker without Docker access is useless, but in the shipped compose files it never gets the raw socket: the proxy refuses privileged containers and binds of `/` or the socket (unless the operator lists one of them in `CASHPILOT_PROXY_BIND_MOUNTS` or `CASHPILOT_COMPOSE_DIR`, which it takes as given), and limits the worker to containers, volumes and networks with its owner label. The Unraid and TrueNAS listings still give it the raw socket, which is root on the host (`docs/app-stores.md`).
3. **Single source of truth.** The UI instance is the only one that collects earnings and stores historical data.
4. **Earnings are never duplicated.** Since only the UI collects, there is no risk of double-counting.
5. **Workers are stateless satellites.** A worker knows which containers to keep running and the UI URL to report to.
diff --git a/docs/faq.md b/docs/faq.md
index 07763a20..4226b6e3 100644
--- a/docs/faq.md
+++ b/docs/faq.md
@@ -38,7 +38,7 @@ If CashPilot cannot write the key to disk at all, because `/data` is unwritable
**What about security?**
-Every service CashPilot deploys runs in its own container with every Linux capability dropped (a few services add back only the ones they declare) and `--security-opt no-new-privileges` set, so a process inside cannot gain privileges. Containers cannot see your host filesystem beyond the volumes a service declares. On a new install the worker also fences them off your LAN, the host, your tailnet and cloud metadata, so they reach the internet and nothing local ([LAN isolation](isolation.md)). An install upgraded from a release before 1.70 keeps its earners on Docker's default bridge, which can reach your LAN, until you click **Turn on isolation** on the Fleet page. The fence does not cover a service on host networking (Mysterium, today); [the isolation page](isolation.md) says what to do about those, and [Protecting Your Home Network](home-network-security.md) covers firewalling your own bridge instead. Service credentials are encrypted at rest using Fernet symmetric encryption. The UI has no Docker access. With the shipped compose files the worker is never given the raw Docker socket: it reaches Docker through `cashpilot-socket-proxy`, which refuses privileged containers and binds of `/` or the socket, and limits the worker to containers, volumes and networks with its owner label ([security defaults](security-defaults.md)). The Unraid templates and the TrueNAS app still mount the raw socket in the worker, which is root on the host ([app stores](app-stores.md)).
+Every service CashPilot deploys runs in its own container with every Linux capability dropped (a few services add back only the ones they declare) and `--security-opt no-new-privileges` set, so a process inside cannot gain privileges. Containers cannot see your host filesystem beyond the volumes a service declares. On a new install the worker also fences them off your LAN, the host, your tailnet and cloud metadata, so they reach the internet and nothing local ([LAN isolation](isolation.md)). On Podman the fence is not available: earners run on Podman's default network, which can reach your LAN, and the Fleet page says so ([LAN isolation](isolation.md) lists the options). An install upgraded from a release before 1.70 keeps its earners on Docker's default bridge, which can reach your LAN, until you click **Turn on isolation** on the Fleet page. The fence does not cover a service on host networking (Mysterium, today); [the isolation page](isolation.md) says what to do about those, and [Protecting Your Home Network](home-network-security.md) covers firewalling your own bridge instead. Service credentials are encrypted at rest using Fernet symmetric encryption. The UI has no Docker access. With the shipped compose files the worker is never given the raw Docker socket: it reaches Docker through `cashpilot-socket-proxy`, which refuses privileged containers and binds of `/` or the socket (unless you put one of them, or a folder above the socket, in `CASHPILOT_PROXY_BIND_MOUNTS` or `CASHPILOT_COMPOSE_DIR`: the proxy takes those as given), and limits the worker to containers, volumes and networks with its owner label ([security defaults](security-defaults.md)). The Unraid templates and the TrueNAS app still mount the raw socket in the worker, which is root on the host ([app stores](app-stores.md)).
That said, no setup is bulletproof. You are still running third-party software that routes external traffic through your network. Docker isolation significantly reduces the attack surface compared to running these services directly on your host, but it does not eliminate all risk. We recommend running CashPilot on a dedicated machine or VLAN, keeping Docker and your host OS up to date, and reviewing the open-source code of any service before deploying it.
diff --git a/docs/security-defaults.md b/docs/security-defaults.md
index b29fbb94..019d7b83 100644
--- a/docs/security-defaults.md
+++ b/docs/security-defaults.md
@@ -124,7 +124,8 @@ dangerous containers, not only dangerous URLs:
only `runc` and the gVisor ones (the worker refuses any other runtime itself,
before the running container is touched). Host folders only from
`CASHPILOT_PROXY_BIND_MOUNTS` and `CASHPILOT_COMPOSE_DIR`, so a bind of `/`
- or of the socket is refused. The body is parsed the way Docker parses it, so
+ or of the socket is refused as long as neither variable lists `/`, the
+ socket or a folder above it: the proxy takes both as given. The body is parsed the way Docker parses it, so
`"privileged": true` in lowercase is refused too.
- **Exec.** Only the exact commands the catalog's readers run (Mysterium's node
and wallet reads, Anyone Protocol's `anonrc`, the isolation check's firewall
diff --git a/tests/test_docs_say_the_socket_proxy_holds_the_socket.py b/tests/test_docs_say_the_socket_proxy_holds_the_socket.py
index d636aa99..1b12979a 100644
--- a/tests/test_docs_say_the_socket_proxy_holds_the_socket.py
+++ b/tests/test_docs_say_the_socket_proxy_holds_the_socket.py
@@ -40,6 +40,8 @@
# worker mounts the socket); the page says so.
("app-stores.md", STALE[0]),
("app-stores.md", STALE[1]),
+ ("app-stores.md", STALE[5]),
+ ("app-stores.md", STALE[6]),
# A proxied copy really is two containers (the earner and its tunnel).
("proxies.md", STALE[0]),
}
@@ -76,3 +78,9 @@ def test_no_page_says_the_worker_holds_the_socket(page: Path):
)
def test_the_check_catches_each_old_claim(text: str):
assert stale_claims("index.md", text)
+
+
+def test_the_app_store_page_may_say_its_listings_need_the_socket():
+ text = "The Unraid worker requires Docker socket access, which is root on the host."
+ assert stale_claims("app-stores.md", text) == []
+ assert stale_claims("index.md", text)