feat(fleet): one click to move a host-network Mysterium behind the fence

UPGRADING.md

@@ -29,11 +29,12 @@ releases run it in its own network namespace on `cashpilot-isolated`, behind
 the fence, and the worker starts it only once the fence is in place
 ([Mysterium on the fenced bridge](docs/isolation.md#mysterium-on-the-fenced-bridge)).
 
-A running node keeps working exactly as it is until you redeploy it, in any
-isolation mode: the upgraded worker does not stop, start or change it. The
-dashboard lists it under drift ("shares the host's network, where its
-NET_ADMIN reaches the host's firewall. Switch isolation on in Fleet, then
-redeploy Mysterium once"), and **Check isolation** fails on it.
+A running node keeps running until you redeploy it, in any isolation mode:
+the upgraded worker does not stop, start or change it. Redeploy it once to
+move it behind the fence. Until then the Fleet page shows it under its worker
+with a **Redeploy MystNodes to move behind the fence** button (or, with
+isolation off, says to switch it on first), the dashboard lists it under
+drift, and **Check isolation** fails on it.
 
 **What to do, in this order.**
 

app/lan_fence.py

@@ -529,6 +529,7 @@ class IsolationConflict(RuntimeError):
     "earners_outside_stopped": [],
     # Fence-gated containers the guard holds stopped, by name, with the reason.
     "guard": {},
+    "unfenced": [],
 }
 #: The worker's public egress IP, set from the heartbeat's detection
 #: (app/worker_api.py). The next apply adds its drop.
@@ -1125,6 +1126,22 @@ def _gated(client: Any) -> list[Any]:
     ]
 
 
+def _unfenced(client: Any) -> list[dict[str, str]]:
+    """Fence-gated services' containers still on the host's network (2.0.6 or earlier).
+
+    Left running as they are; the Fleet page offers the redeploy that moves them.
+    """
+    orch = _orchestrator()
+    from app.constants import LABEL_SERVICE
+
+    found = []
+    for c in _managed(client):
+        slug = str((c.labels or {}).get(LABEL_SERVICE) or "")
+        if orch.fence_gated(slug) and orch.predates_fence(c):
+            found.append({"name": c.name, "slug": slug, "service": orch.service_name(slug)})
+    return found
+
+
 def guard() -> dict[str, str]:
     """Every GUARD_SECONDS: hold each fence-gated container to the fence. Never raises.
 
@@ -1143,8 +1160,10 @@ def guard() -> dict[str, str]:
     held: dict[str, str] = {}
     try:
         if not orch.docker_available():
+            _state["unfenced"] = []
             return {}
         client = orch._get_client()
+        _state["unfenced"] = _unfenced(client)
         gated = _gated(client)
         if not gated:
             _state["guard"] = {}
@@ -1186,6 +1205,8 @@ def guard() -> dict[str, str]:
                 orch.GATED_DEPLOY_LOCK.release()
     except Exception as exc:  # noqa: BLE001 - the next pass tries again
         logger.warning("Fence guard pass failed: %s", exc)
+        # Not a fresh scan: no Fleet button on a report that may be stale.
+        _state["unfenced"] = []
     _state["guard"] = held
     return held
 
@@ -1348,6 +1369,7 @@ def status() -> dict[str, Any]:
         "earners_outside": list(_state["earners_outside"]),
         "earners_outside_stopped": list(_state.get("earners_outside_stopped") or []),
         "guard": dict(_state.get("guard") or {}),
+        "unfenced": [dict(u) for u in _state.get("unfenced") or []],
     }
 
 

app/orchestrator.py

@@ -893,13 +893,25 @@ def deploy_raw(
         # chance to flush (storj declares 300 s for that reason) and pays for it
         # with an unclean-shutdown recovery. A stop that fails is not a reason to
         # abandon the deploy - the forced removal below is what ran before.
+        if gated:
+            # A deploy is the owner's way out of a hold, written down or not.
+            _MISMATCH_HELD.pop(name, None)
         if gated and not _pending_identity(name):
             # Read while it still runs; the guard compares the new one with it.
             # A redeploy over a stopped node clears the record instead. A record
             # still pending is kept: the container being replaced is itself a
             # replacement whose identity was never confirmed, so this retry is
             # compared with the node as it was before the first redeploy.
-            identity = read_node_identity(old, slug) if getattr(old, "status", "") == "running" else None
+            running = getattr(old, "status", "") == "running"
+            identity = read_node_identity(old, slug) if running else None
+            if running and identity is None and _reads_identity(slug):
+                # Without it the new node could run on another keystore unchecked.
+                raise ContainerRuntimeError(
+                    f"{name} was not replaced: its identity could not be read from the running node, so the new "
+                    "one could not be checked against it. The running container was not touched. Try again in a "
+                    "minute. If the node is broken, stop it from the dashboard first: a stopped node is replaced "
+                    "without the check."
+                )
             _remember_identity(name, identity)
         try:
             old.stop(timeout=_get_stop_timeout(slug))
@@ -1254,10 +1266,23 @@ def read_node_identity(container: Any, slug: str) -> str | None:
     return match.group(1) if match else None
 
 
+def _reads_identity(slug: str) -> bool:
+    """Whether the catalog says how to read this service's node identity (collector.node_read)."""
+    svc = (get_service(slug) if get_service else None) or {}
+    spec = (svc.get("collector") or {}).get("node_read") or {}
+    return isinstance(spec.get("command"), list) and bool(spec.get("command")) and isinstance(spec.get("node"), str)
+
+
 def _identity_file(name: str) -> str:
     return os.path.join(IDENTITY_DIR, re.sub(r"[^A-Za-z0-9_.-]", "_", name))
 
 
+#: Mismatch holds by container name, kept in memory as well as on disk: a hold
+#: whose record could not be written must still hold, or the stopped node
+#: would read as unchecked and the guard would start it again every pass.
+_MISMATCH_HELD: dict[str, str] = {}
+
+
 def _pending_identity(name: str) -> str | None:
     """The identity a replacement still has to match, or None (no record, or a held mismatch)."""
     try:
@@ -1269,6 +1294,9 @@ def _pending_identity(name: str) -> str | None:
 
 
 def _remember_identity(name: str, identity: str | None) -> None:
+    if not str(identity or "").startswith("MISMATCH "):
+        # A deploy rewrites the record: the owner's way out of a hold.
+        _MISMATCH_HELD.pop(name, None)
     path = _identity_file(name)
     try:
         if identity:
@@ -1287,6 +1315,8 @@ def identity_problem(container: Any) -> str | None:
     None while there is nothing to compare (no record, or the node not
     answering yet) and once it matches, when the record is deleted.
     """
+    if held_reason := _MISMATCH_HELD.get(container.name):
+        return held_reason
     path = _identity_file(container.name)
     try:
         with open(path, encoding="utf-8") as fh:
@@ -1312,6 +1342,7 @@ def identity_problem(container: Any) -> str | None:
         f"its identity is {current}, not {expected} as before the redeploy, so it is not running on the data "
         "volume that holds its keystore. It stays stopped: restore that volume, then redeploy"
     )
+    _MISMATCH_HELD[container.name] = reason
     _remember_identity(container.name, f"MISMATCH {reason}")
     return reason
 

app/templates/fleet.html

@@ -460,6 +460,7 @@ <h2 class="block-title" id="proxies-title">Proxies</h2>
 
   // Turning isolation on, per worker id: {busy}, {done}, or {error}.
   const _fenceSwitch = {};
+  const _movedBehindFence = new Set();
 
   // The LAN fence of one worker (app/lan_fence.py), from its heartbeat. Says
   // nothing when the fence holds and every earner sits behind it.
@@ -498,13 +499,54 @@ <h2 class="block-title" id="proxies-title">Proxies</h2>
     if (held.length) {
       line += `${line ? '<div class="iso-line">' : ''}<span class="text-fail">Held stopped: ${held.map(([n, why]) => `${esc(n)} (${esc(why)})`).join('; ')}. A missing LAN fence clears by itself; any other reason needs a redeploy from the dashboard.</span>${line ? '</div>' : ''}`;
     }
+    // A Mysterium from 2.0.6 or earlier, still on the host's network: left
+    // running until its owner redeploys it, which moves it behind the fence.
+    // A node moved from this page stays hidden until the worker stops reporting
+    // it: a heartbeat sent before the move must not offer the move again.
+    for (const key of [..._movedBehindFence]) {
+      if (key.startsWith(`${w.id}:`) && !(iso.unfenced || []).some((u) => key === `${w.id}:${u.slug}`)) _movedBehindFence.delete(key);
+    }
+    const unfenced = (iso.unfenced || []).filter((u) => !_movedBehindFence.has(`${w.id}:${u.slug}`));
+    if (unfenced.length) {
+      const fenced = iso.mode === 'on' || iso.mode === 'operator';
+      const what = unfenced.map((u) => esc(u.service || u.slug)).join(', ');
+      const order = fenced
+        ? 'It keeps running as it is until you redeploy it once, which moves it behind the fence and keeps its identity.'
+        : iso.mode === 'unsupported'
+          ? 'It keeps running as it is. Podman cannot be fenced: run it on Docker with isolation on, or on your own network (CASHPILOT_CONTAINER_NETWORK).'
+          : iso.mode === 'off'
+            ? 'It keeps running as it is. Set CASHPILOT_ISOLATION=auto on this worker and restart it, then redeploy it once to move it behind the fence.'
+            : 'It keeps running as it is. Switch isolation on first, then redeploy it once to move it behind the fence.';
+      const redeploy = _isOwner && fenced && !(sw && sw.busy)
+        ? unfenced.map((u) => ` <button class="btn btn-primary btn-sm btn-unfenced-redeploy" data-worker-id="${w.id}" data-slug="${esc(u.slug)}" data-service="${esc(u.service || u.slug)}">Redeploy ${esc(u.service || u.slug)} to move behind the fence</button>`).join('')
+        : '';
+      line += `${line ? '<div class="iso-line">' : ''}<span class="text-fail">${what} still shares this host's network, where its NET_ADMIN can rewrite the host's firewall. ${order}</span>${redeploy}${line ? '</div>' : ''}`;
+    }
     if (sw) {
       const tone = sw.error ? 'text-fail' : sw.busy ? 'muted' : 'text-ok';
       line += `<div class="iso-line ${tone}">${esc(sw.error || sw.busy || sw.done)}</div>`;
     }
     return line ? `<tr class="worker-detail"><td colspan="7"><div class="iso-report">${line} <a href="https://geiserx.github.io/CashPilot/isolation/#lan-isolation-by-default" target="_blank" rel="noopener noreferrer">How it works</a></div></td></tr>` : '';
   }
 
+  async function moveBehindFence(workerId, slug, service) {
+    if (!confirm(`Redeploy ${service} to move it behind the LAN fence?\n\n`
+      + '- The worker reads the node\'s identity, replaces the container on the fenced network and checks the new one has the same identity.\n'
+      + '- The data volume is kept. If the identity differs, the node is held stopped and the volume is left as it is.\n'
+      + '- The node is down for under a minute.')) return;
+    _fenceSwitch[workerId] = { busy: `Moving ${service} behind the fence…` };
+    loadFleet();
+    try {
+      // An empty body redeploys from the recorded spec; a fence-gated service lands on the fenced bridge.
+      await CP.api(`/api/deploy/${encodeURIComponent(slug)}?worker_id=${workerId}`, { method: 'POST', body: {} });
+      _movedBehindFence.add(`${workerId}:${slug}`);
+      _fenceSwitch[workerId] = { done: `${service} was redeployed behind the fence.` };
+    } catch (err) {
+      _fenceSwitch[workerId] = { error: `${service} was not moved: ${err.message}` };
+    }
+    loadFleet();
+  }
+
   async function enableFence(workerId, name) {
     if (!confirm(`Turn on LAN isolation on "${name}"?\n\n`
       + '- The worker creates a Docker network (cashpilot-isolated) and one firewall table (ip cashpilot) on its host.\n'
@@ -843,6 +885,13 @@ <h2 class="block-title" id="proxies-title">Proxies</h2>
       return;
     }
 
+    const unfencedBtn = e.target.closest('.btn-unfenced-redeploy');
+    if (unfencedBtn) {
+      const id = parseInt(unfencedBtn.dataset.workerId, 10);
+      if (!isNaN(id) && !(_fenceSwitch[id] && _fenceSwitch[id].busy)) moveBehindFence(id, unfencedBtn.dataset.slug, unfencedBtn.dataset.service);
+      return;
+    }
+
     const isoBtn = e.target.closest('.btn-isolation-check');
     if (isoBtn) {
       const id = parseInt(isoBtn.dataset.workerId, 10);

tests/test_lan_fence.py

@@ -1291,6 +1291,7 @@ def test_the_worker_image_ships_the_fence():
 const esc = s => String(s).replace(/[&<>"']/g, c => ({'&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;'}[c]));
 const _isOwner = true;
 const _fenceSwitch = {};
+const _movedBehindFence = new Set();
 const calls = [];
 const loadFleet = () => {};
 const confirm = () => true;