diff --git a/UPGRADING.md b/UPGRADING.md
index 0769805b..d4ca2059 100644
--- a/UPGRADING.md
+++ b/UPGRADING.md
@@ -29,11 +29,12 @@ releases run it in its own network namespace on `cashpilot-isolated`, behind
the fence, and the worker starts it only once the fence is in place
([Mysterium on the fenced bridge](docs/isolation.md#mysterium-on-the-fenced-bridge)).
-A running node keeps working exactly as it is until you redeploy it, in any
-isolation mode: the upgraded worker does not stop, start or change it. The
-dashboard lists it under drift ("shares the host's network, where its
-NET_ADMIN reaches the host's firewall. Switch isolation on in Fleet, then
-redeploy Mysterium once"), and **Check isolation** fails on it.
+A running node keeps running until you redeploy it, in any isolation mode:
+the upgraded worker does not stop, start or change it. Redeploy it once to
+move it behind the fence. Until then the Fleet page shows it under its worker
+with a **Redeploy MystNodes to move behind the fence** button (or, with
+isolation off, says to switch it on first), the dashboard lists it under
+drift, and **Check isolation** fails on it.
**What to do, in this order.**
diff --git a/app/lan_fence.py b/app/lan_fence.py
index b43ecfca..0f26f846 100644
--- a/app/lan_fence.py
+++ b/app/lan_fence.py
@@ -529,6 +529,7 @@ class IsolationConflict(RuntimeError):
"earners_outside_stopped": [],
# Fence-gated containers the guard holds stopped, by name, with the reason.
"guard": {},
+ "unfenced": [],
}
#: The worker's public egress IP, set from the heartbeat's detection
#: (app/worker_api.py). The next apply adds its drop.
@@ -1125,6 +1126,22 @@ def _gated(client: Any) -> list[Any]:
]
+def _unfenced(client: Any) -> list[dict[str, str]]:
+ """Fence-gated services' containers still on the host's network (2.0.6 or earlier).
+
+ Left running as they are; the Fleet page offers the redeploy that moves them.
+ """
+ orch = _orchestrator()
+ from app.constants import LABEL_SERVICE
+
+ found = []
+ for c in _managed(client):
+ slug = str((c.labels or {}).get(LABEL_SERVICE) or "")
+ if orch.fence_gated(slug) and orch.predates_fence(c):
+ found.append({"name": c.name, "slug": slug, "service": orch.service_name(slug)})
+ return found
+
+
def guard() -> dict[str, str]:
"""Every GUARD_SECONDS: hold each fence-gated container to the fence. Never raises.
@@ -1143,8 +1160,10 @@ def guard() -> dict[str, str]:
held: dict[str, str] = {}
try:
if not orch.docker_available():
+ _state["unfenced"] = []
return {}
client = orch._get_client()
+ _state["unfenced"] = _unfenced(client)
gated = _gated(client)
if not gated:
_state["guard"] = {}
@@ -1186,6 +1205,8 @@ def guard() -> dict[str, str]:
orch.GATED_DEPLOY_LOCK.release()
except Exception as exc: # noqa: BLE001 - the next pass tries again
logger.warning("Fence guard pass failed: %s", exc)
+ # Not a fresh scan: no Fleet button on a report that may be stale.
+ _state["unfenced"] = []
_state["guard"] = held
return held
@@ -1348,6 +1369,7 @@ def status() -> dict[str, Any]:
"earners_outside": list(_state["earners_outside"]),
"earners_outside_stopped": list(_state.get("earners_outside_stopped") or []),
"guard": dict(_state.get("guard") or {}),
+ "unfenced": [dict(u) for u in _state.get("unfenced") or []],
}
diff --git a/app/orchestrator.py b/app/orchestrator.py
index e5e786ba..5087fbeb 100644
--- a/app/orchestrator.py
+++ b/app/orchestrator.py
@@ -893,13 +893,25 @@ def deploy_raw(
# chance to flush (storj declares 300 s for that reason) and pays for it
# with an unclean-shutdown recovery. A stop that fails is not a reason to
# abandon the deploy - the forced removal below is what ran before.
+ if gated:
+ # A deploy is the owner's way out of a hold, written down or not.
+ _MISMATCH_HELD.pop(name, None)
if gated and not _pending_identity(name):
# Read while it still runs; the guard compares the new one with it.
# A redeploy over a stopped node clears the record instead. A record
# still pending is kept: the container being replaced is itself a
# replacement whose identity was never confirmed, so this retry is
# compared with the node as it was before the first redeploy.
- identity = read_node_identity(old, slug) if getattr(old, "status", "") == "running" else None
+ running = getattr(old, "status", "") == "running"
+ identity = read_node_identity(old, slug) if running else None
+ if running and identity is None and _reads_identity(slug):
+ # Without it the new node could run on another keystore unchecked.
+ raise ContainerRuntimeError(
+ f"{name} was not replaced: its identity could not be read from the running node, so the new "
+ "one could not be checked against it. The running container was not touched. Try again in a "
+ "minute. If the node is broken, stop it from the dashboard first: a stopped node is replaced "
+ "without the check."
+ )
_remember_identity(name, identity)
try:
old.stop(timeout=_get_stop_timeout(slug))
@@ -1254,10 +1266,23 @@ def read_node_identity(container: Any, slug: str) -> str | None:
return match.group(1) if match else None
+def _reads_identity(slug: str) -> bool:
+ """Whether the catalog says how to read this service's node identity (collector.node_read)."""
+ svc = (get_service(slug) if get_service else None) or {}
+ spec = (svc.get("collector") or {}).get("node_read") or {}
+ return isinstance(spec.get("command"), list) and bool(spec.get("command")) and isinstance(spec.get("node"), str)
+
+
def _identity_file(name: str) -> str:
return os.path.join(IDENTITY_DIR, re.sub(r"[^A-Za-z0-9_.-]", "_", name))
+#: Mismatch holds by container name, kept in memory as well as on disk: a hold
+#: whose record could not be written must still hold, or the stopped node
+#: would read as unchecked and the guard would start it again every pass.
+_MISMATCH_HELD: dict[str, str] = {}
+
+
def _pending_identity(name: str) -> str | None:
"""The identity a replacement still has to match, or None (no record, or a held mismatch)."""
try:
@@ -1269,6 +1294,9 @@ def _pending_identity(name: str) -> str | None:
def _remember_identity(name: str, identity: str | None) -> None:
+ if not str(identity or "").startswith("MISMATCH "):
+ # A deploy rewrites the record: the owner's way out of a hold.
+ _MISMATCH_HELD.pop(name, None)
path = _identity_file(name)
try:
if identity:
@@ -1287,6 +1315,8 @@ def identity_problem(container: Any) -> str | None:
None while there is nothing to compare (no record, or the node not
answering yet) and once it matches, when the record is deleted.
"""
+ if held_reason := _MISMATCH_HELD.get(container.name):
+ return held_reason
path = _identity_file(container.name)
try:
with open(path, encoding="utf-8") as fh:
@@ -1312,6 +1342,7 @@ def identity_problem(container: Any) -> str | None:
f"its identity is {current}, not {expected} as before the redeploy, so it is not running on the data "
"volume that holds its keystore. It stays stopped: restore that volume, then redeploy"
)
+ _MISMATCH_HELD[container.name] = reason
_remember_identity(container.name, f"MISMATCH {reason}")
return reason
diff --git a/app/templates/fleet.html b/app/templates/fleet.html
index 5fe8d7cc..d73cef54 100644
--- a/app/templates/fleet.html
+++ b/app/templates/fleet.html
@@ -460,6 +460,7 @@
Proxies
// Turning isolation on, per worker id: {busy}, {done}, or {error}.
const _fenceSwitch = {};
+ const _movedBehindFence = new Set();
// The LAN fence of one worker (app/lan_fence.py), from its heartbeat. Says
// nothing when the fence holds and every earner sits behind it.
@@ -498,6 +499,29 @@
Proxies
if (held.length) {
line += `${line ? '
' : ''}Held stopped: ${held.map(([n, why]) => `${esc(n)} (${esc(why)})`).join('; ')}. A missing LAN fence clears by itself; any other reason needs a redeploy from the dashboard.${line ? '
' : ''}`;
}
+ // A Mysterium from 2.0.6 or earlier, still on the host's network: left
+ // running until its owner redeploys it, which moves it behind the fence.
+ // A node moved from this page stays hidden until the worker stops reporting
+ // it: a heartbeat sent before the move must not offer the move again.
+ for (const key of [..._movedBehindFence]) {
+ if (key.startsWith(`${w.id}:`) && !(iso.unfenced || []).some((u) => key === `${w.id}:${u.slug}`)) _movedBehindFence.delete(key);
+ }
+ const unfenced = (iso.unfenced || []).filter((u) => !_movedBehindFence.has(`${w.id}:${u.slug}`));
+ if (unfenced.length) {
+ const fenced = iso.mode === 'on' || iso.mode === 'operator';
+ const what = unfenced.map((u) => esc(u.service || u.slug)).join(', ');
+ const order = fenced
+ ? 'It keeps running as it is until you redeploy it once, which moves it behind the fence and keeps its identity.'
+ : iso.mode === 'unsupported'
+ ? 'It keeps running as it is. Podman cannot be fenced: run it on Docker with isolation on, or on your own network (CASHPILOT_CONTAINER_NETWORK).'
+ : iso.mode === 'off'
+ ? 'It keeps running as it is. Set CASHPILOT_ISOLATION=auto on this worker and restart it, then redeploy it once to move it behind the fence.'
+ : 'It keeps running as it is. Switch isolation on first, then redeploy it once to move it behind the fence.';
+ const redeploy = _isOwner && fenced && !(sw && sw.busy)
+ ? unfenced.map((u) => ` `).join('')
+ : '';
+ line += `${line ? '
' : ''}${what} still shares this host's network, where its NET_ADMIN can rewrite the host's firewall. ${order}${redeploy}${line ? '
' : ''}`;
+ }
if (sw) {
const tone = sw.error ? 'text-fail' : sw.busy ? 'muted' : 'text-ok';
line += `
` : '';
}
+ async function moveBehindFence(workerId, slug, service) {
+ if (!confirm(`Redeploy ${service} to move it behind the LAN fence?\n\n`
+ + '- The worker reads the node\'s identity, replaces the container on the fenced network and checks the new one has the same identity.\n'
+ + '- The data volume is kept. If the identity differs, the node is held stopped and the volume is left as it is.\n'
+ + '- The node is down for under a minute.')) return;
+ _fenceSwitch[workerId] = { busy: `Moving ${service} behind the fence…` };
+ loadFleet();
+ try {
+ // An empty body redeploys from the recorded spec; a fence-gated service lands on the fenced bridge.
+ await CP.api(`/api/deploy/${encodeURIComponent(slug)}?worker_id=${workerId}`, { method: 'POST', body: {} });
+ _movedBehindFence.add(`${workerId}:${slug}`);
+ _fenceSwitch[workerId] = { done: `${service} was redeployed behind the fence.` };
+ } catch (err) {
+ _fenceSwitch[workerId] = { error: `${service} was not moved: ${err.message}` };
+ }
+ loadFleet();
+ }
+
async function enableFence(workerId, name) {
if (!confirm(`Turn on LAN isolation on "${name}"?\n\n`
+ '- The worker creates a Docker network (cashpilot-isolated) and one firewall table (ip cashpilot) on its host.\n'
@@ -843,6 +885,13 @@
Proxies
return;
}
+ const unfencedBtn = e.target.closest('.btn-unfenced-redeploy');
+ if (unfencedBtn) {
+ const id = parseInt(unfencedBtn.dataset.workerId, 10);
+ if (!isNaN(id) && !(_fenceSwitch[id] && _fenceSwitch[id].busy)) moveBehindFence(id, unfencedBtn.dataset.slug, unfencedBtn.dataset.service);
+ return;
+ }
+
const isoBtn = e.target.closest('.btn-isolation-check');
if (isoBtn) {
const id = parseInt(isoBtn.dataset.workerId, 10);
diff --git a/tests/test_lan_fence.py b/tests/test_lan_fence.py
index 11c706d3..f1a20e25 100644
--- a/tests/test_lan_fence.py
+++ b/tests/test_lan_fence.py
@@ -1291,6 +1291,7 @@ def test_the_worker_image_ships_the_fence():
const esc = s => String(s).replace(/[&<>"']/g, c => ({'&': '&', '<': '<', '>': '>', '"': '"', "'": '''}[c]));
const _isOwner = true;
const _fenceSwitch = {};
+const _movedBehindFence = new Set();
const calls = [];
const loadFleet = () => {};
const confirm = () => true;
diff --git a/tests/test_mysterium_fenced.py b/tests/test_mysterium_fenced.py
index faf06ec7..2fb3ab1a 100644
--- a/tests/test_mysterium_fenced.py
+++ b/tests/test_mysterium_fenced.py
@@ -271,6 +271,7 @@ def test_control_the_catalog_spec_passes(self, real_catalog):
def markers(tmp_path, monkeypatch):
monkeypatch.setattr(orchestrator, "STOPPED_MARKER", str(tmp_path / "stopped"))
monkeypatch.setattr(orchestrator, "IDENTITY_DIR", str(tmp_path / "identity"))
+ monkeypatch.setattr(orchestrator, "_MISMATCH_HELD", {}, raising=False)
return tmp_path
@@ -981,3 +982,171 @@ def svc(default):
assert lan_isolation.published_ports(svc("127.0.0.2")) == []
assert lan_isolation.published_ports(svc("::1,localhost")) == []
assert lan_isolation.published_ports(svc("192.168.1.5")) == ["192.168.1.5:4449"]
+
+
+# ---------------------------------------------------------------------------
+# (m) The upgrade: the old node keeps earning, the Fleet page offers the move,
+# and the move compares the identity with the node it replaces
+# ---------------------------------------------------------------------------
+
+IDENTITY_A = "0x" + "a" * 40
+IDENTITY_B = "0x" + "b" * 40
+
+
+def _says(identity):
+ return (0, b'{"identities":[{"id":"' + identity.encode() + b'"}]}')
+
+
+class TestTheUpgrade:
+ @pytest.mark.parametrize("mode", [lan_fence.MODE_ON, lan_fence.MODE_LEGACY])
+ def test_the_old_node_keeps_running_and_the_fleet_page_is_told(self, markers, mode):
+ node = _old_node()
+ problem = None if mode == lan_fence.MODE_ON else lan_fence.mode_problem(mode)
+ _guard([node], problem=problem, mode=mode)
+ node.stop.assert_not_called()
+ node.restart.assert_not_called()
+ node.remove.assert_not_called()
+ assert lan_fence.status()["unfenced"] == [
+ {"name": "cashpilot-mysterium", "slug": "mysterium", "service": MYST["name"]}
+ ]
+
+ def test_control_a_fenced_node_is_not_reported_as_unfenced(self, markers):
+ _guard([_node()])
+ assert lan_fence.status()["unfenced"] == []
+
+ def _old_with_data(self, identity):
+ old = _old_node()
+ old.attrs["Mounts"] = [
+ {"Type": "bind", "Source": "/mnt/appdata/myst/data", "Destination": "/var/lib/mysterium-node", "RW": True}
+ ]
+ old.exec_run.return_value = _says(identity)
+ return old
+
+ def test_the_redeploy_reads_the_running_old_nodes_identity_before_replacing_it(self, markers):
+ old = self._old_with_data(IDENTITY_A)
+ order = []
+ old.exec_run.side_effect = lambda *a, **k: order.append("read") or _says(IDENTITY_A)
+ old.stop.side_effect = lambda **_: order.append("stop")
+ old.remove.side_effect = lambda **_: order.append("remove")
+ client, _ = _raw_deploy(old=old, volumes={"mysterium-data": {"bind": "/var/lib/mysterium-node", "mode": "rw"}})
+ assert order[0] == "read" and "stop" in order
+ assert orchestrator._pending_identity("cashpilot-mysterium") == IDENTITY_A
+ volumes = client.containers.create.call_args.kwargs["volumes"]
+ assert "/mnt/appdata/myst/data" in str(volumes), "the node's own data must come with it"
+
+ def test_a_different_identity_after_the_move_holds_it_and_keeps_the_volume(self, markers):
+ _raw_deploy(old=self._old_with_data(IDENTITY_A))
+ new = _node()
+ new.exec_run.return_value = _says(IDENTITY_B)
+ held, _ = _guard([new])
+ new.stop.assert_called_once()
+ new.remove.assert_not_called()
+ assert IDENTITY_A in held["cashpilot-mysterium"] and IDENTITY_B in held["cashpilot-mysterium"]
+ # Sticky: stopped, it reads no identity, and it still is not started.
+ stopped = _node(status="exited")
+ _guard([stopped])
+ stopped.start.assert_not_called()
+
+ def test_control_the_same_identity_after_the_move_runs(self, markers):
+ _raw_deploy(old=self._old_with_data(IDENTITY_A))
+ new = _node()
+ new.exec_run.return_value = _says(IDENTITY_A)
+ held, _ = _guard([new])
+ new.stop.assert_not_called()
+ assert held == {}
+ assert orchestrator._pending_identity("cashpilot-mysterium") is None
+
+
+def test_the_fleet_page_offers_the_move_and_says_isolation_first():
+ fleet = (ROOT / "app" / "templates" / "fleet.html").read_text(encoding="utf-8")
+ assert "iso.unfenced" in fleet
+ assert "to move behind the fence" in fleet
+ assert "Switch isolation on first, then redeploy it once" in fleet
+ assert "moveBehindFence(" in fleet
+
+
+class TestAHoldThatCannotBeWritten:
+ """Found in the lab: a pending record the worker could not overwrite let the guard restart the node each pass."""
+
+ def _unwritable(self, markers):
+ orchestrator._remember_identity("cashpilot-mysterium", IDENTITY_A)
+ (markers / "identity").chmod(0o500)
+ (markers / "identity" / "cashpilot-mysterium").chmod(0o400)
+
+ def test_the_hold_survives_and_the_node_is_not_started_again(self, markers):
+ self._unwritable(markers)
+ try:
+ new = _node()
+ new.exec_run.return_value = _says(IDENTITY_B)
+ _guard([new])
+ new.stop.assert_called_once()
+ stopped = _node(status="exited")
+ held, _ = _guard([stopped])
+ stopped.start.assert_not_called()
+ assert IDENTITY_B in held["cashpilot-mysterium"]
+ with pytest.raises(orchestrator.ContainerNetworkError, match="identity"):
+ _start(_node(status="exited"))
+ finally:
+ (markers / "identity").chmod(0o700)
+
+ def test_control_a_redeploy_lifts_the_hold(self, markers):
+ self._unwritable(markers)
+ try:
+ new = _node()
+ new.exec_run.return_value = _says(IDENTITY_B)
+ _guard([new])
+ finally:
+ (markers / "identity").chmod(0o700)
+ (markers / "identity" / "cashpilot-mysterium").chmod(0o600)
+ orchestrator._remember_identity("cashpilot-mysterium", None)
+ stopped = _node(status="exited")
+ _guard([stopped])
+ stopped.start.assert_called_once()
+
+
+def test_a_failed_scan_drops_the_unfenced_report(markers):
+ """No Fleet button on a report the worker could not refresh."""
+ _guard([_old_node()])
+ assert lan_fence.status()["unfenced"]
+ client = MagicMock()
+ client.containers.list.side_effect = RuntimeError("docker is gone")
+ with (
+ patch.object(orchestrator, "docker_available", return_value=True),
+ patch.object(orchestrator, "_get_client", return_value=client),
+ ):
+ lan_fence.guard()
+ assert lan_fence.status()["unfenced"] == []
+
+
+def test_the_fleet_page_does_not_offer_a_move_it_just_made():
+ fleet = (ROOT / "app" / "templates" / "fleet.html").read_text(encoding="utf-8")
+ assert "_movedBehindFence.add(" in fleet
+ assert "!_movedBehindFence.has(" in fleet
+
+
+class TestTheMoveNeedsTheOldIdentity:
+ def test_a_running_node_whose_identity_cannot_be_read_is_not_replaced(self, markers):
+ old = _node()
+ old.exec_run.return_value = (1, b"")
+ old.attrs["Mounts"] = []
+ with pytest.raises(orchestrator.ContainerRuntimeError, match="not touched"):
+ _raw_deploy(old=old)
+ old.stop.assert_not_called()
+ old.remove.assert_not_called()
+
+ def test_control_a_stopped_node_is_replaced_without_the_check(self, markers):
+ old = _node(status="exited")
+ old.exec_run.return_value = (1, b"")
+ old.attrs["Mounts"] = []
+ _raw_deploy(old=old)
+ old.remove.assert_called_once()
+
+ def test_a_redeploy_lifts_a_hold_kept_only_in_memory(self, markers):
+ orchestrator._remember_identity("cashpilot-mysterium", IDENTITY_A)
+ orchestrator._MISMATCH_HELD["cashpilot-mysterium"] = "its identity is 0xb, not 0xa"
+ old = _node(status="exited")
+ old.attrs["Mounts"] = []
+ _raw_deploy(old=old)
+ assert "cashpilot-mysterium" not in orchestrator._MISMATCH_HELD
+ # The identity to match is kept, so the restored volume is still checked.
+ assert orchestrator._pending_identity("cashpilot-mysterium") == IDENTITY_A