From 2436517218facf839c9813c86afd7c1a5c60a2c5 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sun, 8 Nov 2026 05:41:51 +0100
Subject: [PATCH 1/5] feat(fleet): offer a one-click move behind the fence for
a host-network Mysterium
An upgraded worker leaves a Mysterium node from 2.0.6 or earlier running on
the host network until its owner redeploys it. The worker now reports such
nodes, and the Fleet page shows them under their worker with a Redeploy to
move behind the fence button, or says to switch isolation on first. The
redeploy reads the running node's identity before replacing it and holds the
new node, volume untouched, if the identity differs.
---
UPGRADING.md | 11 ++---
app/lan_fence.py | 19 ++++++++
app/templates/fleet.html | 40 +++++++++++++++++
tests/test_mysterium_fenced.py | 81 ++++++++++++++++++++++++++++++++++
4 files changed, 146 insertions(+), 5 deletions(-)
diff --git a/UPGRADING.md b/UPGRADING.md
index 0769805b..d4ca2059 100644
--- a/UPGRADING.md
+++ b/UPGRADING.md
@@ -29,11 +29,12 @@ releases run it in its own network namespace on `cashpilot-isolated`, behind
the fence, and the worker starts it only once the fence is in place
([Mysterium on the fenced bridge](docs/isolation.md#mysterium-on-the-fenced-bridge)).
-A running node keeps working exactly as it is until you redeploy it, in any
-isolation mode: the upgraded worker does not stop, start or change it. The
-dashboard lists it under drift ("shares the host's network, where its
-NET_ADMIN reaches the host's firewall. Switch isolation on in Fleet, then
-redeploy Mysterium once"), and **Check isolation** fails on it.
+A running node keeps running until you redeploy it, in any isolation mode:
+the upgraded worker does not stop, start or change it. Redeploy it once to
+move it behind the fence. Until then the Fleet page shows it under its worker
+with a **Redeploy MystNodes to move behind the fence** button (or, with
+isolation off, says to switch it on first), the dashboard lists it under
+drift, and **Check isolation** fails on it.
**What to do, in this order.**
diff --git a/app/lan_fence.py b/app/lan_fence.py
index b43ecfca..912b480d 100644
--- a/app/lan_fence.py
+++ b/app/lan_fence.py
@@ -529,6 +529,7 @@ class IsolationConflict(RuntimeError):
"earners_outside_stopped": [],
# Fence-gated containers the guard holds stopped, by name, with the reason.
"guard": {},
+ "unfenced": [],
}
#: The worker's public egress IP, set from the heartbeat's detection
#: (app/worker_api.py). The next apply adds its drop.
@@ -1125,6 +1126,22 @@ def _gated(client: Any) -> list[Any]:
]
+def _unfenced(client: Any) -> list[dict[str, str]]:
+ """Fence-gated services' containers still on the host's network (2.0.6 or earlier).
+
+ Left running as they are; the Fleet page offers the redeploy that moves them.
+ """
+ orch = _orchestrator()
+ from app.constants import LABEL_SERVICE
+
+ found = []
+ for c in _managed(client):
+ slug = str((c.labels or {}).get(LABEL_SERVICE) or "")
+ if orch.fence_gated(slug) and orch.predates_fence(c):
+ found.append({"name": c.name, "slug": slug, "service": orch.service_name(slug)})
+ return found
+
+
def guard() -> dict[str, str]:
"""Every GUARD_SECONDS: hold each fence-gated container to the fence. Never raises.
@@ -1145,6 +1162,7 @@ def guard() -> dict[str, str]:
if not orch.docker_available():
return {}
client = orch._get_client()
+ _state["unfenced"] = _unfenced(client)
gated = _gated(client)
if not gated:
_state["guard"] = {}
@@ -1348,6 +1366,7 @@ def status() -> dict[str, Any]:
"earners_outside": list(_state["earners_outside"]),
"earners_outside_stopped": list(_state.get("earners_outside_stopped") or []),
"guard": dict(_state.get("guard") or {}),
+ "unfenced": [dict(u) for u in _state.get("unfenced") or []],
}
diff --git a/app/templates/fleet.html b/app/templates/fleet.html
index 5fe8d7cc..ceb6ee4d 100644
--- a/app/templates/fleet.html
+++ b/app/templates/fleet.html
@@ -498,6 +498,22 @@
Proxies
if (held.length) {
line += `${line ? '
' : ''}Held stopped: ${held.map(([n, why]) => `${esc(n)} (${esc(why)})`).join('; ')}. A missing LAN fence clears by itself; any other reason needs a redeploy from the dashboard.${line ? '
' : ''}`;
}
+ // A Mysterium from 2.0.6 or earlier, still on the host's network: left
+ // running until its owner redeploys it, which moves it behind the fence.
+ const unfenced = iso.unfenced || [];
+ if (unfenced.length) {
+ const fenced = iso.mode === 'on' || iso.mode === 'operator';
+ const what = unfenced.map((u) => esc(u.service || u.slug)).join(', ');
+ const order = fenced
+ ? 'It keeps running as it is until you redeploy it once, which moves it behind the fence and keeps its identity.'
+ : iso.mode === 'unsupported'
+ ? 'It keeps running as it is. Podman cannot be fenced: run it on Docker with isolation on, or on your own network (CASHPILOT_CONTAINER_NETWORK).'
+ : 'It keeps running as it is. Switch isolation on first, then redeploy it once to move it behind the fence.';
+ const redeploy = _isOwner && fenced && !(sw && sw.busy)
+ ? unfenced.map((u) => ` `).join('')
+ : '';
+ line += `${line ? '
' : ''}${what} still shares this host's network, where its NET_ADMIN can rewrite the host's firewall. ${order}${redeploy}${line ? '
' : ''}`;
+ }
if (sw) {
const tone = sw.error ? 'text-fail' : sw.busy ? 'muted' : 'text-ok';
line += `
` : '';
}
+ async function moveBehindFence(workerId, slug, service) {
+ if (!confirm(`Redeploy ${service} to move it behind the LAN fence?\n\n`
+ + '- The worker reads the node\'s identity, replaces the container on the fenced network and checks the new one has the same identity.\n'
+ + '- The data volume is kept. If the identity differs, the node is held stopped and the volume is left as it is.\n'
+ + '- The node is down for under a minute.')) return;
+ _fenceSwitch[workerId] = { busy: `Moving ${service} behind the fence…` };
+ loadFleet();
+ try {
+ // An empty body redeploys from the recorded spec; a fence-gated service lands on the fenced bridge.
+ await CP.api(`/api/deploy/${encodeURIComponent(slug)}?worker_id=${workerId}`, { method: 'POST', body: {} });
+ _fenceSwitch[workerId] = { done: `${service} was redeployed behind the fence.` };
+ } catch (err) {
+ _fenceSwitch[workerId] = { error: `${service} was not moved: ${err.message}` };
+ }
+ loadFleet();
+ }
+
async function enableFence(workerId, name) {
if (!confirm(`Turn on LAN isolation on "${name}"?\n\n`
+ '- The worker creates a Docker network (cashpilot-isolated) and one firewall table (ip cashpilot) on its host.\n'
@@ -843,6 +876,13 @@
Proxies
return;
}
+ const unfencedBtn = e.target.closest('.btn-unfenced-redeploy');
+ if (unfencedBtn) {
+ const id = parseInt(unfencedBtn.dataset.workerId, 10);
+ if (!isNaN(id) && !(_fenceSwitch[id] && _fenceSwitch[id].busy)) moveBehindFence(id, unfencedBtn.dataset.slug, unfencedBtn.dataset.service);
+ return;
+ }
+
const isoBtn = e.target.closest('.btn-isolation-check');
if (isoBtn) {
const id = parseInt(isoBtn.dataset.workerId, 10);
diff --git a/tests/test_mysterium_fenced.py b/tests/test_mysterium_fenced.py
index faf06ec7..d0edc043 100644
--- a/tests/test_mysterium_fenced.py
+++ b/tests/test_mysterium_fenced.py
@@ -981,3 +981,84 @@ def svc(default):
assert lan_isolation.published_ports(svc("127.0.0.2")) == []
assert lan_isolation.published_ports(svc("::1,localhost")) == []
assert lan_isolation.published_ports(svc("192.168.1.5")) == ["192.168.1.5:4449"]
+
+
+# ---------------------------------------------------------------------------
+# (m) The upgrade: the old node keeps earning, the Fleet page offers the move,
+# and the move compares the identity with the node it replaces
+# ---------------------------------------------------------------------------
+
+IDENTITY_A = "0x" + "a" * 40
+IDENTITY_B = "0x" + "b" * 40
+
+
+def _says(identity):
+ return (0, b'{"identities":[{"id":"' + identity.encode() + b'"}]}')
+
+
+class TestTheUpgrade:
+ @pytest.mark.parametrize("mode", [lan_fence.MODE_ON, lan_fence.MODE_LEGACY])
+ def test_the_old_node_keeps_running_and_the_fleet_page_is_told(self, markers, mode):
+ node = _old_node()
+ problem = None if mode == lan_fence.MODE_ON else lan_fence.mode_problem(mode)
+ _guard([node], problem=problem, mode=mode)
+ node.stop.assert_not_called()
+ node.restart.assert_not_called()
+ node.remove.assert_not_called()
+ assert lan_fence.status()["unfenced"] == [
+ {"name": "cashpilot-mysterium", "slug": "mysterium", "service": MYST["name"]}
+ ]
+
+ def test_control_a_fenced_node_is_not_reported_as_unfenced(self, markers):
+ _guard([_node()])
+ assert lan_fence.status()["unfenced"] == []
+
+ def _old_with_data(self, identity):
+ old = _old_node()
+ old.attrs["Mounts"] = [
+ {"Type": "bind", "Source": "/mnt/appdata/myst/data", "Destination": "/var/lib/mysterium-node", "RW": True}
+ ]
+ old.exec_run.return_value = _says(identity)
+ return old
+
+ def test_the_redeploy_reads_the_running_old_nodes_identity_before_replacing_it(self, markers):
+ old = self._old_with_data(IDENTITY_A)
+ order = []
+ old.exec_run.side_effect = lambda *a, **k: order.append("read") or _says(IDENTITY_A)
+ old.stop.side_effect = lambda **_: order.append("stop")
+ old.remove.side_effect = lambda **_: order.append("remove")
+ client, _ = _raw_deploy(old=old, volumes={"mysterium-data": {"bind": "/var/lib/mysterium-node", "mode": "rw"}})
+ assert order[0] == "read" and "stop" in order
+ assert orchestrator._pending_identity("cashpilot-mysterium") == IDENTITY_A
+ volumes = client.containers.create.call_args.kwargs["volumes"]
+ assert "/mnt/appdata/myst/data" in str(volumes), "the node's own data must come with it"
+
+ def test_a_different_identity_after_the_move_holds_it_and_keeps_the_volume(self, markers):
+ _raw_deploy(old=self._old_with_data(IDENTITY_A))
+ new = _node()
+ new.exec_run.return_value = _says(IDENTITY_B)
+ held, _ = _guard([new])
+ new.stop.assert_called_once()
+ new.remove.assert_not_called()
+ assert IDENTITY_A in held["cashpilot-mysterium"] and IDENTITY_B in held["cashpilot-mysterium"]
+ # Sticky: stopped, it reads no identity, and it still is not started.
+ stopped = _node(status="exited")
+ _guard([stopped])
+ stopped.start.assert_not_called()
+
+ def test_control_the_same_identity_after_the_move_runs(self, markers):
+ _raw_deploy(old=self._old_with_data(IDENTITY_A))
+ new = _node()
+ new.exec_run.return_value = _says(IDENTITY_A)
+ held, _ = _guard([new])
+ new.stop.assert_not_called()
+ assert held == {}
+ assert orchestrator._pending_identity("cashpilot-mysterium") is None
+
+
+def test_the_fleet_page_offers_the_move_and_says_isolation_first():
+ fleet = (ROOT / "app" / "templates" / "fleet.html").read_text(encoding="utf-8")
+ assert "iso.unfenced" in fleet
+ assert "to move behind the fence" in fleet
+ assert "Switch isolation on first, then redeploy it once" in fleet
+ assert "moveBehindFence(" in fleet
From 4a8ed7f4576e92725201bd8b709020dcc56cecef Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sun, 8 Nov 2026 06:11:15 +0100
Subject: [PATCH 2/5] fix(mysterium): keep an identity hold when its record
cannot be written
The lab showed a node held for a different identity being started again
every guard pass when the worker could not overwrite the pending record:
stopped, it read as unchecked. The hold now lives in memory as well until
the next deploy. The Fleet line in off mode names the setting to change.
---
app/orchestrator.py | 12 ++++++++++
app/templates/fleet.html | 4 +++-
tests/test_mysterium_fenced.py | 40 ++++++++++++++++++++++++++++++++++
3 files changed, 55 insertions(+), 1 deletion(-)
diff --git a/app/orchestrator.py b/app/orchestrator.py
index e5e786ba..f1176997 100644
--- a/app/orchestrator.py
+++ b/app/orchestrator.py
@@ -1258,6 +1258,12 @@ def _identity_file(name: str) -> str:
return os.path.join(IDENTITY_DIR, re.sub(r"[^A-Za-z0-9_.-]", "_", name))
+#: Mismatch holds by container name, kept in memory as well as on disk: a hold
+#: whose record could not be written must still hold, or the stopped node
+#: would read as unchecked and the guard would start it again every pass.
+_MISMATCH_HELD: dict[str, str] = {}
+
+
def _pending_identity(name: str) -> str | None:
"""The identity a replacement still has to match, or None (no record, or a held mismatch)."""
try:
@@ -1269,6 +1275,9 @@ def _pending_identity(name: str) -> str | None:
def _remember_identity(name: str, identity: str | None) -> None:
+ if not str(identity or "").startswith("MISMATCH "):
+ # A deploy rewrites the record: the owner's way out of a hold.
+ _MISMATCH_HELD.pop(name, None)
path = _identity_file(name)
try:
if identity:
@@ -1287,6 +1296,8 @@ def identity_problem(container: Any) -> str | None:
None while there is nothing to compare (no record, or the node not
answering yet) and once it matches, when the record is deleted.
"""
+ if held_reason := _MISMATCH_HELD.get(container.name):
+ return held_reason
path = _identity_file(container.name)
try:
with open(path, encoding="utf-8") as fh:
@@ -1312,6 +1323,7 @@ def identity_problem(container: Any) -> str | None:
f"its identity is {current}, not {expected} as before the redeploy, so it is not running on the data "
"volume that holds its keystore. It stays stopped: restore that volume, then redeploy"
)
+ _MISMATCH_HELD[container.name] = reason
_remember_identity(container.name, f"MISMATCH {reason}")
return reason
diff --git a/app/templates/fleet.html b/app/templates/fleet.html
index ceb6ee4d..c7e3b8ae 100644
--- a/app/templates/fleet.html
+++ b/app/templates/fleet.html
@@ -508,7 +508,9 @@
Proxies
? 'It keeps running as it is until you redeploy it once, which moves it behind the fence and keeps its identity.'
: iso.mode === 'unsupported'
? 'It keeps running as it is. Podman cannot be fenced: run it on Docker with isolation on, or on your own network (CASHPILOT_CONTAINER_NETWORK).'
- : 'It keeps running as it is. Switch isolation on first, then redeploy it once to move it behind the fence.';
+ : iso.mode === 'off'
+ ? 'It keeps running as it is. Set CASHPILOT_ISOLATION=auto on this worker and restart it, then redeploy it once to move it behind the fence.'
+ : 'It keeps running as it is. Switch isolation on first, then redeploy it once to move it behind the fence.';
const redeploy = _isOwner && fenced && !(sw && sw.busy)
? unfenced.map((u) => ` `).join('')
: '';
diff --git a/tests/test_mysterium_fenced.py b/tests/test_mysterium_fenced.py
index d0edc043..664a13bb 100644
--- a/tests/test_mysterium_fenced.py
+++ b/tests/test_mysterium_fenced.py
@@ -271,6 +271,7 @@ def test_control_the_catalog_spec_passes(self, real_catalog):
def markers(tmp_path, monkeypatch):
monkeypatch.setattr(orchestrator, "STOPPED_MARKER", str(tmp_path / "stopped"))
monkeypatch.setattr(orchestrator, "IDENTITY_DIR", str(tmp_path / "identity"))
+ monkeypatch.setattr(orchestrator, "_MISMATCH_HELD", {}, raising=False)
return tmp_path
@@ -1062,3 +1063,42 @@ def test_the_fleet_page_offers_the_move_and_says_isolation_first():
assert "to move behind the fence" in fleet
assert "Switch isolation on first, then redeploy it once" in fleet
assert "moveBehindFence(" in fleet
+
+
+class TestAHoldThatCannotBeWritten:
+ """Found in the lab: a pending record the worker could not overwrite let the guard restart the node each pass."""
+
+ def _unwritable(self, markers):
+ orchestrator._remember_identity("cashpilot-mysterium", IDENTITY_A)
+ (markers / "identity").chmod(0o500)
+ (markers / "identity" / "cashpilot-mysterium").chmod(0o400)
+
+ def test_the_hold_survives_and_the_node_is_not_started_again(self, markers):
+ self._unwritable(markers)
+ try:
+ new = _node()
+ new.exec_run.return_value = _says(IDENTITY_B)
+ _guard([new])
+ new.stop.assert_called_once()
+ stopped = _node(status="exited")
+ held, _ = _guard([stopped])
+ stopped.start.assert_not_called()
+ assert IDENTITY_B in held["cashpilot-mysterium"]
+ with pytest.raises(orchestrator.ContainerNetworkError, match="identity"):
+ _start(_node(status="exited"))
+ finally:
+ (markers / "identity").chmod(0o700)
+
+ def test_control_a_redeploy_lifts_the_hold(self, markers):
+ self._unwritable(markers)
+ try:
+ new = _node()
+ new.exec_run.return_value = _says(IDENTITY_B)
+ _guard([new])
+ finally:
+ (markers / "identity").chmod(0o700)
+ (markers / "identity" / "cashpilot-mysterium").chmod(0o600)
+ orchestrator._remember_identity("cashpilot-mysterium", None)
+ stopped = _node(status="exited")
+ _guard([stopped])
+ stopped.start.assert_called_once()
From 29c64278aff3b293c172120ded6fb82f4fffd566 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sun, 8 Nov 2026 12:59:21 +0100
Subject: [PATCH 3/5] fix(fleet): no move button on a stale report or right
after a move
A guard pass that could not list containers kept its last unfenced report,
so Fleet could offer a move for a node already moved or removed; it now
drops the report until a fresh scan. A node moved from the page is not
offered again until the worker stops reporting it.
---
app/lan_fence.py | 3 +++
app/templates/fleet.html | 9 ++++++++-
tests/test_mysterium_fenced.py | 20 ++++++++++++++++++++
3 files changed, 31 insertions(+), 1 deletion(-)
diff --git a/app/lan_fence.py b/app/lan_fence.py
index 912b480d..0f26f846 100644
--- a/app/lan_fence.py
+++ b/app/lan_fence.py
@@ -1160,6 +1160,7 @@ def guard() -> dict[str, str]:
held: dict[str, str] = {}
try:
if not orch.docker_available():
+ _state["unfenced"] = []
return {}
client = orch._get_client()
_state["unfenced"] = _unfenced(client)
@@ -1204,6 +1205,8 @@ def guard() -> dict[str, str]:
orch.GATED_DEPLOY_LOCK.release()
except Exception as exc: # noqa: BLE001 - the next pass tries again
logger.warning("Fence guard pass failed: %s", exc)
+ # Not a fresh scan: no Fleet button on a report that may be stale.
+ _state["unfenced"] = []
_state["guard"] = held
return held
diff --git a/app/templates/fleet.html b/app/templates/fleet.html
index c7e3b8ae..d73cef54 100644
--- a/app/templates/fleet.html
+++ b/app/templates/fleet.html
@@ -460,6 +460,7 @@
Proxies
// Turning isolation on, per worker id: {busy}, {done}, or {error}.
const _fenceSwitch = {};
+ const _movedBehindFence = new Set();
// The LAN fence of one worker (app/lan_fence.py), from its heartbeat. Says
// nothing when the fence holds and every earner sits behind it.
@@ -500,7 +501,12 @@
Proxies
}
// A Mysterium from 2.0.6 or earlier, still on the host's network: left
// running until its owner redeploys it, which moves it behind the fence.
- const unfenced = iso.unfenced || [];
+ // A node moved from this page stays hidden until the worker stops reporting
+ // it: a heartbeat sent before the move must not offer the move again.
+ for (const key of [..._movedBehindFence]) {
+ if (key.startsWith(`${w.id}:`) && !(iso.unfenced || []).some((u) => key === `${w.id}:${u.slug}`)) _movedBehindFence.delete(key);
+ }
+ const unfenced = (iso.unfenced || []).filter((u) => !_movedBehindFence.has(`${w.id}:${u.slug}`));
if (unfenced.length) {
const fenced = iso.mode === 'on' || iso.mode === 'operator';
const what = unfenced.map((u) => esc(u.service || u.slug)).join(', ');
@@ -533,6 +539,7 @@
Proxies
try {
// An empty body redeploys from the recorded spec; a fence-gated service lands on the fenced bridge.
await CP.api(`/api/deploy/${encodeURIComponent(slug)}?worker_id=${workerId}`, { method: 'POST', body: {} });
+ _movedBehindFence.add(`${workerId}:${slug}`);
_fenceSwitch[workerId] = { done: `${service} was redeployed behind the fence.` };
} catch (err) {
_fenceSwitch[workerId] = { error: `${service} was not moved: ${err.message}` };
diff --git a/tests/test_mysterium_fenced.py b/tests/test_mysterium_fenced.py
index 664a13bb..70b44626 100644
--- a/tests/test_mysterium_fenced.py
+++ b/tests/test_mysterium_fenced.py
@@ -1102,3 +1102,23 @@ def test_control_a_redeploy_lifts_the_hold(self, markers):
stopped = _node(status="exited")
_guard([stopped])
stopped.start.assert_called_once()
+
+
+def test_a_failed_scan_drops_the_unfenced_report(markers):
+ """No Fleet button on a report the worker could not refresh."""
+ _guard([_old_node()])
+ assert lan_fence.status()["unfenced"]
+ client = MagicMock()
+ client.containers.list.side_effect = RuntimeError("docker is gone")
+ with (
+ patch.object(orchestrator, "docker_available", return_value=True),
+ patch.object(orchestrator, "_get_client", return_value=client),
+ ):
+ lan_fence.guard()
+ assert lan_fence.status()["unfenced"] == []
+
+
+def test_the_fleet_page_does_not_offer_a_move_it_just_made():
+ fleet = (ROOT / "app" / "templates" / "fleet.html").read_text(encoding="utf-8")
+ assert "_movedBehindFence.add(" in fleet
+ assert "!_movedBehindFence.has(" in fleet
From 447258fd5bb35cbb20915e4d922ac4ac14da7f15 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sun, 8 Nov 2026 13:13:25 +0100
Subject: [PATCH 4/5] test(fleet): give the fence harness the page's new
moved-node set
The harness runs the Fleet page's own render code with the page's globals;
the moved-node set added for the move button was missing from it, so every
case failed with a ReferenceError.
---
tests/test_lan_fence.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/tests/test_lan_fence.py b/tests/test_lan_fence.py
index 11c706d3..f1a20e25 100644
--- a/tests/test_lan_fence.py
+++ b/tests/test_lan_fence.py
@@ -1291,6 +1291,7 @@ def test_the_worker_image_ships_the_fence():
const esc = s => String(s).replace(/[&<>"']/g, c => ({'&': '&', '<': '<', '>': '>', '"': '"', "'": '''}[c]));
const _isOwner = true;
const _fenceSwitch = {};
+const _movedBehindFence = new Set();
const calls = [];
const loadFleet = () => {};
const confirm = () => true;
From d57926f3a4c5c21be94e80fff80d8a40582a27b6 Mon Sep 17 00:00:00 2001
From: GeiserX <9169332+GeiserX@users.noreply.github.com>
Date: Sun, 8 Nov 2026 13:30:55 +0100
Subject: [PATCH 5/5] fix(mysterium): never move a running node whose identity
cannot be read
A move over a running node that did not answer its identity went ahead
with nothing to compare the new one against, so it could run on another
keystore unchecked; it is now refused before the node is touched, and a
stopped node is still replaced without the check. A redeploy also lifts a
mismatch hold kept only in memory, keeping the identity to match.
---
app/orchestrator.py | 21 ++++++++++++++++++++-
tests/test_mysterium_fenced.py | 28 ++++++++++++++++++++++++++++
2 files changed, 48 insertions(+), 1 deletion(-)
diff --git a/app/orchestrator.py b/app/orchestrator.py
index f1176997..5087fbeb 100644
--- a/app/orchestrator.py
+++ b/app/orchestrator.py
@@ -893,13 +893,25 @@ def deploy_raw(
# chance to flush (storj declares 300 s for that reason) and pays for it
# with an unclean-shutdown recovery. A stop that fails is not a reason to
# abandon the deploy - the forced removal below is what ran before.
+ if gated:
+ # A deploy is the owner's way out of a hold, written down or not.
+ _MISMATCH_HELD.pop(name, None)
if gated and not _pending_identity(name):
# Read while it still runs; the guard compares the new one with it.
# A redeploy over a stopped node clears the record instead. A record
# still pending is kept: the container being replaced is itself a
# replacement whose identity was never confirmed, so this retry is
# compared with the node as it was before the first redeploy.
- identity = read_node_identity(old, slug) if getattr(old, "status", "") == "running" else None
+ running = getattr(old, "status", "") == "running"
+ identity = read_node_identity(old, slug) if running else None
+ if running and identity is None and _reads_identity(slug):
+ # Without it the new node could run on another keystore unchecked.
+ raise ContainerRuntimeError(
+ f"{name} was not replaced: its identity could not be read from the running node, so the new "
+ "one could not be checked against it. The running container was not touched. Try again in a "
+ "minute. If the node is broken, stop it from the dashboard first: a stopped node is replaced "
+ "without the check."
+ )
_remember_identity(name, identity)
try:
old.stop(timeout=_get_stop_timeout(slug))
@@ -1254,6 +1266,13 @@ def read_node_identity(container: Any, slug: str) -> str | None:
return match.group(1) if match else None
+def _reads_identity(slug: str) -> bool:
+ """Whether the catalog says how to read this service's node identity (collector.node_read)."""
+ svc = (get_service(slug) if get_service else None) or {}
+ spec = (svc.get("collector") or {}).get("node_read") or {}
+ return isinstance(spec.get("command"), list) and bool(spec.get("command")) and isinstance(spec.get("node"), str)
+
+
def _identity_file(name: str) -> str:
return os.path.join(IDENTITY_DIR, re.sub(r"[^A-Za-z0-9_.-]", "_", name))
diff --git a/tests/test_mysterium_fenced.py b/tests/test_mysterium_fenced.py
index 70b44626..2fb3ab1a 100644
--- a/tests/test_mysterium_fenced.py
+++ b/tests/test_mysterium_fenced.py
@@ -1122,3 +1122,31 @@ def test_the_fleet_page_does_not_offer_a_move_it_just_made():
fleet = (ROOT / "app" / "templates" / "fleet.html").read_text(encoding="utf-8")
assert "_movedBehindFence.add(" in fleet
assert "!_movedBehindFence.has(" in fleet
+
+
+class TestTheMoveNeedsTheOldIdentity:
+ def test_a_running_node_whose_identity_cannot_be_read_is_not_replaced(self, markers):
+ old = _node()
+ old.exec_run.return_value = (1, b"")
+ old.attrs["Mounts"] = []
+ with pytest.raises(orchestrator.ContainerRuntimeError, match="not touched"):
+ _raw_deploy(old=old)
+ old.stop.assert_not_called()
+ old.remove.assert_not_called()
+
+ def test_control_a_stopped_node_is_replaced_without_the_check(self, markers):
+ old = _node(status="exited")
+ old.exec_run.return_value = (1, b"")
+ old.attrs["Mounts"] = []
+ _raw_deploy(old=old)
+ old.remove.assert_called_once()
+
+ def test_a_redeploy_lifts_a_hold_kept_only_in_memory(self, markers):
+ orchestrator._remember_identity("cashpilot-mysterium", IDENTITY_A)
+ orchestrator._MISMATCH_HELD["cashpilot-mysterium"] = "its identity is 0xb, not 0xa"
+ old = _node(status="exited")
+ old.attrs["Mounts"] = []
+ _raw_deploy(old=old)
+ assert "cashpilot-mysterium" not in orchestrator._MISMATCH_HELD
+ # The identity to match is kept, so the restored volume is still checked.
+ assert orchestrator._pending_identity("cashpilot-mysterium") == IDENTITY_A