From 28009baf93aea1f967db94612fb2aff4a49aed1f Mon Sep 17 00:00:00 2001 From: eigmax Date: Sat, 14 Nov 2026 09:50:27 +0000 Subject: [PATCH 1/5] Keep an app with the node that has its files, and let an app be anchored to its machine When an app's start found no room on its node, the scheduler moved the app to a node with room. That node took it although a volume the app runs from wasn't there: Docker mounted a new, empty directory owned by root in its place, and the app failed to start there, while its files stayed on the node it left. - A node refuses to create an app when a host path of its volumes isn't there (a link to nothing counts as missing). The scheduler then doesn't pick that node for the app again, and the start stays on its own node. A first placement whose create was refused goes to another node when one can take it, and back to the same one only when none can. - A container app can be anchored to a machine: with `machine` in its spec, it is placed on that machine's node only; `fleetctl app update NAME --machine MACHINE` (`{"machine": ...}` in PUT /apps/NAME/settings) anchors a placed app to the machine it runs on, and "" lifts the anchor. Only the node's configuration changes: the app is neither drained nor restarted. An anchored app's start waits for room on its node and is never moved. - A node's report reads its command results before its configuration, so a create is never reported done before the entry it wrote. The scheduler placed an app a second time on the same node when that happened, and the second create failed. --- bridge/fleet_bridge/node.py | 19 +++- bridge/tests/test_node.py | 35 +++++++ crates/fleet-mcp/src/main.rs | 5 +- crates/fleet-proto/src/models.rs | 15 ++- crates/fleet-scheduler/src/workloads_scale.rs | 67 ++++++++++++-- crates/fleet-scheduler/src/workloads_tests.rs | 92 +++++++++++++++++++ crates/fleetctl/src/main.rs | 16 +++- crates/fleetctl/tests/apps.rs | 2 + docs/reference.md | 19 ++-- 9 files changed, 242 insertions(+), 28 deletions(-) diff --git a/bridge/fleet_bridge/node.py b/bridge/fleet_bridge/node.py index 49c5628..9495f5d 100644 --- a/bridge/fleet_bridge/node.py +++ b/bridge/fleet_bridge/node.py @@ -413,7 +413,9 @@ def app(self, p: dict[str, Any]) -> tuple[dict[str, Any], Any]: (`p["entry"]`, by name, set only): `remove` it, or `set` some of its configuration. Settings are written to the configuration only: the scheduler drains and stops an app first and starts it again afterwards. `p["create"]` adds a new app's - entry to the configuration, disabled until the app is enabled. + entry to the configuration, disabled until the app is enabled. It is refused when a + host path of its volumes isn't here: what an app keeps on a node's disks stays on + that node, and Docker would mount a missing path as a new, empty directory. A removed entry has what is left of its replicas cleaned up the way disabling does. Removing an app from this node alone (a scale left it no replicas here) keeps its @@ -427,10 +429,15 @@ def app(self, p: dict[str, Any]) -> tuple[dict[str, Any], Any]: nothing = lambda: {} if "create" in p: entry = p["create"] - AppSpec.load(entry) + spec = AppSpec.load(entry) entries = (yaml.safe_load(self.apps_file.read_text()) or {}).get("apps") or [] if any(e.get("name") == entry["name"] for e in entries): raise ValueError(f"{entry['name']} is already configured on this node") + missing = [path for path in (v.split(":")[0] for v in spec.volumes) + if path.startswith("/") and not os.path.exists(path)] + if missing: + raise ValueError(f"{entry['name']} can't run on this node: {', '.join(missing)} isn't here, " + "and Fleet copies nothing an app keeps on a node's disks") append_entry(self.apps_file, "apps", entry) return {"created": [entry["name"]]}, nothing if "batch" in p: @@ -585,6 +592,11 @@ def _delete_jobs(self, batch: str, ids: list[str]) -> dict[str, Any]: return {"jobs_deleted": deleted, "kept": kept} def report(self) -> dict[str, Any]: + # The results first: a command finished by then has changed the configuration read + # below, so the scheduler never sees a create done before the node reports its entry. + with self.lock: + results = [{"id": p.stem, "result": v["result"], "error": v["error"]} + for p in self.journal.glob("*.json") if (v := read_json(p)).get("done")] nodes = alive_nodes() ready = len(nodes) == 1 total, free = resources() @@ -635,9 +647,6 @@ def report(self) -> dict[str, Any]: "logs_ready": (entry / f".fleet-logs-{lease}").exists()}) except (OSError, ValueError, SystemExit): log.exception("reading local job %s", entry.name) - with self.lock: - results = [{"id": p.stem, "result": v["result"], "error": v["error"]} - for p in self.journal.glob("*.json") if (v := read_json(p)).get("done")] return {"version": 3, "machine": socket.gethostname(), "ready": ready, "total_gpus": total.get("GPU", 0), "total_cpus": total.get("CPU", 0), "free_gpus": free.get("GPU", 0), "free_cpus": free.get("CPU", 0), diff --git a/bridge/tests/test_node.py b/bridge/tests/test_node.py index 8030c1b..db0e3c2 100644 --- a/bridge/tests/test_node.py +++ b/bridge/tests/test_node.py @@ -568,6 +568,41 @@ def test_create_adds_a_disabled_app_and_refuses_a_taken_name(executor): assert error and "already configured" in str(error) +def test_create_refuses_an_app_whose_volumes_are_not_on_this_node(executor,tmp_path): + """Its data stays on the node it ran on; Docker would mount an empty directory in its place.""" + here=tmp_path/"models";here.mkdir() + gone=tmp_path/"ComfyUI" + entry={"name":"web","kind":"app","namespace":"a"*32,"disabled":True,"image":"web:1","port_base":50100, + "volumes":[f"{here}:ro",str(gone)]} + before=executor.apps_file.read_text() + result,error=app_message(executor,"create-1",{"create":entry},app="web",ns="a"*32) + assert error and f"web can't run on this node: {gone} isn't here" in str(error), error + assert executor.apps_file.read_text()==before and not gone.exists() + gone.symlink_to(tmp_path/"elsewhere") + result,error=app_message(executor,"create-2",{"create":entry},app="web",ns="a"*32) + assert error and "isn't here" in str(error), "a link to nothing is not there either" + gone.unlink();gone.mkdir() + assert app_message(executor,"create-3",{"create":entry},app="web",ns="a"*32)==({"created":["web"]},None) + + +def test_a_report_carries_a_create_done_only_with_its_entry(executor,monkeypatch): + """A create that finishes while a report is read is reported with its entry, in the next one.""" + monkeypatch.setattr(node.ray,"cluster_resources",lambda:{"GPU":8,"CPU":48}) + monkeypatch.setattr(node.ray,"available_resources",lambda:{"GPU":8,"CPU":48}) + entry={"name":"web","kind":"app","namespace":"a"*32,"disabled":True,"image":"web:1","port_base":50100} + def running(): + # The create runs after this report read the configuration. + if not (executor.journal/"create-1.json").exists(): + app_message(executor,"create-1",{"create":entry},app="web",ns="a"*32) + return {} + monkeypatch.setattr(reconcile,"running",running) + for report in [executor.report(),executor.report()]: + done=[r["id"] for r in report["results"]] + if "create-1" in done: + assert "web" in [a["name"] for a in report["apps"]], report + assert "create-1" in done + + def test_create_adds_a_prover_and_refuses_an_invalid_entry(executor): prover={"name":"proofs","kind":"ziren-gpu","namespace":"a"*32,"disabled":True,"image":"ziren-gpu:1","elf_id":"b"*64} assert app_message(executor,"create-1",{"create":prover},app="proofs",ns="a"*32)==({"created":["proofs"]},None) diff --git a/crates/fleet-mcp/src/main.rs b/crates/fleet-mcp/src/main.rs index 0f2c509..a5b548e 100644 --- a/crates/fleet-mcp/src/main.rs +++ b/crates/fleet-mcp/src/main.rs @@ -218,8 +218,9 @@ struct CreateApp { /// The app's container settings, the keys of its apps.yaml entry: kind (app, the default, /// or ziren-gpu/ziren-cpu for a prover), image, replicas, gpus, cpus, shm_gb, memory_gb, env, /// volumes, ready_timeout_s, stop_timeout_s; for an app also command, port_base, health_path, - /// workdir, user; for a prover proof_storage_root, backend, rust_log. image is required, and - /// port_base for an app; user, volumes and proof_storage_root require administrator credentials. + /// workdir, user, machine (anchors it to that machine: placed there, never moved); for a + /// prover proof_storage_root, backend, rust_log. image is required, and port_base for an app; + /// user, volumes and proof_storage_root require administrator credentials. spec: serde_json::Map, /// Priority inherited by the app's replicas and jobs (default 0; higher values run first). priority: Option, diff --git a/crates/fleet-proto/src/models.rs b/crates/fleet-proto/src/models.rs index da28e5c..5a00391 100644 --- a/crates/fleet-proto/src/models.rs +++ b/crates/fleet-proto/src/models.rs @@ -262,8 +262,9 @@ impl RegisterAppRequest { /// priority and program, and creates it disabled. pub const SPEC_SETTINGS: [&'static str; 11] = ["kind", "image", "replicas", "gpus", "cpus", "shm_gb", "memory_gb", "env", "volumes", "ready_timeout_s", "stop_timeout_s"]; - /// Settings only a container app's spec (`kind: app`) may give. - pub const CONTAINER_SETTINGS: [&'static str; 5] = ["command", "port_base", "health_path", "workdir", "user"]; + /// Settings only a container app's spec (`kind: app`) may give. `machine` anchors it to + /// that machine: it is placed there, and never moved to another node. + pub const CONTAINER_SETTINGS: [&'static str; 6] = ["command", "port_base", "health_path", "workdir", "user", "machine"]; /// Settings only a prover's spec (`kind: ziren-gpu` or `ziren-cpu`) may give. pub const PROVER_SETTINGS: [&'static str; 3] = ["proof_storage_root", "backend", "rust_log"]; @@ -311,8 +312,9 @@ impl RegisterAppRequest { } let strings = |v: &Value| v.as_array().is_none_or(|a| a.iter().any(|v| !v.is_string())); let positive = |v: &Value| v.as_f64().is_none_or(|v| v <= 0.0 || !v.is_finite()); - let invalid = ["user", "health_path", "workdir", "proof_storage_root", "backend", "rust_log"].into_iter() + let invalid = ["user", "health_path", "workdir", "proof_storage_root", "backend", "rust_log", "machine"].into_iter() .find(|k| set(k).is_some_and(|v| !v.is_string())) + .or_else(|| set("machine").and_then(Value::as_str).is_some_and(str::is_empty).then_some("machine")) .or_else(|| ["cpus", "shm_gb", "port_base", "stop_timeout_s"].into_iter().find(|k| set(k).is_some_and(|v| v.as_u64().is_none()))) .or_else(|| ["memory_gb", "ready_timeout_s"].into_iter().find(|k| set(k).is_some_and(positive))) .or_else(|| ["command", "volumes"].into_iter().find(|k| set(k).is_some_and(strings))) @@ -339,7 +341,10 @@ mod tests { }; let container = json!({"image":"serve:1","port_base":50100,"gpus":2,"replicas":1,"command":["serve","{port}"],"env":{"A":"1"}}); app(json!({"name":"model","spec":container})).unwrap(); - for (key, value) in [("gpus", json!("2")), ("replicas", json!(0)), ("port_base", json!(-1)), ("port_base", Value::Null), + let mut anchored = container.clone(); + anchored["machine"] = json!("gpu-box"); + app(json!({"name":"model","spec":anchored})).unwrap(); + for (key, value) in [("machine", json!("")), ("machine", json!(1)), ("gpus", json!("2")), ("replicas", json!(0)), ("port_base", json!(-1)), ("port_base", Value::Null), ("image", Value::Null), ("workdir", json!(1)), ("env", json!({"A":1})), ("kind", json!("vm")), ("proof_storage_root", json!("/proofs")), ("name", json!("x"))] { let mut spec = container.clone(); @@ -354,6 +359,8 @@ mod tests { assert!(app(json!({"name":"model","spec":container,"proof":proof})).is_err(), "a container app proves none"); assert!(app(json!({"name":"eth","spec":{"image":"ziren-gpu:1","port_base":1},"proof":proof})).is_err(), "a prover takes no container settings"); + assert!(app(json!({"name":"eth","spec":{"image":"ziren-gpu:1","machine":"gpu-box"},"proof":proof})).is_err(), + "a prover's node configures its machine"); assert!(app(json!({"name":"eth","elf_id":"ab".repeat(32)})).is_err(), "an app is created from its spec"); assert!(app(json!({"name":"model","node":"gpu","spec":container})).is_err(), "the scheduler picks the node"); } diff --git a/crates/fleet-scheduler/src/workloads_scale.rs b/crates/fleet-scheduler/src/workloads_scale.rs index a367286..1346cd6 100644 --- a/crates/fleet-scheduler/src/workloads_scale.rs +++ b/crates/fleet-scheduler/src/workloads_scale.rs @@ -238,8 +238,9 @@ impl WorkloadController { /// Settings an update may change on an app. The rest is identity, placement or the /// node's own configuration, or has its own operation (scale, enable/disable, priority). - pub(super) const APP_SETTINGS: [&'static str; 11] = ["image", "env", "command", "user", "workdir", "volumes", - "shm_gb", "memory_gb", "health_path", "stop_timeout_s", "ready_timeout_s"]; + /// `machine` anchors a container app to the machine it runs on, and changes nothing else. + pub(super) const APP_SETTINGS: [&'static str; 12] = ["image", "env", "command", "user", "workdir", "volumes", + "shm_gb", "memory_gb", "health_path", "stop_timeout_s", "ready_timeout_s", "machine"]; // Apps can access host networking, storage and prover credentials. Changing // their executable or environment is as privileged as changing their mounts. pub(super) const ADMIN_APP_SETTINGS: [&'static str; 6] = ["image", "env", "command", "user", "workdir", "volumes"]; @@ -310,6 +311,13 @@ impl WorkloadController { let _ops = self.ops.lock().await; let set = set.as_object().filter(|o| !o.is_empty()).cloned() .ok_or_else(|| fail(400, "set at least one setting"))?; + if let Some(machine) = set.get("machine") { + if set.len() > 1 { + return Err(fail(400, "set machine on its own: anchoring an app changes nothing else and doesn't restart it")); + } + let machine = machine.as_str().ok_or_else(|| fail(400, "machine: invalid value"))?; + return self.anchor(namespace, app, node, machine); + } for (key, value) in &set { let valid = match key.as_str() { "image" => value.as_str().is_some_and(|v| !v.is_empty() && !v.starts_with('-')), @@ -343,6 +351,40 @@ impl WorkloadController { Ok(self.scale_value(&intent)) } + /// Anchors `app`'s deployment on `node` (its only one, without a node) to `machine`, the + /// machine it runs on, or lifts the anchor with "". A start of an anchored app waits for + /// room on its node and is never moved to another, so an app that keeps its data on its + /// node's disks stays with it. Only the node's configuration changes: the app runs on. + /// The caller holds the operation lock. + fn anchor(&self, namespace: &str, app: &str, node: Option<&str>, machine: &str) -> anyhow::Result { + let (key, mut intent) = self.deployment_to_change(namespace, app, node)?; + if !intent.entry().is_container() { + return Err(fail(400, "only a container app is anchored this way: a prover's node configures its own machine")); + } + let on = self.inner.lock().workers.get(&intent.worker).map(|w| w.report.machine.clone()).unwrap_or_default(); + if !machine.is_empty() && machine != on { + return Err(fail(409, format!( + "{app} runs on node {} (machine {on}): it can be anchored only to the machine it runs on", intent.worker))); + } + if intent.entry().machine() == machine { + return Ok(self.scale_value(&intent)); + } + let name = intent.entry().name().to_owned(); + self.tell_nodes([&intent.worker], message::entry(&name), message::set(json!({"machine": machine})))?; + intent.spec["machine"] = json!(machine); + self.save("deployments", &key, &intent)?; + self.inner.lock().deployments.insert(key, intent.clone()); + // Placing the app anew follows the anchor too. + let template = self.inner.lock().apps.get(app).filter(|a| a.persistent && !a.removed && a.namespace == namespace).cloned(); + if let Some(mut template) = template { + template.spec["machine"] = json!(machine); + self.save("apps", app, &template)?; + self.inner.lock().apps.insert(app.into(), template); + } + tracing::info!(app, node = intent.worker, machine, "anchoring an app to the machine it runs on"); + Ok(self.scale_value(&intent)) + } + /// Sets an app's replica count, and its replicas' GPUs and CPUs: the app's settings, not any /// node's. The count is split over the app's deployments in the order its replicas are kept /// in: each keeps the replicas it has, up to the count; new ones go to the first, whose start @@ -831,7 +873,13 @@ impl WorkloadController { match pending { Some(pending) => pending, None => { - let (node, _) = self.place_app(template.entry(), &[])?; + // A node that refused to create it, as one without its volumes does, is + // picked again only when no other can take it. + let refused: Vec = self.inner.lock().commands.values() + .filter(|c| c.is(NodeOperation::App) && c.error.is_some() && c.created().is_some_and(|e| e.name() == app)) + .map(|c| c.worker.clone()).collect(); + let (node, _) = self.place_app(template.entry(), &refused) + .or_else(|_| self.place_app(template.entry(), &[]))?; let create = message::with(self.app_message(namespace, app), &message::create(&template.spec)); let create = self.command(&node, NodeOperation::App, create)?; template.workers = vec![node.clone()]; @@ -875,11 +923,14 @@ impl WorkloadController { /// and CPUs of all its replicas. One that has them free now is preferred, the one with the /// fewest free GPUs among those, as a batch-class proof app is placed, so larger requests /// keep the larger gaps; otherwise the one with the most free GPUs, where the app waits for - /// room. Whether it has the room now comes along. `except` are nodes not to pick. + /// room. Whether it has the room now comes along. `except` are nodes not to pick. An entry + /// anchored to a `machine` goes to that machine's node only. fn place_app(&self, entry: Entry<'_>, except: &[String]) -> anyhow::Result<(String, bool)> { let (gpus, cpus) = entry.total(1.0); + let machine = entry.machine(); let large: Vec = self.inner.lock().workers.iter() - .filter(|(id, w)| !except.contains(id) && w.report.total_gpus + 1e-9 >= gpus && w.report.total_cpus + 1e-9 >= cpus) + .filter(|(id, w)| !except.contains(id) && (machine.is_empty() || w.report.machine == machine) + && w.report.total_gpus + 1e-9 >= gpus && w.report.total_cpus + 1e-9 >= cpus) .map(|(id, _)| id.clone()).collect(); let sizes: Vec<(String, f64, f64)> = large.into_iter() .filter_map(|id| self.capacity(&id).map(|(g, c)| (id, g, c))) @@ -888,7 +939,11 @@ impl WorkloadController { sizes.iter().filter(fits).min_by(|a, b| a.1.total_cmp(&b.1).then(a.2.total_cmp(&b.2))) .map(|(id, _, _)| (id.clone(), true)) .or_else(|| sizes.iter().max_by(|a, b| a.1.total_cmp(&b.1).then(b.0.cmp(&a.0))).map(|(id, _, _)| (id.clone(), false))) - .ok_or_else(|| fail(503, format!("no ready node has {gpus} GPUs and {cpus} CPUs for this app"))) + .ok_or_else(|| fail(503, if machine.is_empty() { + format!("no ready node has {gpus} GPUs and {cpus} CPUs for this app") + } else { + format!("no ready node on machine {machine} has {gpus} GPUs and {cpus} CPUs for this app") + })) } /// Places a queued `start` of an app the scheduler placed, as `place_app` places a new diff --git a/crates/fleet-scheduler/src/workloads_tests.rs b/crates/fleet-scheduler/src/workloads_tests.rs index 22d0b4c..e7ffb37 100644 --- a/crates/fleet-scheduler/src/workloads_tests.rs +++ b/crates/fleet-scheduler/src/workloads_tests.rs @@ -2588,6 +2588,69 @@ async fn an_app_start_is_placed_on_a_node_with_room() { assert!(!i.deployments.contains_key("gpu-model") && !i.deployments["big-model"].disabled); } +/// An app created anchored to a machine is placed on that machine's node only, though another +/// fits it better; one anchored to a machine no node is on is placed nowhere. +#[tokio::test] +async fn an_app_created_with_a_machine_is_placed_on_that_machine_only() { + let (_dir, c, ns) = setup().await; + c.report("gpu", "session", inventory(&ns)).await.unwrap(); + let mut big = inventory(&ns); + (big.machine, big.total_gpus, big.free_gpus) = ("big".into(), 4.0, 4.0); + c.report("big", "session", big).await.unwrap(); + let create = |name: &str, machine: Option<&str>| { + let mut spec = json!({"image":"serve:1","gpus":1,"cpus":2,"port_base":50100}); + if let Some(machine) = machine { + spec["machine"] = json!(machine); + } + scale_call(c.clone(), "admin", "POST", "/apps", json!({"name":name,"namespace":ns,"spec":spec})) + }; + for (name, machine, node) in [("free", None, "gpu"), ("pinned", Some("big"), "big")] { + assert_eq!(create(name, machine).await.status(), StatusCode::CREATED); + let (placed, entry) = deployed(&c, &ns, name).await.unwrap(); + assert_eq!((placed.as_str(), entry.get("machine").and_then(Value::as_str)), (node, machine), "{name}"); + } + assert_eq!(create("empty", Some("")).await.status(), StatusCode::BAD_REQUEST); + assert_eq!(create("nowhere", Some("gone")).await.status(), StatusCode::CREATED); + let refused = deployed(&c, &ns, "nowhere").await.unwrap_err(); + assert!(refused.to_string().contains("no ready node on machine gone has 1 GPUs"), "{refused}"); +} + +/// Anchoring a running app to the machine it runs on has its node write the anchor, with no +/// restart; a start of the app then waits for room on that node instead of moving to another. +#[tokio::test] +async fn an_app_anchored_by_an_update_restarts_only_on_its_machine() { + let (_dir, c, mut nodes) = running_on_a_full_node(None).await; + let ns = c.inner.lock().apps["model"].namespace.clone(); + let settings = format!("/apps/model/settings?namespace={ns}"); + for (body, status) in [(json!({"machine":"big"}), StatusCode::CONFLICT), + (json!({"machine":"machine","image":"serve:2"}), StatusCode::BAD_REQUEST), (json!({"machine":1}), StatusCode::BAD_REQUEST)] { + let response = scale_call(c.clone(), "admin", "PUT", &settings, body.clone()).await; + assert_eq!(response.status(), status, "{body}"); + } + let open = |c: &WorkloadController| c.inner.lock().commands.values().filter(|v| !v.done) + .map(|v| (v.worker.clone(), v.operation, v.parameters.0.clone())).collect::>(); + assert!(open(&c).is_empty(), "a refused anchor sends nothing"); + assert_eq!(scale_call(c.clone(), "admin", "PUT", &settings, json!({"machine":"machine"})).await.status(), StatusCode::ACCEPTED); + assert_eq!(open(&c), [("gpu".to_string(), NodeOperation::App as i32, json!({"entry":{"name":"model"},"set":{"machine":"machine"}}))], + "only the anchor is written: the app isn't drained or restarted"); + assert_eq!(c.inner.lock().apps["model"].spec["machine"], json!("machine"), "placing it anew follows the anchor"); + nodes_run_commands(&c, &mut nodes, &[]).await; + assert_eq!((nodes["gpu"].apps[0]["machine"].clone(), nodes["gpu"].apps[0]["ready"].clone()), (json!("machine"), json!(true))); + assert_eq!(c.inner.lock().deployments["gpu-model"].entry().machine(), "machine"); + + // Its replica stops while a job fills its node, and node `big` has room. + let gpu = nodes.get_mut("gpu").unwrap(); + (gpu.apps[0]["ready"], gpu.apps[0]["cleaned"]) = (json!(false), json!(true)); + for (id, report) in &nodes { + c.report(id, "session", report.clone()).await.unwrap(); + } + nodes_run_commands(&c, &mut nodes, &[]).await; + assert_eq!(created_on(&c), ["gpu"], "no other node is sent it"); + assert!(nodes["big"].apps.is_empty()); + assert_eq!(scale_call(c.clone(), "admin", "PUT", &settings, json!({"machine":""})).await.status(), StatusCode::ACCEPTED); + assert!(open(&c).iter().any(|(w, _, p)| w == "gpu" && p["set"] == json!({"machine":""})), "\"\" lifts the anchor: {:?}", open(&c)); +} + /// App `model` runs on node `gpu`, its entry anchored to `machine` when given; a job then has /// the node's other card, and node `big` has room. async fn running_on_a_full_node(machine: Option<&str>) -> (tempfile::TempDir, Arc, BTreeMap) { @@ -2795,6 +2858,35 @@ async fn an_app_a_node_refuses_to_create_is_placed_again_on_the_next_enable() { let refused = tokio::time::timeout(std::time::Duration::from_secs(5), task).await.expect("enabling answers").unwrap().unwrap_err(); assert!(refused.to_string().contains("could not create the app: apps.yaml is not writable"), "{refused}"); assert!(c.inner.lock().apps["model"].workers.is_empty()); + // A node that fits it less tightly takes it next, as one with the volumes it lacks there would. + let mut big = inventory(&ns); + (big.machine, big.total_gpus, big.free_gpus) = ("big".into(), 4.0, 4.0); + c.report("big", "session", big).await.unwrap(); + assert_eq!(deployed(&c, &ns, "model").await.unwrap().0, "big"); +} + +#[tokio::test] +async fn a_node_that_refused_an_app_is_picked_again_when_no_other_can_take_it() { + let (_dir, c, ns) = setup().await; + c.report("gpu", "session", inventory(&ns)).await.unwrap(); + let body = json!({"name":"model","namespace":ns,"spec":{"image":"serve:1","gpus":1,"port_base":50100}}); + assert_eq!(scale_call(c.clone(), "admin", "POST", "/apps", body).await.status(), StatusCode::CREATED); + let task = tokio::spawn({ + let (c, ns) = (c.clone(), ns.clone()); + async move { c.deploy(&ns, "model").await } + }); + let create = loop { + let sent = c.inner.lock().commands.values().find(|v| !v.done && !v.parameters["create"].is_null()).cloned(); + if let Some(sent) = sent { + break sent; + } + tokio::task::yield_now().await; + }; + let mut report = inventory(&ns); + report.results = vec![CommandResult { id: create.id, result: Value::Null.into(), error: Some("apps.yaml is not writable".into()) }]; + c.report("gpu", "session", report).await.unwrap(); + assert!(task.await.unwrap().is_err()); + assert_eq!(deployed(&c, &ns, "model").await.unwrap().0, "gpu", "a failure may pass"); } #[tokio::test] diff --git a/crates/fleetctl/src/main.rs b/crates/fleetctl/src/main.rs index 8975c64..d69997a 100644 --- a/crates/fleetctl/src/main.rs +++ b/crates/fleetctl/src/main.rs @@ -114,9 +114,14 @@ enum AppCmd { }, /// Change a proof or persistent app's settings. Image and environment changes require /// administrator credentials. The app drains, stops, - /// takes the new settings on its node and starts again, like a scale. + /// takes the new settings on its node and starts again, like a scale; --machine alone + /// doesn't restart it. Update { name: String, + /// Anchor a container app to MACHINE, the machine it runs on (`fleetctl node` lists + /// it), so that its start waits there and never moves it to another node, as for an + /// app that keeps its data on that node's disks. "" lifts the anchor. Set on its own. + #[arg(long, value_name = "MACHINE")] machine: Option, /// Container image. #[arg(long)] image: Option, /// K=V; repeat. Replaces the app's whole environment. @@ -469,9 +474,9 @@ struct AddApp { /// A JSON file with the app's container settings, the keys of its apps.yaml entry: kind /// (app, the default, or ziren-gpu/ziren-cpu for a prover), image, replicas, gpus, cpus, /// shm_gb, memory_gb, env, volumes, ready_timeout_s, stop_timeout_s; for an app also command, - /// port_base, health_path, workdir, user; for a prover proof_storage_root, backend, rust_log. - /// image is required, and port_base for an app; user, volumes and proof_storage_root need - /// administrator credentials. + /// port_base, health_path, workdir, user, machine (anchors it to that machine: placed there, + /// never moved); for a prover proof_storage_root, backend, rust_log. image is required, and + /// port_base for an app; user, volumes and proof_storage_root need administrator credentials. #[arg(long)] spec: PathBuf, /// A prover's program: its full ELF SHA-256. @@ -1261,8 +1266,9 @@ fn app(p: &AppArgs, out: &OutputWriter) -> anyhow::Result<()> { Some(AppCmd::Priority { name, priority }) => return set_app_priority(name, *priority, &p.conn, out), Some(AppCmd::Logs { name, lines, node }) => return app_logs(name, *lines, node.as_deref(), &p.conn, out), Some(AppCmd::Scale { name, gpus, cpus, replicas, node }) => return scale_app(name, *gpus, *cpus, *replicas, node.as_deref(), &p.conn, out), - Some(AppCmd::Update { name, image, env, shm_gb, node }) => { + Some(AppCmd::Update { name, machine, image, env, shm_gb, node }) => { let mut set = json!({}); + if let Some(v) = machine { set["machine"] = json!(v); } if let Some(v) = image { set["image"] = json!(v); } if !env.is_empty() { set["env"] = env_map(env)?; } if let Some(v) = shm_gb { set["shm_gb"] = json!(v); } diff --git a/crates/fleetctl/tests/apps.rs b/crates/fleetctl/tests/apps.rs index bc4904a..1303e19 100644 --- a/crates/fleetctl/tests/apps.rs +++ b/crates/fleetctl/tests/apps.rs @@ -195,6 +195,8 @@ fn app_commands_address_every_app_by_its_name() { r#"{"state":"draining","replicas":2,"gpus":2,"cpus":16,"nodes":["gpu-box"]}"#), (vec!["update", "comfyui", "--image", "comfyui:2"], "PUT", "/settings", serde_json::json!({"image":"comfyui:2"}), r#"{"state":"draining","node":"gpu-box"}"#), + (vec!["update", "sam3", "--machine", "10-60-85-215"], "PUT", "/settings", serde_json::json!({"machine":"10-60-85-215"}), + r#"{"state":"running","node":"gpu-box"}"#), ] { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let url = format!("http://{}", listener.local_addr().unwrap()); diff --git a/docs/reference.md b/docs/reference.md index 6e89d08..8476332 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -129,7 +129,7 @@ the app's `apps.yaml` entry; unknown or mistyped settings are refused. |---|---| | `kind`: `app` by default, `ziren-gpu` by default for a prover, or `ziren-cpu` | Every app | | `image` (required), `replicas`, `gpus`, `cpus`, `shm_gb`, `memory_gb`, `env`, `volumes`, `ready_timeout_s`, `stop_timeout_s` | Every app | -| `command`, `port_base` (required; replica i serves on `port_base` + i), `health_path`, `workdir`, `user` | A `kind: app` container | +| `command`, `port_base` (required; replica i serves on `port_base` + i), `health_path`, `workdir`, `user`, `machine` (anchors it: see [Moves](#apps)) | A `kind: app` container | | `proof_storage_root`, `backend`, `rust_log` | A prover | A prover also takes its program, `--elf-id ID` or `--elf FILE`, and a placement @@ -151,16 +151,18 @@ the others and their jobs intact. In the API: `{"name":"eth","spec":{"image":"IMAGE","gpus":1},"proof":{"elf_id":"FULL_ELF_SHA256"}}`. **Placement.** An app is created disabled and on no node. Its first `app enable` -places it; users never choose the node. Candidates are live, ready, undrained nodes +places it; users never choose the node, though an app anchored to a `machine` goes to +that machine's node only. Candidates are live, ready, undrained nodes whose total capacity holds the GPUs and CPUs of all its replicas. The scheduler prefers a node where they're free now, and among those the one with the fewest free GPUs; otherwise it takes the one with the most free GPUs, where the app waits for room. With no node big enough, enabling answers 503. The chosen node gets a `create`, which writes the entry into its `apps.yaml`, -disabled. Once the node reports it, within 120 seconds, the scheduler enables it and +disabled. A node refuses the create when a host path of the app's `volumes` isn't +there: Docker would mount an empty directory in its place. Once the node reports it, within 120 seconds, the scheduler enables it and admits it against the namespace quota. A failed create clears the placement, so the -next enable places the app again. If the node hasn't reported the entry in time, +next enable places the app again, on another node when one can take it. If the node hasn't reported the entry in time, enabling answers 503, and enabling again waits for the same create. An app that a node's `apps.yaml` already runs, such as an integrated worker's prover bound with `app: NAME`, isn't placed again. @@ -175,7 +177,11 @@ anywhere, the start preempts lower-priority work on its own node and keeps the r that frees. A start whose entry is anchored with `machine` never moves; every integrated worker's entry is. It waits for room on its node, preempting lower-priority batch jobs of its namespace when it can. Fleet copies nothing an app -keeps on a node's disks: the app looks after its own storage. +keeps on a node's disks: the app looks after its own storage. Anchor an app that keeps +data there, or runs from it: with `machine` in its spec, or with +`fleetctl app update NAME --machine MACHINE` (`{"machine":"MACHINE"}`), which anchors it +to the machine it runs on (`fleetctl node` lists it) without restarting it. `""` lifts +the anchor; give `--node` when the app runs on several nodes. **States.** An app is `configured`, `starting`, `running`, `draining`, `disabled`, `offline` or `failed`; a proof app without provers is `registered`. The table shows @@ -195,7 +201,8 @@ removes the app. An active proof can't be resized ([issue #3](https://github.com/ProjectZKM/zkm-fleet/issues/3)). `fleetctl app update NAME` (`PUT /apps/NAME/settings`) changes the image, environment, command, user, workdir, volumes, shm and memory sizes, health path or -timeouts. The image, environment, command, user, workdir and volumes need an +timeouts, and drains and restarts the app to do so; `--machine` (see Moves) is set on +its own and restarts nothing. The image, environment, command, user, workdir and volumes need an administrator, since an app can reach host networking, storage and prover credentials; an update mixing them with other fields is refused whole. Changing priority alone doesn't restart replicas. From ba8c9f6176f93dd79d43de9ac358eb0fbc6e538f Mon Sep 17 00:00:00 2001 From: eigmax Date: Sat, 14 Nov 2026 12:48:14 +0000 Subject: [PATCH 2/5] Write an app's list settings right when its items aren't indented, and never write a node configuration that doesn't read back A settings update writes each new value in flow style where the old one was. For a block list or map, the old value starts at its first item on the next line, so the new list went there, at that line's indentation. With a list's "- " items level with the entry's keys (valid YAML, and how hand-written entries often look), the new line read as a stray key: the update of an app's volumes, command and workdir failed, and an update of just its volumes or command would have written a broken apps.yaml. A new block list or map value now goes on its key's own line, after the colon, keeping a comment there. And the file is replaced only when the new text parses and holds the values asked for; otherwise the update fails and the file stays as it was. --- bridge/fleet_bridge/node.py | 21 +++++++++++++++++++-- bridge/tests/test_node.py | 24 ++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/bridge/fleet_bridge/node.py b/bridge/fleet_bridge/node.py index 9495f5d..45bae95 100644 --- a/bridge/fleet_bridge/node.py +++ b/bridge/fleet_bridge/node.py @@ -69,7 +69,7 @@ def set_fields(path: Path, section: str, name: str, fields: dict[str, Any]) -> N entry = _entry(yaml.compose(text), section, name) if not isinstance(entry, yaml.MappingNode): raise ValueError(f"{section}: no entry {name}") - value = next((v for k, v in entry.value if k.value == key), None) + name_node, value = next(((k, v) for k, v in entry.value if k.value == key), (None, None)) written = json.dumps(setting) if value is not None: # A block value runs on to the next key; it ends with its last line of content. @@ -78,13 +78,30 @@ def set_fields(path: Path, section: str, name: str, fields: dict[str, Any]) -> N offset += len(line) if _is_content(line): end = offset - (1 if line.endswith("\n") else 0) - text = text[:value.start_mark.index] + written + text[end:] + start = value.start_mark.index + if isinstance(value, yaml.CollectionNode) and not value.flow_style: + # A block list or map starts on the next line, and a list's items may sit at + # the key's own indentation, so the new value goes right after the key's colon. + colon = text.index(":", name_node.end_mark.index) + 1 + rest = text[colon:text.index("\n", colon)].strip() + written = " " + written + (f" {rest}" if rest else "") + start = colon + text = text[:start] + written + text[end:] elif entry.flow_style: position = entry.end_mark.index - 1 text = text[:position] + f", {key}: {written}" + text[position:] else: position = entry.start_mark.index text = text[:position] + f"{key}: {written}\n" + " " * entry.start_mark.column + text[position:] + # The file is replaced only with text that reads back as the values asked for. + try: + data = (yaml.safe_load(text) or {}).get(section) + except yaml.YAMLError as error: + raise ValueError(f"{section}: writing {', '.join(fields)} into {name} would break {path.name}: {error}") + got = (next((e for e in data if isinstance(e, dict) and e.get("name") == name), None) + if isinstance(data, list) else (data or {}).get(name)) + if not isinstance(got, dict) or any(got.get(k) != v for k, v in fields.items()): + raise ValueError(f"{section}: {name} doesn't read back {', '.join(fields)} as written") _replace(path, text) diff --git a/bridge/tests/test_node.py b/bridge/tests/test_node.py index db0e3c2..2937416 100644 --- a/bridge/tests/test_node.py +++ b/bridge/tests/test_node.py @@ -222,6 +222,30 @@ def test_resource_update_preserves_comments_and_handles_flow_yaml(tmp_path): assert data["image"]=="image:latest" and "# machine config" in p.read_text() +@pytest.mark.parametrize("lists",["- "," - "]) +def test_settings_update_replaces_block_lists_and_maps_however_indented(tmp_path,lists): + p=tmp_path/"apps.yaml" + p.write_text(f"apps:\n- name: app\n kind: app\n volumes: # mounts\n {lists}/old/a\n {lists}/old/b:ro\n" + f" env:\n HOME: /home/old\n command:\n {lists}python\n {lists}main.py\n image: image:latest\n" + "- name: other\n kind: app\n volumes:\n - /keep\n") + for fields in [{"volumes":["/new/a"]},{"command":["/new/python","main.py"]},{"env":{"HOME":"/home/new"}}, + {"volumes":["/new/b"],"command":["/new/b/python"],"workdir":"/new/b"}]: + node.set_fields(p,"apps","app",fields) + apps=yaml.safe_load(p.read_text())["apps"] + assert {k:apps[0][k] for k in fields}==fields and apps[0]["image"]=="image:latest" + assert apps[1]=={"name":"other","kind":"app","volumes":["/keep"]} + assert "# mounts" in p.read_text() + + +def test_settings_update_never_writes_a_file_that_does_not_read_back(tmp_path,monkeypatch): + p=tmp_path/"apps.yaml";p.write_text("apps:\n- name: app\n kind: app\n volumes:\n - /old\n") + before=p.read_bytes() + monkeypatch.setattr(node.json,"dumps",lambda value:"[unclosed") + with pytest.raises(ValueError,match="would break"): + node.set_fields(p,"apps","app",{"volumes":["/new"]}) + assert p.read_bytes()==before and not list(tmp_path.glob("*.node-tmp")) + + APPS = """# machine config From 8ea0df30928139ec90e4372d96a291a151fae09b Mon Sep 17 00:00:00 2001 From: eigmax Date: Sat, 14 Nov 2026 12:59:56 +0000 Subject: [PATCH 3/5] Leave out the volume check: an app stays with its files by being anchored to its machine A node no longer refuses to create an app whose volume paths it doesn't have, and a first placement whose create was refused goes to the best-fitting node again, as before, not first to another one. An app that runs from or keeps data on a node's disks is anchored to that node's machine instead, with `machine` in its spec or with `fleetctl app update NAME --machine MACHINE`. --- bridge/fleet_bridge/node.py | 11 ++----- bridge/tests/test_node.py | 17 ----------- crates/fleet-scheduler/src/workloads_scale.rs | 8 +---- crates/fleet-scheduler/src/workloads_tests.rs | 29 ------------------- docs/reference.md | 5 ++-- 5 files changed, 5 insertions(+), 65 deletions(-) diff --git a/bridge/fleet_bridge/node.py b/bridge/fleet_bridge/node.py index 45bae95..15b91e8 100644 --- a/bridge/fleet_bridge/node.py +++ b/bridge/fleet_bridge/node.py @@ -430,9 +430,7 @@ def app(self, p: dict[str, Any]) -> tuple[dict[str, Any], Any]: (`p["entry"]`, by name, set only): `remove` it, or `set` some of its configuration. Settings are written to the configuration only: the scheduler drains and stops an app first and starts it again afterwards. `p["create"]` adds a new app's - entry to the configuration, disabled until the app is enabled. It is refused when a - host path of its volumes isn't here: what an app keeps on a node's disks stays on - that node, and Docker would mount a missing path as a new, empty directory. + entry to the configuration, disabled until the app is enabled. A removed entry has what is left of its replicas cleaned up the way disabling does. Removing an app from this node alone (a scale left it no replicas here) keeps its @@ -446,15 +444,10 @@ def app(self, p: dict[str, Any]) -> tuple[dict[str, Any], Any]: nothing = lambda: {} if "create" in p: entry = p["create"] - spec = AppSpec.load(entry) + AppSpec.load(entry) entries = (yaml.safe_load(self.apps_file.read_text()) or {}).get("apps") or [] if any(e.get("name") == entry["name"] for e in entries): raise ValueError(f"{entry['name']} is already configured on this node") - missing = [path for path in (v.split(":")[0] for v in spec.volumes) - if path.startswith("/") and not os.path.exists(path)] - if missing: - raise ValueError(f"{entry['name']} can't run on this node: {', '.join(missing)} isn't here, " - "and Fleet copies nothing an app keeps on a node's disks") append_entry(self.apps_file, "apps", entry) return {"created": [entry["name"]]}, nothing if "batch" in p: diff --git a/bridge/tests/test_node.py b/bridge/tests/test_node.py index 2937416..4aaad83 100644 --- a/bridge/tests/test_node.py +++ b/bridge/tests/test_node.py @@ -592,23 +592,6 @@ def test_create_adds_a_disabled_app_and_refuses_a_taken_name(executor): assert error and "already configured" in str(error) -def test_create_refuses_an_app_whose_volumes_are_not_on_this_node(executor,tmp_path): - """Its data stays on the node it ran on; Docker would mount an empty directory in its place.""" - here=tmp_path/"models";here.mkdir() - gone=tmp_path/"ComfyUI" - entry={"name":"web","kind":"app","namespace":"a"*32,"disabled":True,"image":"web:1","port_base":50100, - "volumes":[f"{here}:ro",str(gone)]} - before=executor.apps_file.read_text() - result,error=app_message(executor,"create-1",{"create":entry},app="web",ns="a"*32) - assert error and f"web can't run on this node: {gone} isn't here" in str(error), error - assert executor.apps_file.read_text()==before and not gone.exists() - gone.symlink_to(tmp_path/"elsewhere") - result,error=app_message(executor,"create-2",{"create":entry},app="web",ns="a"*32) - assert error and "isn't here" in str(error), "a link to nothing is not there either" - gone.unlink();gone.mkdir() - assert app_message(executor,"create-3",{"create":entry},app="web",ns="a"*32)==({"created":["web"]},None) - - def test_a_report_carries_a_create_done_only_with_its_entry(executor,monkeypatch): """A create that finishes while a report is read is reported with its entry, in the next one.""" monkeypatch.setattr(node.ray,"cluster_resources",lambda:{"GPU":8,"CPU":48}) diff --git a/crates/fleet-scheduler/src/workloads_scale.rs b/crates/fleet-scheduler/src/workloads_scale.rs index 1346cd6..09dd34d 100644 --- a/crates/fleet-scheduler/src/workloads_scale.rs +++ b/crates/fleet-scheduler/src/workloads_scale.rs @@ -873,13 +873,7 @@ impl WorkloadController { match pending { Some(pending) => pending, None => { - // A node that refused to create it, as one without its volumes does, is - // picked again only when no other can take it. - let refused: Vec = self.inner.lock().commands.values() - .filter(|c| c.is(NodeOperation::App) && c.error.is_some() && c.created().is_some_and(|e| e.name() == app)) - .map(|c| c.worker.clone()).collect(); - let (node, _) = self.place_app(template.entry(), &refused) - .or_else(|_| self.place_app(template.entry(), &[]))?; + let (node, _) = self.place_app(template.entry(), &[])?; let create = message::with(self.app_message(namespace, app), &message::create(&template.spec)); let create = self.command(&node, NodeOperation::App, create)?; template.workers = vec![node.clone()]; diff --git a/crates/fleet-scheduler/src/workloads_tests.rs b/crates/fleet-scheduler/src/workloads_tests.rs index e7ffb37..4b25f9f 100644 --- a/crates/fleet-scheduler/src/workloads_tests.rs +++ b/crates/fleet-scheduler/src/workloads_tests.rs @@ -2858,35 +2858,6 @@ async fn an_app_a_node_refuses_to_create_is_placed_again_on_the_next_enable() { let refused = tokio::time::timeout(std::time::Duration::from_secs(5), task).await.expect("enabling answers").unwrap().unwrap_err(); assert!(refused.to_string().contains("could not create the app: apps.yaml is not writable"), "{refused}"); assert!(c.inner.lock().apps["model"].workers.is_empty()); - // A node that fits it less tightly takes it next, as one with the volumes it lacks there would. - let mut big = inventory(&ns); - (big.machine, big.total_gpus, big.free_gpus) = ("big".into(), 4.0, 4.0); - c.report("big", "session", big).await.unwrap(); - assert_eq!(deployed(&c, &ns, "model").await.unwrap().0, "big"); -} - -#[tokio::test] -async fn a_node_that_refused_an_app_is_picked_again_when_no_other_can_take_it() { - let (_dir, c, ns) = setup().await; - c.report("gpu", "session", inventory(&ns)).await.unwrap(); - let body = json!({"name":"model","namespace":ns,"spec":{"image":"serve:1","gpus":1,"port_base":50100}}); - assert_eq!(scale_call(c.clone(), "admin", "POST", "/apps", body).await.status(), StatusCode::CREATED); - let task = tokio::spawn({ - let (c, ns) = (c.clone(), ns.clone()); - async move { c.deploy(&ns, "model").await } - }); - let create = loop { - let sent = c.inner.lock().commands.values().find(|v| !v.done && !v.parameters["create"].is_null()).cloned(); - if let Some(sent) = sent { - break sent; - } - tokio::task::yield_now().await; - }; - let mut report = inventory(&ns); - report.results = vec![CommandResult { id: create.id, result: Value::Null.into(), error: Some("apps.yaml is not writable".into()) }]; - c.report("gpu", "session", report).await.unwrap(); - assert!(task.await.unwrap().is_err()); - assert_eq!(deployed(&c, &ns, "model").await.unwrap().0, "gpu", "a failure may pass"); } #[tokio::test] diff --git a/docs/reference.md b/docs/reference.md index 8476332..840d658 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -159,10 +159,9 @@ GPUs; otherwise it takes the one with the most free GPUs, where the app waits fo room. With no node big enough, enabling answers 503. The chosen node gets a `create`, which writes the entry into its `apps.yaml`, -disabled. A node refuses the create when a host path of the app's `volumes` isn't -there: Docker would mount an empty directory in its place. Once the node reports it, within 120 seconds, the scheduler enables it and +disabled. Once the node reports it, within 120 seconds, the scheduler enables it and admits it against the namespace quota. A failed create clears the placement, so the -next enable places the app again, on another node when one can take it. If the node hasn't reported the entry in time, +next enable places the app again. If the node hasn't reported the entry in time, enabling answers 503, and enabling again waits for the same create. An app that a node's `apps.yaml` already runs, such as an integrated worker's prover bound with `app: NAME`, isn't placed again. From 120809ffb926851f38e77b3aa6223f5b5e0ed874 Mon Sep 17 00:00:00 2001 From: eigmax Date: Sat, 14 Nov 2026 13:08:08 +0000 Subject: [PATCH 4/5] Leave out anchoring through the API: an app is anchored only in its node's apps.yaml Neither `machine` in a new app's spec nor `fleetctl app update NAME --machine` anchors an app any more. As before, an app is anchored by `machine` in its entry in its node's apps.yaml; the reference now says so where it explains that Fleet copies nothing an app keeps on a node's disks. --- crates/fleet-mcp/src/main.rs | 5 +- crates/fleet-proto/src/models.rs | 15 ++--- crates/fleet-scheduler/src/workloads_scale.rs | 59 ++--------------- crates/fleet-scheduler/src/workloads_tests.rs | 63 ------------------- crates/fleetctl/src/main.rs | 16 ++--- crates/fleetctl/tests/apps.rs | 2 - docs/reference.md | 14 ++--- 7 files changed, 21 insertions(+), 153 deletions(-) diff --git a/crates/fleet-mcp/src/main.rs b/crates/fleet-mcp/src/main.rs index a5b548e..0f2c509 100644 --- a/crates/fleet-mcp/src/main.rs +++ b/crates/fleet-mcp/src/main.rs @@ -218,9 +218,8 @@ struct CreateApp { /// The app's container settings, the keys of its apps.yaml entry: kind (app, the default, /// or ziren-gpu/ziren-cpu for a prover), image, replicas, gpus, cpus, shm_gb, memory_gb, env, /// volumes, ready_timeout_s, stop_timeout_s; for an app also command, port_base, health_path, - /// workdir, user, machine (anchors it to that machine: placed there, never moved); for a - /// prover proof_storage_root, backend, rust_log. image is required, and port_base for an app; - /// user, volumes and proof_storage_root require administrator credentials. + /// workdir, user; for a prover proof_storage_root, backend, rust_log. image is required, and + /// port_base for an app; user, volumes and proof_storage_root require administrator credentials. spec: serde_json::Map, /// Priority inherited by the app's replicas and jobs (default 0; higher values run first). priority: Option, diff --git a/crates/fleet-proto/src/models.rs b/crates/fleet-proto/src/models.rs index 5a00391..da28e5c 100644 --- a/crates/fleet-proto/src/models.rs +++ b/crates/fleet-proto/src/models.rs @@ -262,9 +262,8 @@ impl RegisterAppRequest { /// priority and program, and creates it disabled. pub const SPEC_SETTINGS: [&'static str; 11] = ["kind", "image", "replicas", "gpus", "cpus", "shm_gb", "memory_gb", "env", "volumes", "ready_timeout_s", "stop_timeout_s"]; - /// Settings only a container app's spec (`kind: app`) may give. `machine` anchors it to - /// that machine: it is placed there, and never moved to another node. - pub const CONTAINER_SETTINGS: [&'static str; 6] = ["command", "port_base", "health_path", "workdir", "user", "machine"]; + /// Settings only a container app's spec (`kind: app`) may give. + pub const CONTAINER_SETTINGS: [&'static str; 5] = ["command", "port_base", "health_path", "workdir", "user"]; /// Settings only a prover's spec (`kind: ziren-gpu` or `ziren-cpu`) may give. pub const PROVER_SETTINGS: [&'static str; 3] = ["proof_storage_root", "backend", "rust_log"]; @@ -312,9 +311,8 @@ impl RegisterAppRequest { } let strings = |v: &Value| v.as_array().is_none_or(|a| a.iter().any(|v| !v.is_string())); let positive = |v: &Value| v.as_f64().is_none_or(|v| v <= 0.0 || !v.is_finite()); - let invalid = ["user", "health_path", "workdir", "proof_storage_root", "backend", "rust_log", "machine"].into_iter() + let invalid = ["user", "health_path", "workdir", "proof_storage_root", "backend", "rust_log"].into_iter() .find(|k| set(k).is_some_and(|v| !v.is_string())) - .or_else(|| set("machine").and_then(Value::as_str).is_some_and(str::is_empty).then_some("machine")) .or_else(|| ["cpus", "shm_gb", "port_base", "stop_timeout_s"].into_iter().find(|k| set(k).is_some_and(|v| v.as_u64().is_none()))) .or_else(|| ["memory_gb", "ready_timeout_s"].into_iter().find(|k| set(k).is_some_and(positive))) .or_else(|| ["command", "volumes"].into_iter().find(|k| set(k).is_some_and(strings))) @@ -341,10 +339,7 @@ mod tests { }; let container = json!({"image":"serve:1","port_base":50100,"gpus":2,"replicas":1,"command":["serve","{port}"],"env":{"A":"1"}}); app(json!({"name":"model","spec":container})).unwrap(); - let mut anchored = container.clone(); - anchored["machine"] = json!("gpu-box"); - app(json!({"name":"model","spec":anchored})).unwrap(); - for (key, value) in [("machine", json!("")), ("machine", json!(1)), ("gpus", json!("2")), ("replicas", json!(0)), ("port_base", json!(-1)), ("port_base", Value::Null), + for (key, value) in [("gpus", json!("2")), ("replicas", json!(0)), ("port_base", json!(-1)), ("port_base", Value::Null), ("image", Value::Null), ("workdir", json!(1)), ("env", json!({"A":1})), ("kind", json!("vm")), ("proof_storage_root", json!("/proofs")), ("name", json!("x"))] { let mut spec = container.clone(); @@ -359,8 +354,6 @@ mod tests { assert!(app(json!({"name":"model","spec":container,"proof":proof})).is_err(), "a container app proves none"); assert!(app(json!({"name":"eth","spec":{"image":"ziren-gpu:1","port_base":1},"proof":proof})).is_err(), "a prover takes no container settings"); - assert!(app(json!({"name":"eth","spec":{"image":"ziren-gpu:1","machine":"gpu-box"},"proof":proof})).is_err(), - "a prover's node configures its machine"); assert!(app(json!({"name":"eth","elf_id":"ab".repeat(32)})).is_err(), "an app is created from its spec"); assert!(app(json!({"name":"model","node":"gpu","spec":container})).is_err(), "the scheduler picks the node"); } diff --git a/crates/fleet-scheduler/src/workloads_scale.rs b/crates/fleet-scheduler/src/workloads_scale.rs index 09dd34d..a367286 100644 --- a/crates/fleet-scheduler/src/workloads_scale.rs +++ b/crates/fleet-scheduler/src/workloads_scale.rs @@ -238,9 +238,8 @@ impl WorkloadController { /// Settings an update may change on an app. The rest is identity, placement or the /// node's own configuration, or has its own operation (scale, enable/disable, priority). - /// `machine` anchors a container app to the machine it runs on, and changes nothing else. - pub(super) const APP_SETTINGS: [&'static str; 12] = ["image", "env", "command", "user", "workdir", "volumes", - "shm_gb", "memory_gb", "health_path", "stop_timeout_s", "ready_timeout_s", "machine"]; + pub(super) const APP_SETTINGS: [&'static str; 11] = ["image", "env", "command", "user", "workdir", "volumes", + "shm_gb", "memory_gb", "health_path", "stop_timeout_s", "ready_timeout_s"]; // Apps can access host networking, storage and prover credentials. Changing // their executable or environment is as privileged as changing their mounts. pub(super) const ADMIN_APP_SETTINGS: [&'static str; 6] = ["image", "env", "command", "user", "workdir", "volumes"]; @@ -311,13 +310,6 @@ impl WorkloadController { let _ops = self.ops.lock().await; let set = set.as_object().filter(|o| !o.is_empty()).cloned() .ok_or_else(|| fail(400, "set at least one setting"))?; - if let Some(machine) = set.get("machine") { - if set.len() > 1 { - return Err(fail(400, "set machine on its own: anchoring an app changes nothing else and doesn't restart it")); - } - let machine = machine.as_str().ok_or_else(|| fail(400, "machine: invalid value"))?; - return self.anchor(namespace, app, node, machine); - } for (key, value) in &set { let valid = match key.as_str() { "image" => value.as_str().is_some_and(|v| !v.is_empty() && !v.starts_with('-')), @@ -351,40 +343,6 @@ impl WorkloadController { Ok(self.scale_value(&intent)) } - /// Anchors `app`'s deployment on `node` (its only one, without a node) to `machine`, the - /// machine it runs on, or lifts the anchor with "". A start of an anchored app waits for - /// room on its node and is never moved to another, so an app that keeps its data on its - /// node's disks stays with it. Only the node's configuration changes: the app runs on. - /// The caller holds the operation lock. - fn anchor(&self, namespace: &str, app: &str, node: Option<&str>, machine: &str) -> anyhow::Result { - let (key, mut intent) = self.deployment_to_change(namespace, app, node)?; - if !intent.entry().is_container() { - return Err(fail(400, "only a container app is anchored this way: a prover's node configures its own machine")); - } - let on = self.inner.lock().workers.get(&intent.worker).map(|w| w.report.machine.clone()).unwrap_or_default(); - if !machine.is_empty() && machine != on { - return Err(fail(409, format!( - "{app} runs on node {} (machine {on}): it can be anchored only to the machine it runs on", intent.worker))); - } - if intent.entry().machine() == machine { - return Ok(self.scale_value(&intent)); - } - let name = intent.entry().name().to_owned(); - self.tell_nodes([&intent.worker], message::entry(&name), message::set(json!({"machine": machine})))?; - intent.spec["machine"] = json!(machine); - self.save("deployments", &key, &intent)?; - self.inner.lock().deployments.insert(key, intent.clone()); - // Placing the app anew follows the anchor too. - let template = self.inner.lock().apps.get(app).filter(|a| a.persistent && !a.removed && a.namespace == namespace).cloned(); - if let Some(mut template) = template { - template.spec["machine"] = json!(machine); - self.save("apps", app, &template)?; - self.inner.lock().apps.insert(app.into(), template); - } - tracing::info!(app, node = intent.worker, machine, "anchoring an app to the machine it runs on"); - Ok(self.scale_value(&intent)) - } - /// Sets an app's replica count, and its replicas' GPUs and CPUs: the app's settings, not any /// node's. The count is split over the app's deployments in the order its replicas are kept /// in: each keeps the replicas it has, up to the count; new ones go to the first, whose start @@ -917,14 +875,11 @@ impl WorkloadController { /// and CPUs of all its replicas. One that has them free now is preferred, the one with the /// fewest free GPUs among those, as a batch-class proof app is placed, so larger requests /// keep the larger gaps; otherwise the one with the most free GPUs, where the app waits for - /// room. Whether it has the room now comes along. `except` are nodes not to pick. An entry - /// anchored to a `machine` goes to that machine's node only. + /// room. Whether it has the room now comes along. `except` are nodes not to pick. fn place_app(&self, entry: Entry<'_>, except: &[String]) -> anyhow::Result<(String, bool)> { let (gpus, cpus) = entry.total(1.0); - let machine = entry.machine(); let large: Vec = self.inner.lock().workers.iter() - .filter(|(id, w)| !except.contains(id) && (machine.is_empty() || w.report.machine == machine) - && w.report.total_gpus + 1e-9 >= gpus && w.report.total_cpus + 1e-9 >= cpus) + .filter(|(id, w)| !except.contains(id) && w.report.total_gpus + 1e-9 >= gpus && w.report.total_cpus + 1e-9 >= cpus) .map(|(id, _)| id.clone()).collect(); let sizes: Vec<(String, f64, f64)> = large.into_iter() .filter_map(|id| self.capacity(&id).map(|(g, c)| (id, g, c))) @@ -933,11 +888,7 @@ impl WorkloadController { sizes.iter().filter(fits).min_by(|a, b| a.1.total_cmp(&b.1).then(a.2.total_cmp(&b.2))) .map(|(id, _, _)| (id.clone(), true)) .or_else(|| sizes.iter().max_by(|a, b| a.1.total_cmp(&b.1).then(b.0.cmp(&a.0))).map(|(id, _, _)| (id.clone(), false))) - .ok_or_else(|| fail(503, if machine.is_empty() { - format!("no ready node has {gpus} GPUs and {cpus} CPUs for this app") - } else { - format!("no ready node on machine {machine} has {gpus} GPUs and {cpus} CPUs for this app") - })) + .ok_or_else(|| fail(503, format!("no ready node has {gpus} GPUs and {cpus} CPUs for this app"))) } /// Places a queued `start` of an app the scheduler placed, as `place_app` places a new diff --git a/crates/fleet-scheduler/src/workloads_tests.rs b/crates/fleet-scheduler/src/workloads_tests.rs index 4b25f9f..22d0b4c 100644 --- a/crates/fleet-scheduler/src/workloads_tests.rs +++ b/crates/fleet-scheduler/src/workloads_tests.rs @@ -2588,69 +2588,6 @@ async fn an_app_start_is_placed_on_a_node_with_room() { assert!(!i.deployments.contains_key("gpu-model") && !i.deployments["big-model"].disabled); } -/// An app created anchored to a machine is placed on that machine's node only, though another -/// fits it better; one anchored to a machine no node is on is placed nowhere. -#[tokio::test] -async fn an_app_created_with_a_machine_is_placed_on_that_machine_only() { - let (_dir, c, ns) = setup().await; - c.report("gpu", "session", inventory(&ns)).await.unwrap(); - let mut big = inventory(&ns); - (big.machine, big.total_gpus, big.free_gpus) = ("big".into(), 4.0, 4.0); - c.report("big", "session", big).await.unwrap(); - let create = |name: &str, machine: Option<&str>| { - let mut spec = json!({"image":"serve:1","gpus":1,"cpus":2,"port_base":50100}); - if let Some(machine) = machine { - spec["machine"] = json!(machine); - } - scale_call(c.clone(), "admin", "POST", "/apps", json!({"name":name,"namespace":ns,"spec":spec})) - }; - for (name, machine, node) in [("free", None, "gpu"), ("pinned", Some("big"), "big")] { - assert_eq!(create(name, machine).await.status(), StatusCode::CREATED); - let (placed, entry) = deployed(&c, &ns, name).await.unwrap(); - assert_eq!((placed.as_str(), entry.get("machine").and_then(Value::as_str)), (node, machine), "{name}"); - } - assert_eq!(create("empty", Some("")).await.status(), StatusCode::BAD_REQUEST); - assert_eq!(create("nowhere", Some("gone")).await.status(), StatusCode::CREATED); - let refused = deployed(&c, &ns, "nowhere").await.unwrap_err(); - assert!(refused.to_string().contains("no ready node on machine gone has 1 GPUs"), "{refused}"); -} - -/// Anchoring a running app to the machine it runs on has its node write the anchor, with no -/// restart; a start of the app then waits for room on that node instead of moving to another. -#[tokio::test] -async fn an_app_anchored_by_an_update_restarts_only_on_its_machine() { - let (_dir, c, mut nodes) = running_on_a_full_node(None).await; - let ns = c.inner.lock().apps["model"].namespace.clone(); - let settings = format!("/apps/model/settings?namespace={ns}"); - for (body, status) in [(json!({"machine":"big"}), StatusCode::CONFLICT), - (json!({"machine":"machine","image":"serve:2"}), StatusCode::BAD_REQUEST), (json!({"machine":1}), StatusCode::BAD_REQUEST)] { - let response = scale_call(c.clone(), "admin", "PUT", &settings, body.clone()).await; - assert_eq!(response.status(), status, "{body}"); - } - let open = |c: &WorkloadController| c.inner.lock().commands.values().filter(|v| !v.done) - .map(|v| (v.worker.clone(), v.operation, v.parameters.0.clone())).collect::>(); - assert!(open(&c).is_empty(), "a refused anchor sends nothing"); - assert_eq!(scale_call(c.clone(), "admin", "PUT", &settings, json!({"machine":"machine"})).await.status(), StatusCode::ACCEPTED); - assert_eq!(open(&c), [("gpu".to_string(), NodeOperation::App as i32, json!({"entry":{"name":"model"},"set":{"machine":"machine"}}))], - "only the anchor is written: the app isn't drained or restarted"); - assert_eq!(c.inner.lock().apps["model"].spec["machine"], json!("machine"), "placing it anew follows the anchor"); - nodes_run_commands(&c, &mut nodes, &[]).await; - assert_eq!((nodes["gpu"].apps[0]["machine"].clone(), nodes["gpu"].apps[0]["ready"].clone()), (json!("machine"), json!(true))); - assert_eq!(c.inner.lock().deployments["gpu-model"].entry().machine(), "machine"); - - // Its replica stops while a job fills its node, and node `big` has room. - let gpu = nodes.get_mut("gpu").unwrap(); - (gpu.apps[0]["ready"], gpu.apps[0]["cleaned"]) = (json!(false), json!(true)); - for (id, report) in &nodes { - c.report(id, "session", report.clone()).await.unwrap(); - } - nodes_run_commands(&c, &mut nodes, &[]).await; - assert_eq!(created_on(&c), ["gpu"], "no other node is sent it"); - assert!(nodes["big"].apps.is_empty()); - assert_eq!(scale_call(c.clone(), "admin", "PUT", &settings, json!({"machine":""})).await.status(), StatusCode::ACCEPTED); - assert!(open(&c).iter().any(|(w, _, p)| w == "gpu" && p["set"] == json!({"machine":""})), "\"\" lifts the anchor: {:?}", open(&c)); -} - /// App `model` runs on node `gpu`, its entry anchored to `machine` when given; a job then has /// the node's other card, and node `big` has room. async fn running_on_a_full_node(machine: Option<&str>) -> (tempfile::TempDir, Arc, BTreeMap) { diff --git a/crates/fleetctl/src/main.rs b/crates/fleetctl/src/main.rs index d69997a..8975c64 100644 --- a/crates/fleetctl/src/main.rs +++ b/crates/fleetctl/src/main.rs @@ -114,14 +114,9 @@ enum AppCmd { }, /// Change a proof or persistent app's settings. Image and environment changes require /// administrator credentials. The app drains, stops, - /// takes the new settings on its node and starts again, like a scale; --machine alone - /// doesn't restart it. + /// takes the new settings on its node and starts again, like a scale. Update { name: String, - /// Anchor a container app to MACHINE, the machine it runs on (`fleetctl node` lists - /// it), so that its start waits there and never moves it to another node, as for an - /// app that keeps its data on that node's disks. "" lifts the anchor. Set on its own. - #[arg(long, value_name = "MACHINE")] machine: Option, /// Container image. #[arg(long)] image: Option, /// K=V; repeat. Replaces the app's whole environment. @@ -474,9 +469,9 @@ struct AddApp { /// A JSON file with the app's container settings, the keys of its apps.yaml entry: kind /// (app, the default, or ziren-gpu/ziren-cpu for a prover), image, replicas, gpus, cpus, /// shm_gb, memory_gb, env, volumes, ready_timeout_s, stop_timeout_s; for an app also command, - /// port_base, health_path, workdir, user, machine (anchors it to that machine: placed there, - /// never moved); for a prover proof_storage_root, backend, rust_log. image is required, and - /// port_base for an app; user, volumes and proof_storage_root need administrator credentials. + /// port_base, health_path, workdir, user; for a prover proof_storage_root, backend, rust_log. + /// image is required, and port_base for an app; user, volumes and proof_storage_root need + /// administrator credentials. #[arg(long)] spec: PathBuf, /// A prover's program: its full ELF SHA-256. @@ -1266,9 +1261,8 @@ fn app(p: &AppArgs, out: &OutputWriter) -> anyhow::Result<()> { Some(AppCmd::Priority { name, priority }) => return set_app_priority(name, *priority, &p.conn, out), Some(AppCmd::Logs { name, lines, node }) => return app_logs(name, *lines, node.as_deref(), &p.conn, out), Some(AppCmd::Scale { name, gpus, cpus, replicas, node }) => return scale_app(name, *gpus, *cpus, *replicas, node.as_deref(), &p.conn, out), - Some(AppCmd::Update { name, machine, image, env, shm_gb, node }) => { + Some(AppCmd::Update { name, image, env, shm_gb, node }) => { let mut set = json!({}); - if let Some(v) = machine { set["machine"] = json!(v); } if let Some(v) = image { set["image"] = json!(v); } if !env.is_empty() { set["env"] = env_map(env)?; } if let Some(v) = shm_gb { set["shm_gb"] = json!(v); } diff --git a/crates/fleetctl/tests/apps.rs b/crates/fleetctl/tests/apps.rs index 1303e19..bc4904a 100644 --- a/crates/fleetctl/tests/apps.rs +++ b/crates/fleetctl/tests/apps.rs @@ -195,8 +195,6 @@ fn app_commands_address_every_app_by_its_name() { r#"{"state":"draining","replicas":2,"gpus":2,"cpus":16,"nodes":["gpu-box"]}"#), (vec!["update", "comfyui", "--image", "comfyui:2"], "PUT", "/settings", serde_json::json!({"image":"comfyui:2"}), r#"{"state":"draining","node":"gpu-box"}"#), - (vec!["update", "sam3", "--machine", "10-60-85-215"], "PUT", "/settings", serde_json::json!({"machine":"10-60-85-215"}), - r#"{"state":"running","node":"gpu-box"}"#), ] { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let url = format!("http://{}", listener.local_addr().unwrap()); diff --git a/docs/reference.md b/docs/reference.md index 840d658..2ea5638 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -129,7 +129,7 @@ the app's `apps.yaml` entry; unknown or mistyped settings are refused. |---|---| | `kind`: `app` by default, `ziren-gpu` by default for a prover, or `ziren-cpu` | Every app | | `image` (required), `replicas`, `gpus`, `cpus`, `shm_gb`, `memory_gb`, `env`, `volumes`, `ready_timeout_s`, `stop_timeout_s` | Every app | -| `command`, `port_base` (required; replica i serves on `port_base` + i), `health_path`, `workdir`, `user`, `machine` (anchors it: see [Moves](#apps)) | A `kind: app` container | +| `command`, `port_base` (required; replica i serves on `port_base` + i), `health_path`, `workdir`, `user` | A `kind: app` container | | `proof_storage_root`, `backend`, `rust_log` | A prover | A prover also takes its program, `--elf-id ID` or `--elf FILE`, and a placement @@ -151,8 +151,7 @@ the others and their jobs intact. In the API: `{"name":"eth","spec":{"image":"IMAGE","gpus":1},"proof":{"elf_id":"FULL_ELF_SHA256"}}`. **Placement.** An app is created disabled and on no node. Its first `app enable` -places it; users never choose the node, though an app anchored to a `machine` goes to -that machine's node only. Candidates are live, ready, undrained nodes +places it; users never choose the node. Candidates are live, ready, undrained nodes whose total capacity holds the GPUs and CPUs of all its replicas. The scheduler prefers a node where they're free now, and among those the one with the fewest free GPUs; otherwise it takes the one with the most free GPUs, where the app waits for @@ -177,10 +176,8 @@ that frees. A start whose entry is anchored with `machine` never moves; every integrated worker's entry is. It waits for room on its node, preempting lower-priority batch jobs of its namespace when it can. Fleet copies nothing an app keeps on a node's disks: the app looks after its own storage. Anchor an app that keeps -data there, or runs from it: with `machine` in its spec, or with -`fleetctl app update NAME --machine MACHINE` (`{"machine":"MACHINE"}`), which anchors it -to the machine it runs on (`fleetctl node` lists it) without restarting it. `""` lifts -the anchor; give `--node` when the app runs on several nodes. +data there, or runs from it, by setting `machine` in its entry in that node's +`apps.yaml` to the node's machine, as `fleetctl node` lists it. **States.** An app is `configured`, `starting`, `running`, `draining`, `disabled`, `offline` or `failed`; a proof app without provers is `registered`. The table shows @@ -200,8 +197,7 @@ removes the app. An active proof can't be resized ([issue #3](https://github.com/ProjectZKM/zkm-fleet/issues/3)). `fleetctl app update NAME` (`PUT /apps/NAME/settings`) changes the image, environment, command, user, workdir, volumes, shm and memory sizes, health path or -timeouts, and drains and restarts the app to do so; `--machine` (see Moves) is set on -its own and restarts nothing. The image, environment, command, user, workdir and volumes need an +timeouts. The image, environment, command, user, workdir and volumes need an administrator, since an app can reach host networking, storage and prover credentials; an update mixing them with other fields is refused whole. Changing priority alone doesn't restart replicas. From 43548ae22f553877f8098d648e19f0d38410b9b3 Mon Sep 17 00:00:00 2001 From: eigmax Date: Sat, 14 Nov 2026 13:17:42 +0000 Subject: [PATCH 5/5] Say in the README that an app may move, and that one keeping data on its node is anchored in its apps.yaml --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a5068e7..4e64a66 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,11 @@ Kubernetes' jobs, deployments and controllers, without needing Kubernetes. Namespace owners create apps and batches and manage their priority, enabled state and scaling, within their namespace's and tag's quotas; the scheduler picks each app's -node. Administrators manage namespaces, quotas, access, host storage for batches and -apps' execution settings. +node, and moves an app's start to a node with room when its own has none. Fleet copies +nothing an app keeps on a node's disks, so an app that keeps data there, or runs from +it, is anchored to that node with `machine` in its entry in the node's `apps.yaml`. +Administrators manage namespaces, quotas, access, host storage for batches and apps' +execution settings. Two integrated app workers prove on their node's own GPU server: