diff --git a/bridge/fleet_bridge/node.py b/bridge/fleet_bridge/node.py index b059b5b..d98ee67 100644 --- a/bridge/fleet_bridge/node.py +++ b/bridge/fleet_bridge/node.py @@ -152,7 +152,8 @@ def remove_entry(path: Path, section: str, name: str) -> bool: def append_entry(path: Path, section: str, entry: dict[str, Any]) -> None: """Appends one entry to an `apps` list, keeping every other line as written. The entry - goes on one line in YAML flow style, under a block list, an empty list or a missing key.""" + goes on one line in YAML flow style, under a block list, an empty list or a missing key. + It follows the list's last entry, so keys after the list stay after it.""" text = path.read_text() root = yaml.compose(text) node = next((v for k, v in root.value if k.value == section), None) if root else None @@ -165,7 +166,13 @@ def append_entry(path: Path, section: str, entry: dict[str, Any]) -> None: if _is_content(line): end = offset if not text[:end].endswith("\n"): - text, end = text + "\n", len(text) + 1 + # The last entry ends mid-line, as a one-line flow entry does: the new one goes + # after the rest of that line, or after a newline added at the end of the file. + newline = text.find("\n", end) + if newline < 0: + text, end = text + "\n", len(text) + 1 + else: + end = newline + 1 text = text[:end] + " " * (last.start_mark.column - 2) + item + text[end:] elif isinstance(node, yaml.SequenceNode) and not node.value: text = text[:node.start_mark.index].rstrip(" ") + "\n " + item + text[node.end_mark.index:] diff --git a/bridge/tests/test_node.py b/bridge/tests/test_node.py index 62e20ff..4e7d2d3 100644 --- a/bridge/tests/test_node.py +++ b/bridge/tests/test_node.py @@ -450,6 +450,24 @@ def test_append_entry_keeps_every_other_line(tmp_path, text): assert line in p.read_text() +@pytest.mark.parametrize("text", [ + 'log_dir: /logs\napps:\n- name: web\n kind: app\nscheduler_admin: https://fleet:8905\n', + 'apps:\n- {"name": "web", "kind": "app"}\nscheduler_admin: https://fleet:8905\n', + 'apps:\n - {"name": "web", "kind": "app"} # placed by the scheduler\n# admin\nscheduler_admin: https://fleet:8905\n', + 'scheduler_admin: https://fleet:8905\napps:\n- {"name": "web", "kind": "app"}', +]) +def test_appended_entries_stay_in_the_list_before_the_keys_after_it(tmp_path, text): + p=tmp_path/"apps.yaml";p.write_text(text) + for name in ("first","second"): + node.append_entry(p,"apps",{"name":name,"kind":"app","namespace":"zkm","disabled":True,"image":"web:1"}) + config=yaml.safe_load(p.read_text()) + assert [a["name"] for a in config["apps"]]==["web","first","second"] + assert config["scheduler_admin"]=="https://fleet:8905" + assert {k:v for k,v in config.items() if k!="apps"}=={k:v for k,v in yaml.safe_load(text).items() if k!="apps"} + for line in text.splitlines(): + assert line in p.read_text() + + def test_create_adds_a_disabled_app_and_refuses_a_taken_name(executor): entry={"name":"web","kind":"app","namespace":"a"*32,"disabled":True,"image":"web:1","port_base":50100,"gpus":1} assert app_message(executor,"create-1",{"create":entry},app="web",ns="a"*32)==({"created":["web"]},None) diff --git a/crates/fleet-scheduler/src/workloads.rs b/crates/fleet-scheduler/src/workloads.rs index 04882c9..cfc4f8d 100644 --- a/crates/fleet-scheduler/src/workloads.rs +++ b/crates/fleet-scheduler/src/workloads.rs @@ -179,11 +179,26 @@ impl WorkloadController { /// Drops a deployment record, its allocation already released. Nothing else removes one, and a /// stale record can restart its app or claim its app's worker slot. fn forget_deployment(&self, key: &str) -> anyhow::Result<()> { - self.inner.lock().deployments.remove(key); + let Some(intent) = self.inner.lock().deployments.remove(key) else { + return Ok(()); + }; match std::fs::remove_file(self.dir.join("deployments").join(format!("{key}.json"))) { Err(e) if e.kind() != std::io::ErrorKind::NotFound => return Err(e.into()), _ => {} } + // Its node no longer runs the app, unless it has another entry of it. + let (app, namespace) = (intent.entry().app().to_owned(), intent.entry().namespace().to_owned()); + let template = { + let i = self.inner.lock(); + let elsewhere = i.deployments.values() + .any(|d| d.worker == intent.worker && d.entry().app() == app && d.entry().namespace() == namespace); + i.apps.get(&app).filter(|a| a.persistent && a.namespace == namespace && !elsewhere && a.workers.contains(&intent.worker)).cloned() + }; + if let Some(mut template) = template { + template.workers.retain(|w| *w != intent.worker); + self.save("apps", &app, &template)?; + self.inner.lock().apps.insert(app, template); + } tracing::info!(deployment = key, "deployment record removed: its node no longer configures it"); Ok(()) } @@ -1282,6 +1297,13 @@ impl WorkloadController { } } if !disabled { + // An app runs its replica count, wherever the replicas are: a start that would + // run more retires its deployment instead (a node back with a copy of an app + // that runs elsewhere since, say). + if self.beyond_replica_count(&c) { + self.retire(&format!("{}-{}", c.worker, s.name()))?; + continue; + } let already_ready = { let i = self.inner.lock(); i.workers.get(&c.worker).is_some_and(|w| { @@ -2127,10 +2149,11 @@ async fn priority_of(request: Request) -> anyhow::Result { } impl WorkloadController { - /// The keys of `app`'s deployment records in `namespace`. + /// The keys of `app`'s deployment records in `namespace`, but those being retired. pub(crate) fn app_deployments(&self, namespace: &str, app: &str) -> Vec { - self.inner.lock().deployments.iter() - .filter(|(_, d)| d.entry().app() == app && d.entry().namespace() == namespace) + let i = self.inner.lock(); + i.deployments.iter() + .filter(|(_, d)| d.entry().app() == app && d.entry().namespace() == namespace && !scaling::retiring(&i, d)) .map(|(k, _)| k.clone()) .collect() } diff --git a/crates/fleet-scheduler/src/workloads_scale.rs b/crates/fleet-scheduler/src/workloads_scale.rs index 41846fa..e9c4e11 100644 --- a/crates/fleet-scheduler/src/workloads_scale.rs +++ b/crates/fleet-scheduler/src/workloads_scale.rs @@ -29,6 +29,42 @@ fn waiting_for_capacity(i: &WorkloadState, intent: &AppDeployment) -> bool { }) } +/// Whether `intent` is leaving its node: disabled, and its node told to remove its app's +/// entries, or no longer configuring it. It is no longer one of its app's deployments, and its +/// record goes once nothing about it is open. +pub(super) fn retiring(i: &WorkloadState, intent: &AppDeployment) -> bool { + intent.disabled + && (i.commands.open().any(|c| c.worker == intent.worker && c.removes_app(intent.entry().app())) + || i.workers.get(&intent.worker).is_some_and(|w| w.report.entry(intent.entry().name()).is_none())) +} + +/// The entry `intent` has once the change under way, if any, is done. +fn planned(intent: &AppDeployment) -> Entry<'_> { + intent.scale.as_ref().filter(|s| s.pending()).map(ScaleStatus::target_entry).unwrap_or(intent.entry()) +} + +/// Whether `intent`'s replicas are to run: its change under way starts them, or, without one, +/// its node was last told to run them. +fn wants_running(intent: &AppDeployment) -> bool { + intent.scale.as_ref().filter(|s| s.pending()).map_or(!intent.disabled, |s| !s.disabled) +} + +/// Whether `intent`'s replicas are up, on a connected node. +fn running_now(i: &WorkloadState, intent: &AppDeployment) -> bool { + !intent.disabled && i.workers.get(&intent.worker).is_some_and(|w| { + now() - w.seen < 30.0 && w.report.ready + && w.report.entry(intent.entry().name()).is_some_and(|e| e.ready() && !e.disabled()) + }) +} + +/// Where deployment `key` comes in the order an app's replicas are kept in: those running now +/// first, then those that are to run, then the rest, by node within each. +fn rank(i: &WorkloadState, key: &str, intent: &AppDeployment) -> (bool, bool, String) { + (!running_now(i, intent), !wants_running(intent), key.to_owned()) +} + +const REPLICA_RANGE: &str = "gpus must be nonnegative, cpus positive, and replicas between 1 and 64; use app disable to stop all replicas"; + impl WorkloadController { pub(crate) async fn app_lifecycle_guard( &self, @@ -49,23 +85,34 @@ impl WorkloadController { Ok(()) } + /// `app`'s deployments in `namespace` (on `node`, when given) but those being retired, in + /// the order its replicas are kept in (`rank`). + fn app_deployment_list(&self, namespace: &str, app: &str, node: Option<&str>) -> Vec<(String, AppDeployment)> { + let i = self.inner.lock(); + let mut list: Vec<(String, AppDeployment)> = i.deployments.iter() + .filter(|(_, intent)| { + let s = intent.entry(); + s.namespace() == namespace + && node.is_none_or(|n| n == intent.worker) + && matches!(s.kind(), "ziren-gpu" | "app") + && s.app() == app + && !retiring(&i, intent) + }) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + list.sort_by_cached_key(|(key, intent)| rank(&i, key, intent)); + list + } + pub(super) fn app_deployment( &self, namespace: &str, app: &str, node: Option<&str>, ) -> anyhow::Result<(String, AppDeployment)> { - let i = self.inner.lock(); - let mut deployments = i.deployments.iter().filter(|(_, intent)| { - let s = intent.entry(); - s.namespace() == namespace - && node.is_none_or(|n| n == intent.worker) - && matches!(s.kind(), "ziren-gpu" | "app") - && s.app() == app - }); + let mut deployments = self.app_deployment_list(namespace, app, node).into_iter(); let first = deployments .next() - .map(|(k, v)| (k.clone(), v.clone())) .ok_or_else(|| fail(404, "the app runs on no node (enable it to place it), or not on that one"))?; if deployments.next().is_some() { return Err(fail(400, "the app runs on several nodes; select --node NODE")); @@ -113,20 +160,59 @@ impl WorkloadController { Ok(bytes.into_response()) } + /// The app's replicas and their resources, with each node's part; one node's part when + /// `node` is given. pub(super) fn scale_status( &self, namespace: &str, app: &str, node: Option<&str>, ) -> anyhow::Result { - let (_, deployment) = self.app_deployment(namespace, app, node)?; - Ok(self.scale_value(&deployment)) + if node.is_some() { + let (_, deployment) = self.app_deployment(namespace, app, node)?; + return Ok(self.scale_value(&deployment)); + } + let deployments = self.app_deployment_list(namespace, app, None); + if deployments.is_empty() { + return Err(fail(404, "the app runs on no node (enable it to place it)")); + } + Ok(self.app_scale_value(&deployments)) + } + + /// An app's scale: its replicas over all its deployments, their resources as the first + /// one has them, and its most pressing state, a failure first, then a change under way. + fn app_scale_value(&self, deployments: &[(String, AppDeployment)]) -> Value { + let parts: Vec = deployments.iter().map(|(_, d)| self.scale_value(d)).collect(); + let state = parts.iter().map(|p| &p["state"]).find(|s| *s == "failed") + .or_else(|| parts.iter().map(|p| &p["state"]).find(|s| !matches!(s.as_str(), Some("complete" | "configured")))) + .unwrap_or(&parts[0]["state"]); + json!({"namespace": parts[0]["namespace"], "replicas": parts.iter().filter_map(|p| p["replicas"].as_u64()).sum::(), + "gpus": parts[0]["gpus"], "cpus": parts[0]["cpus"], "state": state, + "disabled": parts.iter().all(|p| p["disabled"] == json!(true)), + "error": parts.iter().find_map(|p| p["error"].as_str()), + "nodes": parts.iter().map(|p| p["node"].clone()).collect::>(), "deployments": parts}) } /// The deployment of `app` (on `node`, when it has several) that a scale or settings update /// changes: the app still exists and no other operation on the deployment is under way. fn deployment_to_change(&self, namespace: &str, app: &str, node: Option<&str>) -> anyhow::Result<(String, AppDeployment)> { let (key, intent) = self.app_deployment(namespace, app, node)?; + self.check_changeable(namespace, app, &[(key.clone(), intent.clone())])?; + Ok((key, intent)) + } + + /// Every deployment of `app` a scale changes, in the order its replicas are kept in. + fn deployments_to_change(&self, namespace: &str, app: &str) -> anyhow::Result> { + let deployments = self.app_deployment_list(namespace, app, None); + if deployments.is_empty() { + return Err(fail(404, "the app runs on no node (enable it to place it)")); + } + self.check_changeable(namespace, app, &deployments)?; + Ok(deployments) + } + + /// The app still exists and no other operation on these deployments of it is under way. + fn check_changeable(&self, namespace: &str, app: &str, deployments: &[(String, AppDeployment)]) -> anyhow::Result<()> { // Recheck existence under the operation lock: deletion may have finished // after the HTTP handler resolved the app but before this request acquired it. let exists = self.fleet.app_known(namespace, app) @@ -138,16 +224,16 @@ impl WorkloadController { if !exists { return Err(fail(404, "unknown app")); } - { - let i = self.inner.lock(); - if intent.scale.as_ref().is_some_and(ScaleStatus::pending) && !waiting_for_capacity(&i, &intent) + let i = self.inner.lock(); + for (_, intent) in deployments { + if intent.scale.as_ref().is_some_and(ScaleStatus::pending) && !waiting_for_capacity(&i, intent) // one still waiting for capacity is replaced by this request || i.commands.open().any(|c| c.admitted && c.changes_entry(&intent.worker, intent.entry().name())) { return Err(fail(409, "an app operation is already pending")); } } - Ok((key, intent)) + Ok(()) } /// Settings an update may change on an app. The rest is identity, placement or the @@ -257,6 +343,11 @@ impl WorkloadController { Ok(self.scale_value(&intent)) } + /// Sets an app's replica count, and its replicas' GPUs and CPUs: the app's settings, not any + /// node's. The count is split over the app's deployments in the order its replicas are kept + /// in: each keeps the replicas it has, up to the count; new ones go to the first, whose start + /// moves to a node with room for all of them when its own has none; one left with none is + /// retired. Without a count, each deployment keeps its own. pub(super) async fn request_scale( &self, namespace: &str, @@ -267,55 +358,21 @@ impl WorkloadController { if req.gpus.is_none() && req.cpus.is_none() && req.replicas.is_none() { return Err(fail(400, "set at least one of gpus, cpus or replicas")); } - let (key, mut intent) = self.deployment_to_change(namespace, app, req.node.as_deref())?; - let mut target = self.with_priority(intent.spec.clone()); - EntryMut(&mut target).set_resources(req.gpus, req.cpus, req.replicas); - let t = Entry(&target); - let (g, cpu, replicas) = (t.gpus().unwrap_or(1.0), t.cpus(), t.replicas()); - if t.integrated_worker() && replicas != 1 { - return Err(fail(400, "integrated app workers require one machine-pinned replica", - )); - } - if !g.is_finite() - || g < 0.0 - || !cpu.is_finite() - || cpu <= 0.0 - || replicas == 0 - || replicas > 64 - { - return Err(fail(400,"gpus must be nonnegative, cpus positive, and replicas between 1 and 64; use app disable to stop all replicas")); - } - if t.kind() == "ziren-gpu" && (g < 1.0 || g.fract() != 0.0) { - return Err(fail( - 400, - "a proof server requires a positive integer GPU count", - )); + if req.node.is_some() { + return Err(fail(400, "replicas, GPUs and CPUs are the app's settings: scale it without a node, and the scheduler places its replicas")); } - let total_g = g * replicas as f64; - let total_cpu = cpu * replicas as f64; - { - let i = self.inner.lock(); - let worker = i - .workers - .get(&intent.worker) - .filter(|w| now() - w.seen < 30.0 && w.report.ready && !w.draining) - .ok_or_else(|| fail(503, "node is offline, unready or draining"))?; - if total_g > worker.report.total_gpus + 1e-9 - || total_cpu > worker.report.total_cpus + 1e-9 - { - return Err(fail( - 422, - format!( - "requested allocation exceeds node {} capacity: requested {total_g} GPUs and {total_cpu} CPUs (replicas: {replicas}); node total {} GPUs and {} CPUs, currently free {} GPUs and {} CPUs", - intent.worker, - worker.report.total_gpus, - worker.report.total_cpus, - worker.report.free_gpus, - worker.report.free_cpus, - ), - )); - } + if req.replicas.is_some_and(|r| r == 0 || r > 64) { + return Err(fail(400, REPLICA_RANGE)); } + let deployments = self.deployments_to_change(namespace, app)?; + let count = req.replicas.map_or_else(|| deployments.iter().map(|(_, d)| d.entry().replicas()).sum(), u64::from); + let mut left = count; + let mut plan: Vec<(String, AppDeployment, u64)> = deployments.into_iter().map(|(key, intent)| { + let keep = intent.entry().replicas().min(left); + left -= keep; + (key, intent, keep) + }).collect(); + plan[0].2 += left; let ns = self .fleet .namespace(namespace) @@ -323,44 +380,174 @@ impl WorkloadController { if ns.suspended_ms.is_some() { return Err(fail(403, "namespace is suspended")); } - // Container replicas reserve the whole app. Proof quota is charged - // per assigned proof; extra replicas can remain idle while that quota is occupied. - let (quota_g, quota_cpu) = if t.holds_allocation() { - (total_g, total_cpu) - } else { - (g, cpu) - }; - let tag_quota = ns - .tag_quotas - .get(t.tag().unwrap_or("")) - .cloned() - .unwrap_or_default(); + let mut changes = Vec::new(); + let mut retired = Vec::new(); + let (mut quota_g, mut quota_cpu, mut tag) = (0.0, 0.0, None); + for (key, intent, replicas) in plan { + if replicas == 0 { + retired.push(key); + continue; + } + let mut target = self.with_priority(intent.spec.clone()); + EntryMut(&mut target).set_resources(req.gpus, req.cpus, Some(replicas as u32)); + let t = Entry(&target); + let (g, cpu) = (t.gpus().unwrap_or(1.0), t.cpus()); + if t.integrated_worker() && replicas != 1 { + return Err(fail(400, "integrated app workers require one machine-pinned replica")); + } + if !g.is_finite() || g < 0.0 || !cpu.is_finite() || cpu <= 0.0 { + return Err(fail(400, REPLICA_RANGE)); + } + if t.kind() == "ziren-gpu" && (g < 1.0 || g.fract() != 0.0) { + return Err(fail(400, "a proof server requires a positive integer GPU count")); + } + // Container replicas reserve the whole app. Proof quota is charged per assigned + // proof; extra replicas can remain idle while that quota is occupied. + if t.holds_allocation() { + (quota_g, quota_cpu) = (quota_g + g * replicas as f64, quota_cpu + cpu * replicas as f64); + } else { + (quota_g, quota_cpu) = (f64::max(quota_g, g), f64::max(quota_cpu, cpu)); + } + tag.get_or_insert_with(|| t.tag().unwrap_or("").to_owned()); + if intent.entry().same_resources(t) && intent.scale.as_ref().is_none_or(|s| s.phase() != Phase::Failed) { + continue; + } + self.check_node_room(&intent.worker, g * replicas as f64, cpu * replicas as f64, replicas)?; + changes.push((key, intent, target)); + } + let tag_quota = ns.tag_quotas.get(tag.as_deref().unwrap_or("")).cloned().unwrap_or_default(); if !ns.quota.fits(quota_g, quota_cpu) || !tag_quota.fits(quota_g, quota_cpu) { return Err(fail( 422, "requested allocation exceeds namespace or tag GPU/CPU quota", )); } - if intent.entry().same_resources(t) - && intent + for (key, mut intent, target) in changes { + let disabled = intent .scale .as_ref() - .is_none_or(|s| s.phase() != Phase::Failed) - { - return Ok(self.scale_value(&intent)); + .filter(|s| s.phase() == Phase::Failed) + .map(|s| s.disabled) + .unwrap_or(intent.disabled); + intent.scale = Some(new_scale(target, disabled)); + // Persist the transition before changing admission or sending any command. + self.save("deployments", &key, &intent)?; + self.inner.lock().deployments.insert(key, intent); + } + for key in &retired { + self.retire(key)?; + } + if req.replicas.is_some() { + self.set_app_replicas(namespace, app, count)?; } - let disabled = intent - .scale - .as_ref() - .filter(|s| s.phase() == Phase::Failed) - .map(|s| s.disabled) - .unwrap_or(intent.disabled); - intent.scale = Some(new_scale(target, disabled)); - // Persist the transition before changing admission or sending any command. - self.save("deployments", &key, &intent)?; - self.inner.lock().deployments.insert(key, intent.clone()); self.notify.notify_waiters(); - Ok(self.scale_value(&intent)) + Ok(self.app_scale_value(&self.app_deployment_list(namespace, app, None))) + } + + /// Refuses a change of `worker`'s deployment to `replicas` replicas of `gpus` GPUs and + /// `cpus` CPUs in all when the node can't take it: it is offline, unready or draining, or + /// smaller than that. + fn check_node_room(&self, worker: &str, gpus: f64, cpus: f64, replicas: u64) -> anyhow::Result<()> { + let i = self.inner.lock(); + let node = i + .workers + .get(worker) + .filter(|w| now() - w.seen < 30.0 && w.report.ready && !w.draining) + .ok_or_else(|| fail(503, format!("node {worker} is offline, unready or draining")))?; + if gpus > node.report.total_gpus + 1e-9 || cpus > node.report.total_cpus + 1e-9 { + return Err(fail( + 422, + format!( + "requested allocation exceeds node {worker} capacity: requested {gpus} GPUs and {cpus} CPUs (replicas: {replicas}); node total {} GPUs and {} CPUs, currently free {} GPUs and {} CPUs", + node.report.total_gpus, + node.report.total_cpus, + node.report.free_gpus, + node.report.free_cpus, + ), + )); + } + Ok(()) + } + + /// Records `count` as the replica count of the app the scheduler keeps a record of. + fn set_app_replicas(&self, namespace: &str, app: &str, count: u64) -> anyhow::Result<()> { + let template = self.inner.lock().apps.get(app).filter(|a| a.persistent && !a.removed && a.namespace == namespace).cloned(); + if let Some(mut template) = template { + template.spec["replicas"] = json!(count); + self.save("apps", app, &template)?; + self.inner.lock().apps.insert(app.into(), template); + } + Ok(()) + } + + /// The replicas `app` has over its deployments once their changes are done. + fn app_replicas(&self, namespace: &str, app: &str) -> u64 { + self.app_deployment_list(namespace, app, None).iter().map(|(_, d)| planned(d).replicas()).sum() + } + + /// Whether starting `start`'s entry would run more replicas of a container app than its + /// count: the app's other deployments, ahead of this one in the order its replicas are kept + /// in, already run them all on connected nodes. Only an app the scheduler keeps a record of + /// has a count; a prover's nodes configure their own replicas, and an entry its node anchors + /// to the machine (`machine`) is the node's own, so it always runs there. + pub(super) fn beyond_replica_count(&self, start: &CommandRecord) -> bool { + let s = start.entry(); + if !s.is_container() || !s.machine().is_empty() { + return false; + } + let key = format!("{}-{}", start.worker, s.name()); + let i = self.inner.lock(); + let Some(count) = i.apps.get(s.app()) + .filter(|a| a.persistent && !a.removed && a.namespace == s.namespace()) + .map(|a| a.entry().replicas()) else { + return false; + }; + let Some(this) = i.deployments.get(&key).filter(|d| d.entry().machine().is_empty()) else { + return false; + }; + let own = rank(&i, &key, this); + let ahead: u64 = i.deployments.iter() + .filter(|(k, d)| **k != key && d.entry().app() == s.app() && d.entry().namespace() == s.namespace()) + .filter(|(k, d)| running_now(&i, d) && !retiring(&i, d) && rank(&i, k, d) < own) + .map(|(_, d)| planned(d).replicas()) + .sum(); + ahead >= count + } + + /// Retires deployment `key`, which its app's replica count leaves no replicas: it is + /// disabled at once, so it no longer counts and nothing starts it, and its node removes the + /// app's entries, stopping what is left of their replicas (once it is back, when it is + /// offline). That frees its room, and the record goes with the node's next report. The + /// app's data stays on the node. + pub(super) fn retire(&self, key: &str) -> anyhow::Result<()> { + let Some(mut intent) = self.inner.lock().deployments.get(key).cloned() else { + return Ok(()); + }; + let entry = intent.entry(); + let (namespace, app, name, worker) = + (entry.namespace().to_owned(), entry.app().to_owned(), entry.name().to_owned(), intent.worker.clone()); + intent.disabled = true; + if let Some(scale) = intent.scale.as_mut().filter(|s| s.pending()) { + scale.phase = Phase::Complete as i32; + } + self.save("deployments", key, &intent)?; + self.inner.lock().deployments.insert(key.into(), intent); + // A start or stop never sent there has nothing left to do. + let unsent: Vec = self.inner.lock().commands.open() + .filter(|c| !c.admitted && c.changes_entry(&worker, &name)).map(|c| c.id.clone()).collect(); + for id in unsent { + let dropped = self.inner.lock().commands.update(&id, |c| { + c.done = true; + c.error = Some("its app's replica count leaves this node none".into()); + c.clone() + }); + if let Some(c) = dropped { + self.save("commands", &id, &c)?; + } + } + self.tell_nodes([&worker], self.app_message(&namespace, &app), message::remove())?; + tracing::info!(deployment = key, app, "retiring a deployment its app's replica count leaves no replicas"); + Ok(()) } fn scale_command( @@ -546,6 +733,8 @@ impl WorkloadController { for k in Entry::RESOURCES { app.spec[k] = scale.target[k].clone(); } + // The count is the app's, over all its nodes. + app.spec["replicas"] = json!(self.app_replicas(intent.entry().namespace(), intent.entry().app())); for (k, v) in scale.set.as_ref().and_then(|set| set.as_object()).into_iter().flatten() { app.spec[k.as_str()] = v.clone(); } diff --git a/crates/fleet-scheduler/src/workloads_tests.rs b/crates/fleet-scheduler/src/workloads_tests.rs index 4ada661..125124d 100644 --- a/crates/fleet-scheduler/src/workloads_tests.rs +++ b/crates/fleet-scheduler/src/workloads_tests.rs @@ -2581,6 +2581,108 @@ async fn an_app_anchored_to_its_machine_restarts_only_there() { assert_eq!(c.inner.lock().apps["model"].workers, ["gpu"]); } +/// App `model` runs one replica on node `gpu`, anchored to `machine` when given, and one on node +/// `big`, which also configures it, as a node that took it over while `gpu` was away would. +async fn running_on_two_nodes(machine: Option<&str>) -> (tempfile::TempDir, Arc, BTreeMap, String) { + let (dir, c, mut nodes) = running_on_a_full_node(machine).await; + let ns = c.inner.lock().apps["model"].namespace.clone(); + let mut copy = nodes["gpu"].apps[0].clone(); + copy.as_object_mut().unwrap().remove("machine"); + let big = nodes.get_mut("big").unwrap(); + (big.free_gpus, big.free_cpus) = (1.0, 6.0); + big.apps.push(copy); + for (id, report) in &nodes { + c.report(id, "session", report.clone()).await.unwrap(); + } + assert!(!c.inner.lock().deployments["big-model"].disabled); + (dir, c, nodes, ns) +} + +async fn json_of(response: Response) -> Value { + serde_json::from_slice(&to_bytes(response.into_body(), 1 << 20).await.unwrap()).unwrap() +} + +/// An app's replicas are the app's, wherever they run: its scale counts them on all its nodes +/// and takes no node, and scaling it down stops the extra ones. A node left with none removes +/// the app, frees its room and drops out of the app; the app's data stays there. +#[tokio::test] +async fn an_apps_replicas_are_its_own_and_a_node_its_scale_leaves_none_removes_it() { + let (_dir, c, mut nodes, ns) = running_on_two_nodes(None).await; + let held = |c: &WorkloadController| { + let mut ids: Vec = c.fleet.allocations(&ns).iter().map(|a| a.id.clone()).collect(); + ids.sort(); + ids + }; + assert_eq!(held(&c), ["deployment:big-model", "deployment:gpu-model"]); + let path = format!("/apps/model/scale?namespace={ns}"); + let view = json_of(scale_call(c.clone(), "admin", "GET", &path, Value::Null).await).await; + assert_eq!((view["replicas"].clone(), view["nodes"].clone()), (json!(2), json!(["big", "gpu"])), "{view}"); + let part = json_of(scale_call(c.clone(), "admin", "GET", &format!("{path}&node=gpu"), Value::Null).await).await; + assert_eq!((part["replicas"].clone(), part["node"].clone()), (json!(1), json!("gpu")), "{part}"); + let refused = scale_call(c.clone(), "admin", "PUT", &path, json!({"replicas":1,"node":"gpu"})).await; + assert_eq!(refused.status(), StatusCode::BAD_REQUEST, "a scale names no node"); + for replicas in [0, 65] { + let refused = scale_call(c.clone(), "admin", "PUT", &path, json!({"replicas":replicas})).await; + assert_eq!(refused.status(), StatusCode::BAD_REQUEST, "{replicas} replicas"); + } + + let scaled = scale_call(c.clone(), "admin", "PUT", &path, json!({"replicas":1})).await; + assert_eq!(scaled.status(), StatusCode::ACCEPTED); + let scaled = json_of(scaled).await; + assert_eq!((scaled["replicas"].clone(), scaled["nodes"].clone()), (json!(1), json!(["big"])), "{scaled}"); + assert_eq!(c.inner.lock().apps["model"].entry().replicas(), 1, "the app's count"); + assert_eq!(c.app_resources(&ns, "model", None)["resources"]["replicas"], json!(1)); + assert!(c.inner.lock().deployments["big-model"].scale.is_none(), "the node that keeps its replica is not changed"); + nodes_run_commands(&c, &mut nodes, &[]).await; + assert!(nodes["gpu"].apps.is_empty(), "the node left with none removed the app"); + let running = nodes["big"].apps.iter().any(|a| a["name"] == "model" && a["ready"] == json!(true) && a["disabled"] == json!(false)); + assert!(running, "the other keeps running it: {:?}", nodes["big"].apps); + assert_eq!(held(&c), ["deployment:big-model"], "its room is free"); + assert_eq!(c.app_resources(&ns, "model", None)["resources"]["replicas"], json!(1)); + c.report("gpu", "session", nodes["gpu"].clone()).await.unwrap(); + assert!(!c.inner.lock().deployments.contains_key("gpu-model")); + assert_eq!(c.inner.lock().apps["model"].workers, ["big"]); + + // More replicas go to the node that runs it. + let scaled = scale_call(c.clone(), "admin", "PUT", &path, json!({"replicas":2})).await; + assert_eq!(scaled.status(), StatusCode::ACCEPTED); + let target = c.inner.lock().deployments["big-model"].scale.as_ref().map(|s| s.target_entry().replicas()); + assert_eq!(target, Some(2)); + assert_eq!(c.inner.lock().apps["model"].entry().replicas(), 2); +} + +/// A node back with a copy of an app that runs all its replicas elsewhere since does not start +/// it again: it removes it. An entry its node anchors to the machine is the node's own and +/// starts there anyway. +#[tokio::test] +async fn a_node_back_with_a_copy_of_an_app_that_runs_its_replicas_elsewhere_removes_it() { + for machine in [None, Some("machine")] { + let (_dir, c, mut nodes, _ns) = running_on_two_nodes(machine).await; + // Its replica went with the node, which then reports its entry stopped. + let gpu = nodes.get_mut("gpu").unwrap(); + (gpu.apps[0]["ready"], gpu.apps[0]["cleaned"]) = (json!(false), json!(true)); + c.report("gpu", "session", nodes["gpu"].clone()).await.unwrap(); + nodes_run_commands(&c, &mut nodes, &[]).await; + let running = nodes["big"].apps.iter().any(|a| a["name"] == "model" && a["ready"] == json!(true)); + assert!(running, "{machine:?}: {:?}", nodes["big"].apps); + let starts_sent = c.inner.lock().commands.values() + .filter(|v| v.worker == "gpu" && v.admitted && v.operation == NodeOperation::SetApp as i32 && v.starts()) + .count(); + if machine.is_none() { + assert!(nodes["gpu"].apps.is_empty(), "its node removed it"); + assert_eq!(starts_sent, 1, "only the first start, before the node went away"); + c.report("gpu", "session", nodes["gpu"].clone()).await.unwrap(); + assert!(!c.inner.lock().deployments.contains_key("gpu-model")); + assert_eq!(c.inner.lock().apps["model"].workers, ["big"]); + } else { + assert_eq!(nodes["gpu"].apps.len(), 1, "an anchored entry stays"); + let waiting = c.inner.lock().commands.values() + .any(|v| !v.done && v.worker == "gpu" && v.operation == NodeOperation::SetApp as i32 && v.starts()); + assert!(waiting, "and its start waits for room there"); + } + } +} + #[tokio::test] async fn an_app_a_node_refuses_to_create_is_placed_again_on_the_next_enable() { let (_dir, c, ns) = setup().await; diff --git a/crates/fleet-scheduler/src/workloads_views.rs b/crates/fleet-scheduler/src/workloads_views.rs index 921ec77..df548b4 100644 --- a/crates/fleet-scheduler/src/workloads_views.rs +++ b/crates/fleet-scheduler/src/workloads_views.rs @@ -57,14 +57,16 @@ impl WorkloadController { v } - /// Configured totals across all deployments and replicas; disabled apps keep their settings. - /// Quotas are shared namespace limits, not independent limits for each app. + /// Configured totals across all deployments and replicas, but those being retired; disabled + /// apps keep their settings. Quotas are shared namespace limits, not independent limits for + /// each app. pub(crate) fn app_resources(&self, namespace: &str, app: &str, tag: Option<&str>) -> Value { let i = self.inner.lock(); let deployments: Vec<_> = i .deployments .values() - .filter(|intent| intent.entry().namespace() == namespace && intent.entry().app() == app) + .filter(|intent| intent.entry().namespace() == namespace && intent.entry().app() == app + && !super::scaling::retiring(&i, intent)) .collect(); let mut gpus = 0.0; let mut cpus = 0.0; diff --git a/crates/fleetctl/src/main.rs b/crates/fleetctl/src/main.rs index c115279..1a2e9ff 100644 --- a/crates/fleetctl/src/main.rs +++ b/crates/fleetctl/src/main.rs @@ -105,9 +105,11 @@ enum AppCmd { #[arg(long)] gpus: Option, /// CPU units per replica. #[arg(long)] cpus: Option, - /// Replicas on the selected node (positive; use app disable to stop all). + /// The app's replicas, wherever the scheduler places them (positive; use app disable to + /// stop all). Fewer than it runs: the scheduler stops the extra ones, and a node left with + /// none removes the app. #[arg(long)] replicas: Option, - /// Select a machine agent when the app runs on several nodes. + /// Inspect only the part of the app on this machine agent; scaling takes no node. #[arg(long)] node: Option, }, /// Change a proof or persistent app's settings. Image and environment changes require @@ -1071,14 +1073,19 @@ fn scale_app(name: &str, gpus: Option, cpus: Option, replicas: Option< if let Some(v) = cpus { body["cpus"] = json!(v); } if let Some(v) = replicas { body["replicas"] = json!(v); } let write = !body.as_object().unwrap().is_empty(); - if let Some(node) = node { body["node"] = json!(node); } + if write && node.is_some() { + bail!("replicas, GPUs and CPUs are the app's settings: scale it without --node, and the scheduler places its replicas"); + } let method = if write { reqwest::Method::PUT } else { reqwest::Method::GET }; let a = FleetClient::new(conn)?; let mut request = a.req(method, &format!("/apps/{name}/scale")); if write { request = request.json(&body); } else if let Some(node) = node { request = request.query(&[("node", node)]); } let v = a.send(request)?; - out.done(&format!("{name}: {} — {} replicas, {} GPUs and {} CPUs per replica on {}", - v["state"].as_str().unwrap_or("requested"), v["replicas"], v["gpus"], v["cpus"], v["node"].as_str().unwrap_or("?")),v); + // One node's part names its node; the app's names all of them. + let nodes = v["nodes"].as_array().map(|n| n.iter().filter_map(Value::as_str).collect::>().join(", ")) + .or_else(|| v["node"].as_str().map(str::to_owned)).filter(|n| !n.is_empty()).unwrap_or_else(|| "?".into()); + out.done(&format!("{name}: {} — {} replicas, {} GPUs and {} CPUs per replica on {nodes}", + v["state"].as_str().unwrap_or("requested"), v["replicas"], v["gpus"], v["cpus"]),v); Ok(()) } diff --git a/crates/fleetctl/tests/apps.rs b/crates/fleetctl/tests/apps.rs index 53e6889..98fd7bb 100644 --- a/crates/fleetctl/tests/apps.rs +++ b/crates/fleetctl/tests/apps.rs @@ -191,8 +191,8 @@ fn app_commands_address_every_app_by_its_name() { (vec!["disable", "ethproofs"], "PUT", "/disabled", serde_json::Value::Null, r#"{"name":"ethproofs","disabled":true,"resources":{"gpus":1},"proof":{"elf_id":"aaaa"}}"#), (vec!["enable", "comfyui"], "DELETE", "/disabled", serde_json::Value::Null, r#"{"name":"comfyui","disabled":false,"spec":{"image":"comfyui:1"}}"#), - (vec!["scale", "ethproofs", "--replicas", "2", "--node", "gpu-box"], "PUT", "/scale", serde_json::json!({"replicas":2,"node":"gpu-box"}), - r#"{"state":"draining","replicas":2,"gpus":2,"cpus":16,"node":"gpu-box"}"#), + (vec!["scale", "ethproofs", "--replicas", "2"], "PUT", "/scale", serde_json::json!({"replicas":2}), + r#"{"state":"draining","replicas":2,"gpus":2,"cpus":16,"nodes":["gpu-box"]}"#), (vec!["update", "comfyui", "--image", "comfyui:2"], "PUT", "/settings", serde_json::json!({"image":"comfyui:2"}), r#"{"state":"draining","node":"gpu-box"}"#), ] { @@ -210,6 +210,28 @@ fn app_commands_address_every_app_by_its_name() { } } +/// An app's replicas and their resources are the app's: scaling names no node, and a node +/// only narrows what is inspected. +#[test] +fn app_scale_takes_no_node_and_inspects_one_nodes_part() { + let token = token_file("app-scale-node"); + let output = cli().args(["--namespace", "alice", "--url", "http://127.0.0.1:9", "--token-file"]).arg(&token) + .args(["app", "scale", "comfyui", "--replicas", "1", "--node", "gpu-box"]).output().unwrap(); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("scale it without --node"), "{}", String::from_utf8_lossy(&output.stderr)); + + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let server = stub(listener, 1, move |_, _| (200, r#"{"state":"complete","replicas":1,"gpus":1,"cpus":4,"node":"gpu-box"}"#.into())); + let output = cli().args(["--namespace", "alice", "--url", &url, "--token-file"]).arg(&token) + .args(["app", "scale", "comfyui", "--node", "gpu-box"]).output().unwrap(); + let seen = server.join().unwrap(); + std::fs::remove_file(token).unwrap(); + assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + assert!(seen[0].line.starts_with("GET /apps/comfyui/scale?namespace=alice&node=gpu-box"), "{}", seen[0].line); + assert!(String::from_utf8_lossy(&output.stdout).contains("on gpu-box"), "{}", String::from_utf8_lossy(&output.stdout)); +} + #[test] fn app_logs_read_the_control_plane_route_and_print_the_raw_log_or_the_error() { for status in [200u16, 503] { diff --git a/docs/operations.md b/docs/operations.md index c69b2f5..cc14e97 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -68,16 +68,24 @@ allocations further. ```bash fleetctl --token-file zkm-owner.token --namespace zkm app scale ethproofs-z4 \ - --gpus 5 --cpus 96 --replicas 1 --node gpu-1 + --gpus 5 --cpus 96 --replicas 1 fleetctl --token-file zkm-owner.token --namespace zkm app scale ethproofs-z4 # desired, observed, status +fleetctl --token-file zkm-owner.token --namespace zkm app scale ethproofs-z4 --node gpu-1 # that node's part ``` Scaling needs `manage` on the app and an online, ready node. It runs in the background: check it until it's `complete` or `failed`. Omitted flags keep their -values, a disabled app stays disabled, and `--node` picks the deployment when an app -has several. Integrated EthProofs runs exactly one replica; other apps may run more, -with resources per replica. The app's current allocation counts toward the room for -its new one. Scaling never adds node capacity. +values, and a disabled app stays disabled. Replicas, GPUs and CPUs are the app's +settings, not a node's: `--replicas` is how many the app runs wherever the scheduler +places them, so a scale takes no `--node` (it only narrows what a check shows). When +an app runs on several nodes, each keeps the replicas it has up to the count, new +ones join a node that runs it, and a node left with none stops them and +removes the app from its `apps.yaml`, which frees its room; the app's data stays on +that node. A node that comes back with a copy of an app whose replicas all run +elsewhere removes it too, unless its entry is anchored to the machine (`machine`). +Integrated EthProofs runs exactly one replica; other apps may run more, with +resources per replica. The app's current allocation counts toward the room for its +new one. Scaling never adds node capacity. A node's pool is `RAY_NUM_CPUS` and `CUDA_VISIBLE_DEVICES` in its `etc/watchdog.env`; empty means detect CPUs automatically, and no GPUs. Changing it restarts Ray, so diff --git a/docs/reference.md b/docs/reference.md index 8276b1e..2b82702 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -189,7 +189,9 @@ stops its GPU server; queued proofs wait for the next enable, and its retained p stay fetchable. **Changing.** `fleetctl app scale` sets an app's GPUs, CPUs and replicas within its -quotas ([operations](operations.md#configure-resources)); an active proof can't be +quotas ([operations](operations.md#configure-resources)). They are the app's, not a +node's: the replica count covers all its nodes, and a node the count leaves none +removes the app. An active proof can't be resized ([issue #3](https://github.com/ProjectZKM/zkm-fleet/issues/3)). `fleetctl app update NAME` (`PUT /apps/NAME/settings`) changes the image, environment, command, user, workdir, volumes, shm and memory sizes, health path or @@ -604,7 +606,7 @@ listener behind the HTTPS gateway. Administrators select a namespace with | Workload | Routes | |---|---| | Apps | `GET` / `POST /apps`, `GET` / `DELETE /apps/NAME`, `PUT /apps/NAME/priority` with `{"priority":N}`, `PUT` / `DELETE /apps/NAME/disabled`, `PUT /apps/NAME/settings`, `GET /apps/NAME/logs` | -| Scaling | `PUT` / `GET /apps/NAME/scale` | +| Scaling | `PUT` / `GET /apps/NAME/scale` (the app's replicas over all its nodes; `GET ...?node=NODE` shows one node's part) | | Proof app jobs | `GET /apps/NAME/jobs?limit=N&active=true`, `POST /apps/NAME/jobs` (app-storage references), `GET /apps/NAME/jobs/ID`, `GET /apps/NAME/jobs/ID/output/proof.json` and `.../output/public_values.bin` | | Batches | `GET` / `POST /batches`, `GET` / `DELETE /batches/NAME`, `PUT /batches/NAME/priority`, `PUT /batches/NAME/settings`, `PUT` / `DELETE /batches/NAME/disabled` | | Batch jobs | `GET` / `POST /batches/NAME/jobs`, `DELETE /batches/NAME/jobs?older_than=DAYS`, `GET` / `DELETE /batches/NAME/jobs/ID`, `POST /batches/NAME/jobs/ID/cancel`, `POST /batches/NAME/jobs/ID/preempt`, `GET /batches/NAME/jobs/ID/logs` |