From 0b7a26fcd95c6ed4d170afcaea3db3452f818cdc Mon Sep 17 00:00:00 2001 From: eigmax Date: Thu, 12 Nov 2026 13:36:07 +0000 Subject: [PATCH 1/4] Show a remote node upgrade's dry run as the upgrade it is A remote dry run ships nothing to the node, so the node got paths to files that aren't there. It took the first one for a release tag and printed a release download, plus installs of the proof app and launch scripts the upgrade would never touch. A release tag is a bare name: a path now always names local files, shipped or not. --- deploy/fleet-ctl.sh | 4 +++- deploy/tests/test_fleet_ctl.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/deploy/fleet-ctl.sh b/deploy/fleet-ctl.sh index 0fb003b..27e0911 100755 --- a/deploy/fleet-ctl.sh +++ b/deploy/fleet-ctl.sh @@ -156,7 +156,9 @@ scheduler_rollback() { node_upgrade() { # node_upgrade FLEET_NODE BRIDGE_DIR [FLEET_PROOF_APP] [DEPLOY_DIR], or node upgrade RELEASE_TAG local bin=${1:-} bridge=${2:-} proof_app=${3:-} scripts=${4:-} - if [ -n "$bin" ] && [ ! -f "$bin" ]; then + # A release tag is a bare name. A path names local files, even ones a dry run never + # shipped to the node. + if [ -n "$bin" ] && [ ! -f "$bin" ] && [[ $bin != */* ]]; then local stage; stage=$(mktemp -d) prepare_release "$bin" "$stage" bin=$RELEASE_ROOT/bin/fleet-node diff --git a/deploy/tests/test_fleet_ctl.py b/deploy/tests/test_fleet_ctl.py index 51b00b0..fc8d66f 100644 --- a/deploy/tests/test_fleet_ctl.py +++ b/deploy/tests/test_fleet_ctl.py @@ -180,6 +180,25 @@ def test_watchdog_repairs_a_dead_ray_link(tmp_path): assert (tmp_path / "ray.log").exists(), "the repaired link let the watchdog start Ray" +def test_a_dry_run_on_the_node_takes_the_files_it_never_shipped_for_local_files(tmp_path): + """A remote dry run ships nothing, so the node sees paths to no files: it prints the local + upgrade of just those files, not a download of a release named after them.""" + prefix, commands = node_fixture(tmp_path) + executable(commands / "curl", f"echo called >> {tmp_path / 'curl.log'}\nexit 1\n") + stage = tmp_path / "fleet-ctl-stage" # never created + environment = {**os.environ, "PATH": f"{commands}:{os.environ['PATH']}", "FLEET_PREFIX": str(prefix), + "FLEET_CTL_ENV": str(tmp_path / "none.env"), "FLEET_CTL_ON_NODE": "1", "FLEET_NODE_SSH": ""} + result = subprocess.run( + ["bash", str(ROOT / "deploy/fleet-ctl.sh"), "-n", "node", "upgrade", f"{stage}/fleet-node", f"{stage}/bridge"], + env=environment, capture_output=True, text=True, timeout=20) + assert result.returncode == 0, result.stderr + assert not (tmp_path / "curl.log").exists() and "curl" not in result.stdout, result.stdout + assert f"+ install -m 755 {stage}/fleet-node {prefix}/bin/fleet-node.new" in result.stdout, result.stdout + assert f"+ cp -a {stage}/bridge/fleet_bridge/. {prefix}/bridge/fleet_bridge/" in result.stdout, result.stdout + assert "fleet-proof-app" not in result.stdout and "ray-start.sh" not in result.stdout, result.stdout + assert (prefix / "bin/fleet-node").read_text() == "#!/bin/bash\necho old-node\n", "a dry run changes nothing" + + def test_remote_upgrade_preserves_scripts_position_without_a_proof_app(tmp_path): prefix, commands = node_fixture(tmp_path) newbin = tmp_path / "fleet-node" From 3d6d16c65a1b97c0f939b22f36721af15076e5f8 Mon Sep 17 00:00:00 2001 From: eigmax Date: Thu, 12 Nov 2026 13:36:13 +0000 Subject: [PATCH 2/4] Admit a restarted node agent at once One agent session at a time drives a node, and the scheduler refused a new session until the old one had been quiet for 30 seconds. Every agent restart, an upgrade included, left the node unheard for that long, right at the threshold where it counts as offline. A stopping agent now ends its own session (WorkerService.EndSession), after it has stopped its bridge, and the scheduler admits the next session at once. The node counts as connected meanwhile, as before. Only the session itself can end it, with the cluster token. An agent that crashed, or that can't reach the scheduler as it stops, leaves its session to go quiet as before, and so does an agent talking to a scheduler without the call. --- crates/fleet-node/src/main.rs | 65 ++++++++++++++++++- crates/fleet-proto/proto/fleet/v1/fleet.proto | 10 +++ crates/fleet-scheduler/src/workloads.rs | 30 ++++++++- crates/fleet-scheduler/tests/flow.rs | 36 ++++++++++ docs/maintenance.md | 8 +++ 5 files changed, 145 insertions(+), 4 deletions(-) diff --git a/crates/fleet-node/src/main.rs b/crates/fleet-node/src/main.rs index 055a5c4..10a7476 100644 --- a/crates/fleet-node/src/main.rs +++ b/crates/fleet-node/src/main.rs @@ -3,7 +3,8 @@ mod logs; use anyhow::{Context, Result}; use clap::Parser; use fleet_proto::fleet::v1::{ - worker_service_client::WorkerServiceClient, LogRequest, ProofFetch, WorkerHeartbeatRequest, + worker_service_client::WorkerServiceClient, LogRequest, ProofFetch, WorkerEndSessionRequest, + WorkerHeartbeatRequest, }; use serde_json::{json, Value}; use std::{ @@ -75,6 +76,24 @@ fn outbound(mut report: Value, stale: bool, version: Option<&str>) -> Value { } report } +/// Tells the control plane this agent's session is over, so the agent that replaces it (a +/// restart, an upgrade) is admitted at once instead of after the session goes quiet. Best +/// effort: a scheduler without the call, or one out of reach, waits it out as before. +async fn end_session(endpoint: Endpoint, auth: tonic::metadata::MetadataValue, worker: &str, session: &str) { + let ended = async { + let mut client = connect(endpoint).await?; + let mut request = tonic::Request::new(WorkerEndSessionRequest { worker_id: worker.into(), session: session.into() }); + request.metadata_mut().insert("authorization", auth); + request.set_timeout(Duration::from_secs(3)); + client.end_session(request).await?; + anyhow::Ok(()) + }; + match tokio::time::timeout(Duration::from_secs(5), ended).await { + Ok(Ok(())) => tracing::info!(worker, "agent session ended"), + Ok(Err(error)) => tracing::warn!(%error, "could not end the agent session; the next agent waits until it goes quiet"), + Err(_) => tracing::warn!("ending the agent session timed out; the next agent waits until it goes quiet"), + } +} async fn connect(endpoint: Endpoint) -> Result> { Ok(WorkerServiceClient::new(endpoint.connect().await?) .max_decoding_message_size(16 << 20) @@ -298,6 +317,8 @@ async fn main() -> Result<()> { log_task.abort(); reader.abort(); let _ = child.kill().await; + // Stopped: nothing of this agent reports any more. + end_session(endpoint, auth, &a.name, &session).await; Ok(()) } @@ -548,4 +569,46 @@ mod tests { let unnamed = outbound(json!({"version":3,"machine":"m","ready":true}), false, None); assert!(unnamed.get("agent_version").is_none()); } + + /// A control plane that records the sessions agents end, with the token they sent. + struct Ends(Arc>>); + #[tonic::async_trait] + impl fleet_proto::fleet::v1::worker_service_server::WorkerService for Ends { + async fn heartbeat(&self, _: tonic::Request, + ) -> Result, tonic::Status> { + Err(tonic::Status::unimplemented("not in this test")) + } + async fn end_session(&self, request: tonic::Request, + ) -> Result, tonic::Status> { + let token = request.metadata().get("authorization").and_then(|v| v.to_str().ok()).unwrap_or("").to_owned(); + let r = request.into_inner(); + self.0.lock().await.push((r.worker_id, r.session, token)); + Ok(tonic::Response::new(fleet_proto::fleet::v1::WorkerEndSessionResponse {})) + } + } + + #[tokio::test] + async fn a_stopping_agent_ends_its_own_session_with_its_token() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let ends = Arc::new(Mutex::new(Vec::new())); + let incoming = tonic::transport::server::TcpIncoming::from_listener(listener, true, None).unwrap(); + tokio::spawn(tonic::transport::Server::builder() + .add_service(fleet_proto::fleet::v1::worker_service_server::WorkerServiceServer::new(Ends(ends.clone()))) + .serve_with_incoming(incoming)); + let endpoint = Endpoint::from_shared(format!("http://{address}")).unwrap(); + end_session(endpoint, "Bearer secret".parse().unwrap(), "gpu-1", "session-1").await; + assert_eq!(*ends.lock().await, [("gpu-1".to_string(), "session-1".to_string(), "Bearer secret".to_string())]); + } + + #[tokio::test] + async fn an_agent_stops_promptly_when_the_control_plane_is_out_of_reach() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + drop(listener); // nothing listens there + let endpoint = Endpoint::from_shared(format!("http://{address}")).unwrap().connect_timeout(Duration::from_secs(5)); + let started = Instant::now(); + end_session(endpoint, "Bearer secret".parse().unwrap(), "gpu-1", "session-1").await; + assert!(started.elapsed() < Duration::from_secs(6), "{:?}", started.elapsed()); + } } diff --git a/crates/fleet-proto/proto/fleet/v1/fleet.proto b/crates/fleet-proto/proto/fleet/v1/fleet.proto index 9d5d400..c756453 100644 --- a/crates/fleet-proto/proto/fleet/v1/fleet.proto +++ b/crates/fleet-proto/proto/fleet/v1/fleet.proto @@ -124,6 +124,8 @@ message DeregisterResponse {} // Machine agents use their own local Ray instance. No Ray cluster crosses machines. service WorkerService { rpc Heartbeat(WorkerHeartbeatRequest) returns (WorkerHeartbeatResponse); + // A stopping agent ends its session, so the agent that replaces it is admitted at once. + rpc EndSession(WorkerEndSessionRequest) returns (WorkerEndSessionResponse); } message NodeCommand { @@ -148,6 +150,14 @@ message WorkerHeartbeatRequest { fleet.model.v1.NodeReport report = 5; } +// Only the session itself can end: another agent's session stays until it is quiet. +message WorkerEndSessionRequest { + string worker_id = 1; + string session = 2; +} + +message WorkerEndSessionResponse {} + message WorkerHeartbeatResponse { uint32 heartbeat_secs = 1; repeated NodeCommand commands = 2; diff --git a/crates/fleet-scheduler/src/workloads.rs b/crates/fleet-scheduler/src/workloads.rs index cfc4f8d..2bc77cf 100644 --- a/crates/fleet-scheduler/src/workloads.rs +++ b/crates/fleet-scheduler/src/workloads.rs @@ -16,8 +16,8 @@ use axum::{ Json, Router, }; use fleet_proto::fleet::v1::{ - worker_service_server::WorkerService, NodeCommand, NodeOperation, WorkerHeartbeatRequest, - WorkerHeartbeatResponse, + worker_service_server::WorkerService, NodeCommand, NodeOperation, WorkerEndSessionRequest, + WorkerEndSessionResponse, WorkerHeartbeatRequest, WorkerHeartbeatResponse, }; use parking_lot::Mutex; use serde::Serialize; @@ -95,6 +95,9 @@ struct NodeSession { seen: f64, report: NodeReport, draining: bool, + /// Its agent stopped and said so: the next agent's session takes over at once, without + /// waiting for this one to go quiet. The node counts as connected until then, as before. + ended: bool, } #[derive(Default)] struct WorkloadState { @@ -622,6 +625,18 @@ impl WorkloadController { self.notify.notify_waiters(); Ok(v) } + /// Ends `worker`'s agent session `session`, which its agent sends as it stops: the agent + /// that replaces it is admitted at once. Another session than the current one changes + /// nothing, so one agent can't end another's. + pub(crate) fn end_session(&self, worker: &str, session: &str) -> bool { + let mut i = self.inner.lock(); + let Some(w) = i.workers.get_mut(worker).filter(|w| w.session == session) else { + return false; + }; + w.ended = true; + tracing::info!(worker, "agent session ended; the next agent session takes over at once"); + true + } async fn report( &self, worker_id: &str, @@ -677,7 +692,7 @@ impl WorkloadController { let mut i = self.inner.lock(); if i.workers .get(worker_id) - .is_some_and(|w| w.session != session && now() - w.seen < 30.0) + .is_some_and(|w| w.session != session && !w.ended && now() - w.seen < 30.0) { return Err(fail(409, "another agent session is active for this worker")); } @@ -692,6 +707,7 @@ impl WorkloadController { seen: now(), report: report.clone(), draining, + ended: false, }, ); } @@ -1559,6 +1575,14 @@ fn heartbeat_commands(i: &WorkloadState, worker: &str) -> Vec { pub struct NodeControlService(pub Arc); #[tonic::async_trait] impl WorkerService for NodeControlService { + async fn end_session( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let r = request.into_inner(); + self.0.end_session(&r.worker_id, &r.session); + Ok(tonic::Response::new(WorkerEndSessionResponse {})) + } async fn heartbeat( &self, request: tonic::Request, diff --git a/crates/fleet-scheduler/tests/flow.rs b/crates/fleet-scheduler/tests/flow.rs index a8c6c37..bd07584 100644 --- a/crates/fleet-scheduler/tests/flow.rs +++ b/crates/fleet-scheduler/tests/flow.rs @@ -1474,6 +1474,42 @@ async fn machine_heartbeat_requires_cluster_token_and_populates_control_plane_no assert!(status["nodes"].as_array().unwrap().iter().any(|n| n["name"]=="gpu-1" && n["backend"]=="ray-local")); } +/// One agent session at a time drives a worker. An agent that stops ends its own session, and +/// the agent that replaces it is admitted at once; no one else can end it. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_agent_that_ends_its_session_hands_its_worker_to_the_next_at_once() { + use fleet_proto::fleet::v1::{worker_service_client::WorkerServiceClient, WorkerEndSessionRequest, + WorkerHeartbeatRequest, + }; + let h = start().await; + let channel = Channel::from_shared(format!("http://127.0.0.1:{}", h.control_port)).unwrap().connect().await.unwrap(); + let mut agent = WorkerServiceClient::with_interceptor(channel.clone(), + fleet_scheduler::token::BearerTokenInterceptor::new(TOKEN).unwrap()); + let report = serde_json::from_value::(serde_json::json!({ + "version": 3, "machine": "gpu-machine", "ready": true, + "total_gpus": 8, "total_cpus": 48, "free_gpus": 8, "free_cpus": 48, + })).unwrap(); + let heartbeat = |session: &str| WorkerHeartbeatRequest { worker_id: "gpu-1".into(), session: session.into(), + report: Some(report.clone()), proof_fetch_ready: false, + }; + let end = |session: &str| WorkerEndSessionRequest { worker_id: "gpu-1".into(), session: session.into() }; + agent.heartbeat(heartbeat("old")).await.unwrap(); + let refused = agent.heartbeat(heartbeat("new")).await.unwrap_err(); + assert_eq!(refused.code(), tonic::Code::FailedPrecondition); + assert!(refused.message().contains("another agent session is active"), "{}", refused.message()); + // Another session than the active one ends nothing, and the call needs the cluster token. + agent.end_session(end("new")).await.unwrap(); + assert_eq!(agent.heartbeat(heartbeat("new")).await.unwrap_err().code(), tonic::Code::FailedPrecondition); + let mut anonymous = WorkerServiceClient::new(channel); + assert_eq!(anonymous.end_session(end("old")).await.unwrap_err().code(), tonic::Code::Unauthenticated); + assert_eq!(agent.heartbeat(heartbeat("new")).await.unwrap_err().code(), tonic::Code::FailedPrecondition); + + agent.end_session(end("old")).await.unwrap(); + agent.heartbeat(heartbeat("new")).await.unwrap(); + assert_eq!(agent.heartbeat(heartbeat("old")).await.unwrap_err().code(), tonic::Code::FailedPrecondition, + "the ended session doesn't come back while the new one is active"); +} + #[tokio::test(flavor = "multi_thread")] async fn integrated_app_worker_rejects_sdk_jobs_in_its_own_namespace() { use fleet_proto::fleet::v1::{worker_service_client::WorkerServiceClient, WorkerHeartbeatRequest, diff --git a/docs/maintenance.md b/docs/maintenance.md index 1e73e69..d5dda65 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -65,6 +65,14 @@ Ray keeps running, and so do apps and batches, through the agent restart. The up touches neither `etc/` (watchdog.env, apps.yaml, batches.yaml, tokens) nor `work/`; leave app endpoints running when their programs are unchanged. +One agent session at a time drives a node. A stopping agent ends its session, so the +scheduler admits the restarted agent at once. An agent that crashed, or one stopped +while the scheduler was out of reach, leaves its session open: the next agent is +refused ("another agent session is active for this worker") until the open session +has been quiet for 30 seconds. Ending a session needs a scheduler at least as new as +the agent, so upgrade the control host first. A remote `-n` run ships nothing to the node, +and the node's part of it prints the steps for the files it would have received. + #### Cron-supervised nodes `node upgrade` and `node rollback` need a systemd unit. On a cron-supervised node, From bc6fadafdc7d4f7d227130a5e8268b94c80a5724 Mon Sep 17 00:00:00 2001 From: eigmax Date: Thu, 12 Nov 2026 13:36:13 +0000 Subject: [PATCH 3/4] Delete apps and batches the same way Deleting an app and deleting a batch are one operation: refused while any of its work runs or may run, then the record goes, every node drops it from its configuration, and its history stays. A batch's work is its queued or running jobs. An app's work is its replicas: it is disabled first, and every node that configures it has stopped them. A refusal now says what still runs and how to stop it. A batch was refused with "disable the app or finish/cancel jobs before deleting", which told a batch owner to disable an app. A node that no longer configures an app no longer holds up its delete forever. An offline node still does, since it can't confirm its replicas stopped. fleetctl batch delete now says what it would delete without --yes, as app delete does. After deleting it says what was deleted and what stays, instead of "updated batch". The help of both commands and the reference describe the shared rule. --- crates/fleet-scheduler/src/workloads.rs | 54 ++++++++++++------- crates/fleet-scheduler/src/workloads_tests.rs | 43 +++++++++++++++ crates/fleetctl/src/main.rs | 36 +++++++++---- crates/fleetctl/tests/apps.rs | 22 ++++++++ docs/reference.md | 23 ++++++-- 5 files changed, 144 insertions(+), 34 deletions(-) diff --git a/crates/fleet-scheduler/src/workloads.rs b/crates/fleet-scheduler/src/workloads.rs index 2bc77cf..5f90869 100644 --- a/crates/fleet-scheduler/src/workloads.rs +++ b/crates/fleet-scheduler/src/workloads.rs @@ -499,6 +499,38 @@ impl WorkloadController { keys.iter().filter(|k| i.deployments.get(*k).and_then(|d| i.workers.get(&d.worker)).is_some_and(|w| now() - w.seen < 30.0)).cloned().collect() } + /// What of app or batch `app` may still run, each with how to stop it; nothing when it can + /// be deleted. Deleting an app or a batch is the same operation: it is refused while any + /// of its work runs or may run. A batch's work is its jobs, queued or running. An app's + /// is its replicas: it is disabled first, and every node that configures it confirms them + /// stopped. An offline node can't, so it holds the delete until it's back. + fn still_running(&self, app: &WorkloadTemplate) -> Vec { + let name = app.name.as_str(); + let i = self.inner.lock(); + let mut running = Vec::new(); + let jobs = i.jobs.open().filter(|r| r.batch == name && (!terminal(r.state()) || r.quota_held)).count(); + if jobs > 0 { + running.push(format!("{jobs} job(s) queued or running: cancel them or let them finish")); + } + if !app.persistent { + return running; + } + if !app.disabled() { + running.push(format!("it is enabled: disable it first (fleetctl app disable {name})")); + return running; + } + for worker in &app.workers { + match i.workers.get(worker).filter(|w| now() - w.seen < 30.0) { + None => running.push(format!("node {worker} is offline, so it can't confirm its replicas stopped")), + // A node that no longer configures it has nothing of it left. + Some(w) => if w.report.entries().any(|e| e.name() == name && !e.cleaned()) { + running.push(format!("its replicas on node {worker} are still stopping")); + }, + } + } + running + } + /// Whether a batch still has job records (not purged). fn has_jobs(&self, batch: &str) -> bool { self.inner.lock().jobs.values().any(|r| r.batch == batch && !r.purged) @@ -1963,24 +1995,10 @@ async fn handle_result(s: Arc, request: Request) -> anyhow:: if method == Method::DELETE && parts.len() == 2 { let _ops = s.ops.lock().await; s.ensure_not_scaling(&app.namespace, name)?; - let persistent_ready_to_remove = !app.persistent - || (app.disabled() - && app.workers.iter().all(|id| { - s.inner.lock().workers.get(id).is_some_and(|w| { - now() - w.seen < 30.0 && w.report.entries().any(|v| v.name() == name && v.cleaned()) - }) - })); - if !persistent_ready_to_remove - || s.inner - .lock() - .jobs - .open() - .any(|r| r.batch == name && (!terminal(r.state()) || r.quota_held)) - { - return Err(fail( - 409, - "disable the app or finish/cancel jobs before deleting", - )); + let running = s.still_running(&app); + if !running.is_empty() { + let kind = if batch { "batch" } else { "app" }; + return Err(fail(409, format!("{kind} {name} still runs: {}; delete it once nothing of it does", running.join("; ")))); } app.removed = true; s.save("apps", name, &app)?; diff --git a/crates/fleet-scheduler/src/workloads_tests.rs b/crates/fleet-scheduler/src/workloads_tests.rs index 125124d..97fa90e 100644 --- a/crates/fleet-scheduler/src/workloads_tests.rs +++ b/crates/fleet-scheduler/src/workloads_tests.rs @@ -2086,6 +2086,49 @@ async fn deleting_a_batch_sends_remove_batch_and_keeps_its_jobs() { assert!(c.job("other-job",None,&BTreeMap::new()).is_err(),"a deleted batch imports no jobs"); } +/// Deleting an app and deleting a batch are one operation: refused while any of its work +/// runs or may run, with what that is and how to stop it. A batch's work is its jobs; an app's +/// is its replicas, which a node that no longer configures it doesn't have. +#[tokio::test] +async fn deleting_an_app_or_a_batch_is_refused_while_any_of_it_runs() { + let (_dir, c, ns) = setup().await; + let refusal = |c: Arc, path: String| async move { + let response = scale_call(c, "admin", "DELETE", &path, Value::Null).await; + let status = response.status(); + (status, String::from_utf8_lossy(&to_bytes(response.into_body(), 1 << 20).await.unwrap()).into_owned()) + }; + let (batch, app) = (format!("/batches/training?namespace={ns}"), format!("/apps/model?namespace={ns}")); + + c.report("gpu", "session", scale_inventory(&ns, false, false)).await.unwrap(); + let job = submit(&c).await; + let (status, body) = refusal(c.clone(), batch.clone()).await; + assert_eq!(status, StatusCode::CONFLICT); + assert!(body.contains("batch training still runs: 1 job(s) queued or running: cancel them or let them finish"), "{body}"); + let (status, body) = refusal(c.clone(), app.clone()).await; + assert_eq!(status, StatusCode::CONFLICT); + assert!(body.contains("app model still runs: it is enabled: disable it first (fleetctl app disable model)"), "{body}"); + + c.inner.lock().apps.get_mut("model").unwrap().spec["disabled"] = json!(true); + let (_, body) = refusal(c.clone(), app.clone()).await; + assert!(body.contains("its replicas on node gpu are still stopping"), "{body}"); + c.inner.lock().workers.get_mut("gpu").unwrap().seen = 0.0; + let (_, body) = refusal(c.clone(), app.clone()).await; + assert!(body.contains("node gpu is offline, so it can't confirm its replicas stopped"), "{body}"); + + c.inner.lock().jobs.update(&job.id, |j| j.status["state"] = json!("cancelled")); + assert_eq!(refusal(c.clone(), batch).await.0, StatusCode::OK); + + // A node the app was on that no longer configures it holds nothing of it: only a node + // that configures it does, until it has stopped it. + let (_dir, c, ns) = setup().await; + c.report("gpu", "session", scale_inventory(&ns, false, true)).await.unwrap(); + c.report("cpu", "session", inventory(&ns)).await.unwrap(); + c.inner.lock().apps.get_mut("model").unwrap().workers.push("cpu".into()); + c.inner.lock().apps.get_mut("model").unwrap().spec["disabled"] = json!(true); + let (status, body) = refusal(c.clone(), format!("/apps/model?namespace={ns}")).await; + assert_eq!(status, StatusCode::OK, "{body}"); +} + fn command_where(c: &WorkloadController, pick: impl Fn(&CommandRecord) -> bool) -> Option { c.inner.lock().commands.values().find(|cmd| pick(cmd)).cloned() } diff --git a/crates/fleetctl/src/main.rs b/crates/fleetctl/src/main.rs index 1a2e9ff..c521172 100644 --- a/crates/fleetctl/src/main.rs +++ b/crates/fleetctl/src/main.rs @@ -134,9 +134,10 @@ enum AppCmd { /// Select a machine agent when the app runs on several nodes. #[arg(long)] node: Option, }, - /// Delete an app, refused while any of its jobs is queued or running. A proof app goes - /// with its jobs' records, inputs and proofs, its anchor, and its ELF when no other app - /// uses it; a running app must be disabled first. + /// Delete an app and remove it from its nodes, like batch delete: refused while any of it + /// runs, so disable a running app first, and wait for a proof app's queued or running + /// jobs. Its past execution records stay; a proof app goes with its jobs' records, inputs + /// and proofs, its anchor, and its ELF when no other app uses it. /// Without --yes it only says what it would delete. Delete { name: String, @@ -553,7 +554,9 @@ enum BatchCmd { /// K=V every batch job gets; repeat. Replaces the batch's environment. #[arg(long)] env: Vec, }, - /// Delete a batch after all accepted jobs finish; its job history stays until purged. + /// Delete a batch and remove it from its nodes, like app delete: refused while any of its + /// jobs is queued or running, so cancel them or let them finish first. Its job records + /// stay until purged. Without --yes it only says what it would delete. Delete { name: String, #[arg(long)] yes: bool }, #[command(flatten)] Job(BatchJobCmd), @@ -1209,6 +1212,20 @@ fn describe_app(name: &str, conn: &ConnectionOptions, out: &OutputWriter) -> any Ok(()) } +/// Deletes a batch (after showing what goes with it, until --yes), as `delete_app` deletes an app. +fn delete_batch(a: &FleetClient, name: &str, batch: &Value, yes: bool, out: &OutputWriter) -> anyhow::Result<()> { + let (total, active) = (&batch["jobs"]["total"], &batch["jobs"]["active"]); + if !yes { + bail!( + "would delete batch {name} and remove it from its nodes; its {total} job record(s) stay until \ + purged. Refused while any job is queued or running ({active} now). Pass --yes to delete" + ); + } + let v = a.send(a.req(reqwest::Method::DELETE, &format!("/batches/{name}")))?; + out.done(&format!("deleted batch {name}; its {total} job record(s) stay until purged"), v); + Ok(()) +} + /// Deletes an app (after showing what goes with it, until --yes). fn delete_app(name: &str, yes: bool, conn: &ConnectionOptions, out: &OutputWriter) -> anyhow::Result<()> { let a = FleetClient::new(conn)?; @@ -1224,8 +1241,8 @@ fn delete_app(name: &str, yes: bool, conn: &ConnectionOptions, out: &OutputWrite ); } bail!( - "would remove app {name}; its {total} past execution record(s) \ - stay ({active} queued or running: refused while any is). Pass --yes to delete" + "would delete app {name} and remove it from its nodes; its {total} past execution record(s) \ + stay. Refused while it runs: disable it first. Pass --yes to delete" ); } let v = a.send(a.req(reqwest::Method::DELETE, &format!("/apps/{name}")))?; @@ -1338,17 +1355,14 @@ fn batch(p: &BatchArgs, out: &OutputWriter) -> anyhow::Result<()> { | BatchCmd::Priority { name, .. } | BatchCmd::Delete { name, .. } => name, _ => unreachable!(), }; - batch_named(&a, name)?; + let batch = batch_named(&a, name)?; let path = format!("/batches/{name}"); let request = match cmd { BatchCmd::Disable { .. } => a.req(reqwest::Method::PUT, &format!("{path}/disabled")), BatchCmd::Enable { .. } => a.req(reqwest::Method::DELETE, &format!("{path}/disabled")), BatchCmd::Priority { priority, .. } => a.req(reqwest::Method::PUT, &format!("{path}/priority")) .json(&json!({"priority": priority})), - BatchCmd::Delete { yes, .. } => { - if !yes { bail!("would remove batch {name}; accepted jobs must finish first. Pass --yes to delete"); } - a.req(reqwest::Method::DELETE, &path) - } + BatchCmd::Delete { yes, .. } => return delete_batch(&a, name, &batch, *yes, out), _ => unreachable!(), }; let value = a.send(request)?; diff --git a/crates/fleetctl/tests/apps.rs b/crates/fleetctl/tests/apps.rs index 98fd7bb..131fb1d 100644 --- a/crates/fleetctl/tests/apps.rs +++ b/crates/fleetctl/tests/apps.rs @@ -277,6 +277,28 @@ fn app_delete_says_what_goes_until_confirmed() { assert!(seen[2].line.starts_with("DELETE /apps/ethproofs?namespace=alice"), "{}", seen[2].line); } +#[test] +fn batch_delete_says_what_goes_until_confirmed_like_app_delete() { + let batch = r#"{"type":"batch","name":"training","persistent":false,"jobs":{"total":5,"active":1}}"#; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let token = token_file("batch-delete"); + let server = stub(listener, 3, move |line, _| (200, if line.starts_with("GET") { batch.into() } else { r#"{"name":"training","removed":true}"#.into() })); + let refused = cli().args(["--namespace", "alice", "--url", &url, "--token-file"]).arg(&token) + .args(["batch", "delete", "training"]).output().unwrap(); + assert!(!refused.status.success()); + let preview = String::from_utf8_lossy(&refused.stderr); + assert!(preview.contains("would delete batch training and remove it from its nodes; its 5 job record(s) stay until purged"), "{preview}"); + assert!(preview.contains("(1 now). Pass --yes to delete"), "{preview}"); + let deleted = cli().args(["--namespace", "alice", "--url", &url, "--token-file"]).arg(&token) + .args(["batch", "delete", "training", "--yes"]).output().unwrap(); + let seen = server.join().unwrap(); + std::fs::remove_file(token).unwrap(); + assert!(deleted.status.success(), "{}", String::from_utf8_lossy(&deleted.stderr)); + assert!(String::from_utf8_lossy(&deleted.stdout).contains("deleted batch training; its 5 job record(s) stay until purged")); + assert!(seen[2].line.starts_with("DELETE /batches/training?namespace=alice"), "{}", seen[2].line); +} + /// Two batches as `GET /batches` returns them: the app record's shape, with the policy under `batch`. fn batch_list() -> String { serde_json::json!([ diff --git a/docs/reference.md b/docs/reference.md index 2b82702..fe74fe1 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -200,9 +200,20 @@ administrator, since an app can reach host networking, storage and prover credentials; an update mixing them with other fields is refused whole. Changing priority alone doesn't restart replicas. -**Removing.** A proof app can't be removed while any of its proofs is queued or -running. Removing it deletes its finished proof records and control-host artifacts; -the app's own storage stays with its owner. +**Deleting.** `fleetctl app delete NAME --yes` and `fleetctl batch delete NAME --yes` +work the same way. Without `--yes` they only say what would go. A delete is refused +while any of the app's or batch's work runs or may run, and the refusal says what that +is and how to stop it: +- An app's work is its replicas. Disable it first, and wait until every node that + configures it has stopped them; an offline node holds the delete until it's back. +- A proof app's work also includes its queued or running proofs. +- A batch's work is its queued or running jobs. Cancel them or let them finish. + +The delete then removes the record and has every node remove the app's entries or the +batch from its configuration. The data on the nodes stays. A container app's past +execution records and a batch's job records stay until purged. A proof app's finished +proof records and control-host artifacts go with it, and the app's own storage stays +with its owner. **Node entries.** Every `apps.yaml` entry has an `app`. A `ziren-gpu` entry binds its server to the one app it serves with `namespace`, `app: NAME` and @@ -259,8 +270,10 @@ In the API, `POST /batches?namespace=alice` takes the batch JSON, or the namespa its `namespace` field; a name or an ID is accepted, and the ID is stored. A batch or `apps.yaml` entry configured on a node may name `namespace: alice`, which is checked before launch. Disabling a batch blocks new submissions and lets accepted jobs -finish. A deleted batch stays listed, as `deleted`, and readable while it has job -records; its jobs keep their own lifecycle until they're purged. +finish. Deleting a batch works like deleting an app ([Deleting](#apps)): it is +refused while any of its jobs is queued or running. A deleted batch stays listed, as +`deleted`, and readable while it has job records; its jobs keep their own lifecycle +until they're purged. **Submissions.** A job takes `command`, a non-empty list of strings, and optionally: From 906d6146ff9c7487c6f05262914df0f8fd5c5e55 Mon Sep 17 00:00:00 2001 From: eigmax Date: Thu, 12 Nov 2026 14:36:12 +0000 Subject: [PATCH 4/4] Delete an app or batch with everything Fleet holds of it, and drop purge Deleting an app or a batch now removes it with its data, on the control host and on its nodes, and is the only way job history goes: - Control host: the batch's job records with their logs, and the commands about them and the app or batch. A proof app's job rows and anchor go from Postgres. - Nodes: the configuration, and what the node holds of it. A batch's jobs lose their directories (record, logs, out/). An app's replicas' logs and workers' directories go, with its proof jobs' files in its proof storage: each job's output directory, and the inputs and ELF no other job or app uses. - Kept: volumes, batch mounts, env files, storage roots, images, and whatever holds or lies in a path another app or batch uses on any node, the filesystem's top level and Fleet's own directories. The node reports what it kept and why. Only a deletion's removal, marked with "delete", deletes files; a scale that takes an app off one node still keeps its data there. The record stays hidden until its nodes have removed it, so a late node report can't bring it back and the name can't be reused before then; deleting again retries a node that failed. Deleted records left by earlier releases go at start once nothing of them is left, and a batch deleted earlier with job records stays listed as deleted until it is deleted again. The purge subcommand and its API routes are gone, with the purge ordering of node commands. --- bridge/fleet_bridge/node.py | 169 +++++++-- bridge/tests/test_node.py | 130 ++++++- .../proto/fleet/model/v1/model.proto | 1 + crates/fleet-proto/proto/fleet/v1/types.proto | 5 + crates/fleet-proto/src/workload.rs | 14 +- crates/fleet-scheduler/src/fleet.rs | 34 +- crates/fleet-scheduler/src/http.rs | 2 +- crates/fleet-scheduler/src/store.rs | 46 ++- crates/fleet-scheduler/src/token.rs | 2 +- crates/fleet-scheduler/src/workloads.rs | 349 ++++++++++++------ .../fleet-scheduler/src/workloads_records.rs | 6 + crates/fleet-scheduler/src/workloads_scale.rs | 3 +- crates/fleet-scheduler/src/workloads_tests.rs | 242 +++++++----- crates/fleet-scheduler/tests/flow.rs | 5 +- crates/fleet-scheduler/tests/pg.rs | 18 +- crates/fleetctl/src/main.rs | 77 ++-- crates/fleetctl/tests/apps.rs | 19 +- docs/maintenance.md | 6 +- docs/reference.md | 37 +- 19 files changed, 819 insertions(+), 346 deletions(-) diff --git a/bridge/fleet_bridge/node.py b/bridge/fleet_bridge/node.py index d98ee67..49c5628 100644 --- a/bridge/fleet_bridge/node.py +++ b/bridge/fleet_bridge/node.py @@ -8,6 +8,7 @@ import json import logging import os +import re import shutil import socket import subprocess @@ -183,6 +184,11 @@ def append_entry(path: Path, section: str, entry: dict[str, Any]) -> None: raise ValueError(f"{section}: must be a list to add an entry to") _replace(path, text) +def _holds(outer: str, inner: str) -> bool: + """Whether directory `outer` is `inner` or holds it.""" + return os.path.commonpath([outer, inner]) == outer + + class NodeExecutor: def __init__(self, apps: str, batches: str, jobs_dir: str) -> None: self.apps_file, self.batches, self.jobs_dir = Path(apps), batches, Path(jobs_dir) @@ -239,20 +245,11 @@ def execute(self, command: dict[str, Any]) -> None: elif operation == 4: with self.apps_lock: result = self.set_app(params) - elif operation == 5: - directory = self.jobs_dir / params["id"] - if directory.exists(): - current = read_json(directory / ".fleet-lease.json", {"lease": "imported"})["lease"] - if current != params["lease"]: - raise ValueError("stale purge lease") - job = jobs.JobDirectory(str(self.jobs_dir), params["id"]) - if not self.cleanup(job): - raise ValueError("job cleanup is not confirmed") - shutil.rmtree(job.path) - result = {"cleaned": True} elif operation == 7: with self.apps_lock: - result = self.app(params) + result, then = self.app(params) + # Deleting files can take long; other app changes needn't wait for it. + result.update(then()) else: raise ValueError("unsupported node operation") except BaseException as e: @@ -410,15 +407,24 @@ def _remove_replicas(names: list[str]) -> None: if not container_gone(actors.gpu_container(name)) or not container_gone(actors.app_container(name)): raise ValueError(f"{name}: app container cleanup is uncertain") - def app(self, p: dict[str, Any]) -> dict[str, Any]: + def app(self, p: dict[str, Any]) -> tuple[dict[str, Any], Any]: """An app-level change from the scheduler, for an app (`p["app"]`: the entries that run the app of that name, in a namespace given by id and by name), a batch (`p["batch"]`, by name), or the one entry a change of an app stopped (`p["entry"]`, by name, set only): `remove` it, or `set` some of its configuration. - A removed entry has what is left of its replicas cleaned up the way disabling does; - app data stays (storage roots, workdirs, env files and images belong to the operator). Settings are written to the configuration only: the scheduler drains and stops an app first and starts it again afterwards. `p["create"]` adds a new app's - entry to the configuration, disabled until the app is enabled.""" + entry to the configuration, disabled until the app is enabled. + + A removed entry has what is left of its replicas cleaned up the way disabling does. + Removing an app from this node alone (a scale left it no replicas here) keeps its + data. Deleting it (`p["delete"]`) removes what this node holds of it too: its + replicas' logs, its worker's directory and the files of its proof jobs + (`delete["files"]`) in its proof storage. A deleted batch's jobs (`delete["jobs"]`) + lose their directories: record, logs and outputs. Volumes, mounts, env files, + storage roots and images stay, and so does whatever holds or lies in a path another + app or batch uses, here or (by `delete["keep"]`) on another node. Returns the + result, and what deletes the files, for after the configuration lock.""" + nothing = lambda: {} if "create" in p: entry = p["create"] AppSpec.load(entry) @@ -426,26 +432,29 @@ def app(self, p: dict[str, Any]) -> dict[str, Any]: if any(e.get("name") == entry["name"] for e in entries): raise ValueError(f"{entry['name']} is already configured on this node") append_entry(self.apps_file, "apps", entry) - return {"created": [entry["name"]]} + return {"created": [entry["name"]]}, nothing if "batch" in p: name, path = p["batch"]["name"], Path(self.batches) if p.get("remove"): - return {"removed": [name] if remove_entry(path, "batches", name) else []} + removed = [name] if remove_entry(path, "batches", name) else [] + if "delete" not in p: + return {"removed": removed}, nothing + return {"removed": removed}, lambda: self._delete_jobs(name, p["delete"].get("jobs") or []) entry = (yaml.safe_load(path.read_text()) or {}).get("batches", {}).get(name) if entry is None: - return {"updated": []} + return {"updated": []}, nothing Batch.load({**entry, **p["set"]}) set_fields(path, "batches", name, p["set"]) - return {"updated": [name]} + return {"updated": [name]}, nothing if "entry" in p: # The one entry a change of an app stopped: its configuration only. name = p["entry"]["name"] entries = {e["name"]: e for e in (yaml.safe_load(self.apps_file.read_text()) or {}).get("apps", [])} if p.get("remove") or name not in entries: - return {"updated": []} + return {"updated": []}, nothing AppSpec.load({**entries[name], **p["set"]}) set_fields(self.apps_file, "apps", name, p["set"]) - return {"updated": [name]} + return {"updated": [name]}, nothing target = p["app"] namespaces = {target["namespace"], target.get("namespace_name") or target["namespace"]} entries = {e["name"]: e for e in (yaml.safe_load(self.apps_file.read_text()) or {}).get("apps", [])} @@ -457,13 +466,123 @@ def app(self, p: dict[str, Any]) -> dict[str, Any]: AppSpec.load({**entries[app.name], **p["set"]}) for app in apps: set_fields(self.apps_file, "apps", app.name, p["set"]) - return {"updated": [a.name for a in apps]} + return {"updated": [a.name for a in apps]}, nothing removed = [] for app in apps: self._remove_replicas([f"{app.name}-{index}" for index in range(app.replicas)]) if remove_entry(self.apps_file, "apps", app.name): - removed.append(app.name) - return {"removed": removed} + removed.append(app) + if "delete" not in p: + return {"removed": [a.name for a in removed]}, nothing + # What the node's other apps and batches use, read under the lock. + in_use = self._paths_in_use(p["delete"].get("keep") or [], removed) + log_dir = cfg.cluster.log_dir + return ({"removed": [a.name for a in removed]}, + lambda: self._delete_app_files(removed, log_dir, in_use, p["delete"].get("files") or [])) + + def _paths_in_use(self, keep: list[str], removed: list[AppSpec]) -> list[str]: + """The host paths that a deletion keeps, with what holds or lies in one: those of + the apps and batches this node still configures, `keep` (the ones on other nodes), + and the volumes and env files of the `removed` entries.""" + paths = list(keep) + for e in (yaml.safe_load(self.apps_file.read_text()) or {}).get("apps") or []: + paths += [e.get(k) for k in ("workdir", "worker_workdir", "proof_storage_root", "worker_env_file")] + paths += [str(v).split(":")[0] for v in e.get("volumes") or []] + for b in ((yaml.safe_load(Path(self.batches).read_text()) or {}).get("batches") or {}).values(): + paths += list((b or {}).get("allow") or []) + for a in removed: + paths += [v.split(":")[0] for v in a.volumes] + [a.worker_env_file] + return [os.path.realpath(p) for p in paths if isinstance(p, str) and os.path.isabs(p)] + + def _refusal(self, path: str, in_use: list[str], log_dir: str, *, inside_ok: bool = False) -> str | None: + """Why `path` must stay, or None when it can go. Fleet's own directories stay, with + what holds or lies in one; so do the filesystem's top level and what holds the home + directory, and what holds a path in use or (unless `inside_ok`) lies in one.""" + if not os.path.isabs(path) or ".." in Path(path).parts: + return "not an absolute path" + if os.path.islink(path): + return "a symbolic link" + real = os.path.realpath(path) + if len(Path(real).parts) < 3 or _holds(real, os.path.realpath(Path.home())): + return "too close to the root of the filesystem" + for own in (self.jobs_dir, self.apps_file.parent, Path(self.batches).parent, log_dir, Path.cwd()): + own = os.path.realpath(own) + if _holds(real, own) or _holds(own, real): + return f"holds or lies in fleet's own {own}" + for used in in_use: + if _holds(real, used) or not inside_ok and _holds(used, real): + return f"another app or batch uses {used}" + return None + + def _delete(self, path: str, reason: str | None, deleted: list[str], kept: list[dict[str, str]]) -> None: + if reason is None: + try: + if os.path.isdir(path) and not os.path.islink(path): + shutil.rmtree(path) + else: + os.unlink(path) + deleted.append(path) + return + except OSError as e: + reason = f"{type(e).__name__}: {e.strerror or e}" + kept.append({"path": path, "reason": reason}) + + def _delete_app_files(self, removed: list[AppSpec], log_dir: str, in_use: list[str], files: list[str]) -> dict[str, Any]: + """Deletes what the `removed` entries of a deleted app held: the files of its proof + jobs in their proof storage, their replicas' logs and their workers' directories.""" + deleted: list[str] = [] + kept: list[dict[str, str]] = [] + roots = [os.path.realpath(a.proof_storage_root) for a in removed if a.proof_storage_root] + for path in files if roots else []: + if not os.path.lexists(path): + continue + real = os.path.realpath(path) + if not any(real != root and _holds(root, real) for root in roots): + kept.append({"path": path, "reason": "not in the app's proof storage on this node"}) + continue + self._delete(path, self._refusal(path, in_use, log_dir, inside_ok=True), deleted, kept) + logs = sorted(os.listdir(log_dir)) if os.path.isdir(log_dir) else [] + for a in removed: + pattern = re.compile(rf"{re.escape(a.name)}-\d+\.node\.log") + for name in filter(pattern.fullmatch, logs): + self._delete(os.path.join(log_dir, name), None, deleted, kept) + if a.worker_workdir and os.path.lexists(a.worker_workdir): + self._delete(a.worker_workdir, self._refusal(a.worker_workdir, in_use, log_dir), deleted, kept) + if deleted or kept: + log.info("deleted app's files: %d deleted, kept %s", len(deleted), kept) + return {"deleted": deleted, "kept": kept} + + def _delete_jobs(self, batch: str, ids: list[str]) -> dict[str, Any]: + """Deletes the directories of the jobs `ids` of deleted batch `batch`: each one's + record, logs and outputs. One whose container may still be there stays.""" + deleted, kept = 0, [] + for ident in ids: + try: + job = jobs.JobDirectory(str(self.jobs_dir), ident) + except jobs.NotFound: + continue + directory = Path(job.path) + try: + try: + owner = job.spec().batch + except Exception: + owner = batch # a record half written: the scheduler lists it as the batch's + if owner != batch: + kept.append({"path": job.path, "reason": f"a job of batch {owner}"}) + continue + lease = read_json(directory / ".fleet-lease.json", {"lease": "imported"})["lease"] + if not (directory / f".fleet-cleaned-{lease}").exists() and not self.cleanup(job): + kept.append({"path": job.path, "reason": "its container may still be there"}) + continue + shutil.rmtree(directory) + for stop in (self.jobs_dir / ".node" / "stops").glob(f"{ident}-*.json"): + stop.unlink(missing_ok=True) + deleted += 1 + except Exception as e: + kept.append({"path": job.path, "reason": f"{type(e).__name__}: {e}"}) + if deleted or kept: + log.info("deleted batch %s: %d job directories deleted, kept %s", batch, deleted, kept) + return {"jobs_deleted": deleted, "kept": kept} def report(self) -> dict[str, Any]: nodes = alive_nodes() diff --git a/bridge/tests/test_node.py b/bridge/tests/test_node.py index 4e7d2d3..8030c1b 100644 --- a/bridge/tests/test_node.py +++ b/bridge/tests/test_node.py @@ -1,4 +1,5 @@ """Local execution fences are independent of user authentication and admission.""" +import os from unittest.mock import Mock import pytest import yaml @@ -93,14 +94,6 @@ def test_start_uses_local_executor_and_no_admin_token(executor,monkeypatch): assert all(not c.kwargs for c in start.call_args_list) -def test_purge_refuses_an_old_execution_lease(executor): - c=command(5);path=executor.jobs_dir/c["parameters"]["id"];path.mkdir(parents=True) - node.write_json(path/".fleet-lease.json",{"lease":"new-lease"}) - executor.execute(c) - assert path.exists() - assert "stale purge" in node.read_json(executor.journal/"command-1.json")["error"] - - def test_local_ray_must_contain_only_this_machine(executor,monkeypatch): monkeypatch.setattr(node.ray,"nodes",lambda:[{"Alive":True},{"Alive":True}]) with pytest.raises(ValueError,match="only this machine"): @@ -295,7 +288,7 @@ def test_app_removal_cleans_up_and_drops_the_entry(executor,monkeypatch): executor.apps_file.write_text(yaml.safe_dump(apps)) assert app_message(executor,"remove-1",{"remove":True},app="eth")==({"removed":["proof"]},None) assert [c.args[0] for c in stop.call_args_list]==["proof-0","proof-1"] - assert app_message(executor,"remove-2",{"remove":True},app="eth")==({"removed":[]},None) + assert app_message(executor,"remove-2",{"remove":True,"delete":{}},app="eth")==({"removed":[],"deleted":[],"kept":[]},None) assert app_message(executor,"remove-3",{"remove":True},app="web",ns="1"*32,ns_name="zkm")==({"removed":["web"]},None) assert [s["name"] for s in yaml.safe_load(executor.apps_file.read_text())["apps"]]==["other-ns"] (executor.apps_file.parent/"batches.yaml").write_text("batches:\n old: {image: a}\n kept: {image: b}\n") @@ -303,21 +296,120 @@ def test_app_removal_cleans_up_and_drops_the_entry(executor,monkeypatch): assert list(yaml.safe_load((executor.apps_file.parent/"batches.yaml").read_text())["batches"])==["kept"] -def test_removing_an_app_keeps_its_data(executor,monkeypatch,tmp_path): - """A start placed on another node has its own node remove the entry; the app's storage is its own.""" +def removable(monkeypatch): + """Replicas that stop at once and leave no container.""" monkeypatch.setattr(reconcile,"stop",Mock());monkeypatch.setattr(actors,"clean_after",Mock()) monkeypatch.setattr(node.ray,"get",lambda *a,**k:None);monkeypatch.setattr(node,"container_gone",lambda _:True) - data={name:tmp_path/"data"/name for name in ("storage","workdir","models")} - for d in data.values(): - d.mkdir(parents=True);(d/"cursor").write_text("42") - env=tmp_path/"data"/"worker.env";env.write_text("RPC=x\n") + + +def files(root, *paths): + for path in paths: + (root/path).parent.mkdir(parents=True,exist_ok=True);(root/path).write_text("x") + + +def configure(executor, *entries, log_dir=None): apps=yaml.safe_load(executor.apps_file.read_text()) - apps["apps"]=[{"name":"eth-worker","kind":"ziren-gpu","namespace":"a"*32,"app":"eth","elf_id":"b"*64,"image":"ziren-gpu:1", - "machine":"m1","worker_command":["/opt/eth-proofs"],"worker_env_file":str(env),"worker_workdir":str(data["workdir"]), - "proof_storage_root":str(data["storage"]),"volumes":[f"{data['models']}:ro"]}] + apps["apps"]=list(entries) + if log_dir: + apps["log_dir"]=str(log_dir) executor.apps_file.write_text(yaml.safe_dump(apps)) + + +def worker(name, app, data, **more): + """An entry with an integrated worker, which works in data//run and keeps proofs in data//store.""" + return {"name":name,"kind":"ziren-gpu","namespace":"a"*32,"app":app,"elf_id":"b"*64,"image":"ziren-gpu:1", + "machine":"m1","worker_command":["/opt/worker"],"worker_workdir":str(data/app/"run"), + "proof_storage_root":str(data/app/"store"),**more} + + +def test_removing_an_app_from_one_node_keeps_its_data(executor,monkeypatch,tmp_path_factory): + """A scale that leaves a node no replicas has it remove the app's entries; the app runs on elsewhere.""" + removable(monkeypatch) + data=tmp_path_factory.mktemp("data") + files(data,"eth/run/checkpoint","eth/store/outputs/job-1/lease/proof.json","logs/eth-worker-0.node.log") + configure(executor,worker("eth-worker","eth",data),log_dir=data/"logs") assert app_message(executor,"remove-1",{"remove":True},app="eth")==({"removed":["eth-worker"]},None) - assert all((d/"cursor").read_text()=="42" for d in data.values()) and env.read_text()=="RPC=x\n" + assert len([f for f in data.rglob("*") if f.is_file()])==3 + + +def test_deleting_an_app_deletes_what_its_entries_held(executor,monkeypatch,tmp_path_factory): + """A deleted app's node deletes its replicas' logs, its worker's directory and the files of its proof + jobs in its proof storage. Its storage root, volumes and env file stay, and so does what isn't its own.""" + removable(monkeypatch) + data=tmp_path_factory.mktemp("data") + files(data,"eth/run/checkpoint","eth/store/outputs/job-1/lease/proof.json","eth/store/outputs/job-2/lease/proof.json", + "eth/store/inputs/i1","eth/store/elf/e1","models/weights","worker.env","elsewhere/proof.json", + "logs/eth-worker-0.node.log","logs/eth-worker-11.node.log","logs/eth-worker-db-0.node.log","logs/node.log") + configure(executor,worker("eth-worker","eth",data,worker_env_file=str(data/"worker.env"),volumes=[f"{data/'models'}:ro"]), + log_dir=data/"logs") + listed=[str(data/p) for p in ("eth/store/outputs/job-1","eth/store/inputs/i1","eth/store/elf/e1","eth/store/inputs/gone", + "eth/store","elsewhere/proof.json")] + result,error=app_message(executor,"remove-1",{"remove":True,"delete":{"keep":[],"files":listed}},app="eth") + assert error is None and result["removed"]==["eth-worker"] + assert sorted(result["deleted"])==sorted(str(data/p) for p in ("eth/store/outputs/job-1","eth/store/inputs/i1","eth/store/elf/e1", + "logs/eth-worker-0.node.log","logs/eth-worker-11.node.log","eth/run")) + assert result["kept"]==[{"path":str(data/p),"reason":"not in the app's proof storage on this node"} for p in ("eth/store","elsewhere/proof.json")] + left=sorted(str(f.relative_to(data)) for f in data.rglob("*") if f.is_file()) + assert left==["elsewhere/proof.json","eth/store/outputs/job-2/lease/proof.json","logs/eth-worker-db-0.node.log","logs/node.log", + "models/weights","worker.env"] + + +def test_deleting_an_app_keeps_what_another_app_or_batch_uses(executor,monkeypatch,tmp_path_factory): + """Its worker's directory stays when it holds or lies in a path that another app or batch uses: on this + node, as its configuration says, or on another, as the scheduler says. So does one that is too close to + the root or holds fleet's own files.""" + removable(monkeypatch) + data=tmp_path_factory.mktemp("data") + files(data,"web/run/state","shared/run/state") + cases=[ + ({}, [worker("web-db","db",data,volumes=[str(data/"web")])], "batches: {}\n", f"another app or batch uses {data/'web'}"), + ({"keep":[str(data/"web/run/state")]}, [], "batches: {}\n", f"another app or batch uses {data/'web/run/state'}"), + ({}, [], f"batches:\n render: {{image: a, allow: [{data}]}}\n", f"another app or batch uses {data}"), + ] + for n,(change,others,batches,reason) in enumerate(cases): + configure(executor,worker("web-gpu","web",data),*others) + (executor.apps_file.parent/"batches.yaml").write_text(batches) + result,error=app_message(executor,f"remove-{n}",{"remove":True,"delete":change},app="web") + assert error is None and result["kept"]==[{"path":str(data/"web/run"),"reason":reason}], result + assert (data/"web/run/state").exists() + (executor.apps_file.parent/"batches.yaml").write_text("batches: {}\n") + for n,(workdir,reason) in enumerate([("/opt","too close to the root of the filesystem"), + (str(executor.jobs_dir.parent),f"holds or lies in fleet's own {executor.jobs_dir}")]): + configure(executor,{**worker("web-gpu","web",data),"worker_workdir":workdir}) + executor.jobs_dir.mkdir(exist_ok=True) + result,error=app_message(executor,f"root-{n}",{"remove":True,"delete":{}},app="web") + assert result["kept"]==[{"path":workdir,"reason":reason}], result + assert os.path.isdir("/opt") and executor.apps_file.exists() + + +def job_directory(executor, ident, batch, *, cleaned=True): + directory=executor.jobs_dir/ident + (directory/"out").mkdir(parents=True) + (directory/"out/frame.png").write_text("output") + JobSpec(id=ident, batch=batch, policy=Batch(image="render:1"), command=["true"], gpus=0, cpus=1, + out=str(directory/"out")).write(str(directory/"spec.json")) + node.write_json(directory/".fleet-lease.json", {"lease": "lease-1"}) + if cleaned: + (directory/".fleet-cleaned-lease-1").touch() + stops=executor.jobs_dir/".node"/"stops";stops.mkdir(parents=True,exist_ok=True) + node.write_json(stops/f"{ident}-lease-1.json", {"operation": 2}) + return directory + + +def test_deleting_a_batch_deletes_its_jobs_directories(executor,monkeypatch): + """Each listed job of the deleted batch loses its directory: record, logs and outputs. One of another + batch stays, and so does one whose container may still be there.""" + (executor.apps_file.parent/"batches.yaml").write_text("batches:\n render: {image: a}\n") + done=job_directory(executor,"job-done","render") + unsure=job_directory(executor,"job-unsure","render",cleaned=False) + other=job_directory(executor,"job-other","train") + monkeypatch.setattr(jobs,"_going",lambda _:False);monkeypatch.setattr(node,"container_gone",lambda _:False) + result,error=app_message(executor,"remove-1",{"remove":True,"delete":{"jobs":["job-done","job-unsure","job-other","job-gone","../etc"]}},batch="render") + assert error is None + assert result=={"removed":["render"],"jobs_deleted":1,"kept":[ + {"path":str(unsure),"reason":"its container may still be there"},{"path":str(other),"reason":"a job of batch train"}]} + assert not done.exists() and unsure.exists() and other.exists() + assert sorted(p.name for p in (executor.jobs_dir/".node"/"stops").iterdir())==["job-other-lease-1.json","job-unsure-lease-1.json"] def test_an_app_whose_cleanup_is_uncertain_stays_configured(executor,monkeypatch): diff --git a/crates/fleet-proto/proto/fleet/model/v1/model.proto b/crates/fleet-proto/proto/fleet/model/v1/model.proto index 5816de3..7fcbc80 100644 --- a/crates/fleet-proto/proto/fleet/model/v1/model.proto +++ b/crates/fleet-proto/proto/fleet/model/v1/model.proto @@ -168,6 +168,7 @@ message Job { required bool quota_held = 11; optional string stop = 12; required bool logs_ready = 13; + // Purged by an earlier release: hidden, and gone with its batch. required bool purged = 14; } diff --git a/crates/fleet-proto/proto/fleet/v1/types.proto b/crates/fleet-proto/proto/fleet/v1/types.proto index 00f8cd4..35baa29 100644 --- a/crates/fleet-proto/proto/fleet/v1/types.proto +++ b/crates/fleet-proto/proto/fleet/v1/types.proto @@ -55,6 +55,7 @@ enum NodeOperation { // Start or stop an app's replicas as the node's apps.yaml entry configures them // (runtime only; NODE_OPERATION_APP writes that configuration). NODE_OPERATION_SET_APP = 4; + // Sent by releases that purged a finished job; deleting its batch removes it now. NODE_OPERATION_PURGE_JOB = 5; reserved 6; reserved "NODE_OPERATION_STAGE_MODEL"; @@ -62,5 +63,9 @@ enum NodeOperation { // configuration (enabled state, resources, settings). Parameters: {"app": {namespace, // namespace_name, name}} or {"batch": {name}} with {"remove": true} or {"set": {...}}, or // {"entry": {name}} with {"set": {...}} for the one apps.yaml entry a change of an app stops. + // A deletion's removal, with {"delete": {...}}, also deletes what the node holds of it: a + // batch's jobs' directories ("jobs"); an app's replicas' logs, its workers' directories and + // its proof jobs' "files", but nothing that holds or lies in one of the paths to "keep". A + // removal without it, which takes an app off one node, keeps that. NODE_OPERATION_APP = 7; } diff --git a/crates/fleet-proto/src/workload.rs b/crates/fleet-proto/src/workload.rs index a2840b2..413720a 100644 --- a/crates/fleet-proto/src/workload.rs +++ b/crates/fleet-proto/src/workload.rs @@ -468,6 +468,18 @@ impl CommandRecord { pub fn removes_app(&self, app: &str) -> bool { self.removal().is_some_and(|(_, name)| name == app) } + /// The app or batch an app message removes, by name. + pub fn removes(&self) -> Option<&str> { + let p = &self.parameters; + (self.is(NodeOperation::App) && p["remove"] == json!(true)) + .then(|| p["app"]["name"].as_str().or(p["batch"]["name"].as_str())) + .flatten() + } + /// The app or batch an app message deletes, by name: it is removed with what the node + /// holds of it. A removal that only takes it off one node keeps that. + pub fn deletes(&self) -> Option<&str> { + self.removes().filter(|_| self.parameters.get("delete").is_some()) + } /// In its result: nothing of what it stopped is left running. pub fn cleaned(&self) -> bool { self.result["cleaned"].as_bool() == Some(true) @@ -503,7 +515,7 @@ pub mod command { json!({"id": job.id, "lease": job.lease, "namespace": job.namespace, "tag": job.tag, "owner": job.owner, "batch": job.batch, "policy": job.policy, "request": job.request, "submitted_at": job.status["submitted_at"]}) } - /// Cancels, preempts or purges a job's execution `lease`. + /// Cancels or preempts a job's execution `lease`. pub fn job(id: &str, lease: &str) -> Value { json!({"id": id, "lease": lease}) } diff --git a/crates/fleet-scheduler/src/fleet.rs b/crates/fleet-scheduler/src/fleet.rs index f533a3f..db27b12 100644 --- a/crates/fleet-scheduler/src/fleet.rs +++ b/crates/fleet-scheduler/src/fleet.rs @@ -754,11 +754,12 @@ impl Scheduler { }).collect()) } - /// Deletes an app, its terminal job records and their inputs and outputs, and its - /// anchor. Removes the ELF only when no other app or job uses it. Refused while any of - /// its jobs is queued or running. + /// Deletes an app with every job record of it, and its anchor. Refused while any of its + /// jobs is queued or running. Also returns the files its jobs left in its nodes' app + /// storage, for them to remove: each job's output directory, and the inputs and ELF no + /// other job or app uses. pub async fn delete_app(&self, namespace: &str, name: &str, - ) -> anyhow::Result { + ) -> anyhow::Result<(serde_json::Value, Vec)> { let _ops = self.app_ops.write().await; let apps = self.store.apps().await?; let app = apps @@ -782,12 +783,12 @@ impl Scheduler { app.active.max(live) ))); } - let ids = self.store.delete_app(namespace, name).await?; + let jobs = self.store.delete_app(namespace, name).await?; self.apps.lock().remove(&(namespace.to_string(), name.to_string())); { let mut inner = self.inner.lock(); - for id in &ids { - inner.jobs.remove(id); + for j in &jobs { + inner.jobs.remove(&j.id); } if let Some(node) = inner.anchors.remove(&(namespace.to_string(), name.to_string())) { if inner.reserved.get(&node) == Some(&placement::app_key(namespace, name)) { @@ -795,8 +796,23 @@ impl Scheduler { } } } - tracing::info!(app = %name, jobs = ids.len(), "app deleted"); - Ok(serde_json::json!({"app": name, "jobs_deleted": ids.len()})) + let elf_shared = apps.iter().any(|a| a.elf_id == app.elf_id && !(a.namespace == namespace && a.name == name)); + let inputs: Vec<&fleet_proto::fleet::v1::ProofInputs> = jobs.iter().filter_map(|j| j.inputs.as_ref()).collect(); + let mut shared: Vec = inputs.iter() + .flat_map(|i| [i.input.as_ref(), i.elf.as_ref().filter(|_| !elf_shared)]) + .flatten() + .map(|r| r.uri.clone()) + .collect(); + shared.sort(); + shared.dedup(); + let used = self.store.referenced(&shared).await?; + let files = jobs.iter() + .filter_map(|j| Some(format!("{}/{}", j.inputs.as_ref()?.output_root.trim_end_matches('/'), j.id))) + .chain(shared.into_iter().filter(|uri| !used.contains(uri))) + .filter_map(|uri| uri.strip_prefix("file://").map(str::to_owned)) + .collect(); + tracing::info!(app = %name, jobs = jobs.len(), "app deleted"); + Ok((serde_json::json!({"app": name, "jobs_deleted": jobs.len()}), files)) } /// The newest jobs (the store has every one) that the filter selects. diff --git a/crates/fleet-scheduler/src/http.rs b/crates/fleet-scheduler/src/http.rs index a37de79..c049745 100644 --- a/crates/fleet-scheduler/src/http.rs +++ b/crates/fleet-scheduler/src/http.rs @@ -55,7 +55,7 @@ pub fn router(state: HttpState) -> Router { /// The permission an API route needs. Reading apps, batches, their jobs and the caller's own /// namespace needs read; submitting a job (or cancelling a batch job) submit; changing an app -/// or batch, or purging its jobs, manage; everything else (namespaces and their quotas, +/// or batch manage; everything else (namespaces and their quotas, /// tokens, allocations, preemption, nodes, the fleet's status) admin. pub fn required(method: &Method, path: &str) -> Permission { let parts: Vec<&str> = path.trim_matches('/').split('/').collect(); diff --git a/crates/fleet-scheduler/src/store.rs b/crates/fleet-scheduler/src/store.rs index 5fff6a1..2aaa4c2 100644 --- a/crates/fleet-scheduler/src/store.rs +++ b/crates/fleet-scheduler/src/store.rs @@ -1,6 +1,6 @@ //! ProofJob metadata. A proof job references its inputs and outputs in app-owned storage. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use async_trait::async_trait; use parking_lot::Mutex; @@ -39,8 +39,10 @@ pub trait Store: Send + Sync { async fn recent(&self, f: &ProofJobFilter, limit: usize) -> anyhow::Result>; /// Every registered app with its job counts, most recently used first. async fn apps(&self) -> anyhow::Result>; - /// Deletes an app and its finished jobs; returns the jobs' ids. - async fn delete_app(&self, namespace: &str, name: &str) -> anyhow::Result>; + /// Deletes an app and its finished jobs; returns the jobs. + async fn delete_app(&self, namespace: &str, name: &str) -> anyhow::Result>; + /// Which of `uris` a job's input or ELF still references. + async fn referenced(&self, uris: &[String]) -> anyhow::Result>; /// Registers an app; false when its name or program is one of the namespace's already. async fn add_app(&self, app: &App) -> anyhow::Result; async fn set_app_priority(&self, namespace: &str, name: &str, priority: i32) -> anyhow::Result<()>; @@ -153,18 +155,26 @@ impl Store for MemoryStore { Ok(v) } - async fn delete_app(&self, namespace: &str, name: &str) -> anyhow::Result> { + async fn delete_app(&self, namespace: &str, name: &str) -> anyhow::Result> { let mut jobs = self.jobs.lock(); let ids: Vec = jobs .values() .filter(|j| j.namespace == namespace && j.app == name && j.state().is_terminal()) .map(|j| j.id.clone()) .collect(); - for id in &ids { - jobs.remove(id); - } + let gone = ids.iter().filter_map(|id| jobs.remove(id)).collect(); self.apps.lock().remove(&(namespace.to_string(), name.to_string())); - Ok(ids) + Ok(gone) + } + + async fn referenced(&self, uris: &[String]) -> anyhow::Result> { + Ok(self.jobs.lock().values() + .filter_map(|j| j.inputs.as_ref()) + .flat_map(|i| [i.input.as_ref(), i.elf.as_ref()]) + .flatten() + .filter(|r| uris.contains(&r.uri)) + .map(|r| r.uri.clone()) + .collect()) } async fn add_app(&self, app: &App) -> anyhow::Result { @@ -659,17 +669,29 @@ impl Store for PgStore { .collect() } - async fn delete_app(&self, namespace: &str, name: &str) -> anyhow::Result> { + async fn delete_app(&self, namespace: &str, name: &str) -> anyhow::Result> { let rows = sqlx::query( - "WITH gone AS (DELETE FROM fleet_jobs WHERE namespace = $1 AND app = $2 AND state IN ('succeeded', 'failed') RETURNING id), \ + "WITH gone AS (DELETE FROM fleet_jobs WHERE namespace = $1 AND app = $2 AND state IN ('succeeded', 'failed') RETURNING *), \ app AS (DELETE FROM fleet_apps WHERE namespace = $1 AND name = $2) \ - SELECT id FROM gone", + SELECT * FROM gone", ) .bind(namespace) .bind(name) .fetch_all(&self.pool) .await?; - rows.iter().map(|r| Ok(r.try_get("id")?)).collect() + rows.iter().map(Self::row_to_job).collect() + } + + async fn referenced(&self, uris: &[String]) -> anyhow::Result> { + let rows: Vec = sqlx::query_scalar( + "SELECT DISTINCT uri FROM fleet_jobs, \ + LATERAL (VALUES (inputs->'input'->>'uri'), (inputs->'elf'->>'uri')) AS r(uri) \ + WHERE uri = ANY($1)", + ) + .bind(uris) + .fetch_all(&self.pool) + .await?; + Ok(rows.into_iter().collect()) } async fn add_app(&self, a: &App) -> anyhow::Result { diff --git a/crates/fleet-scheduler/src/token.rs b/crates/fleet-scheduler/src/token.rs index cc4237d..9381ef5 100644 --- a/crates/fleet-scheduler/src/token.rs +++ b/crates/fleet-scheduler/src/token.rs @@ -15,7 +15,7 @@ pub enum Permission { Read, /// Submit proof and batch jobs; cancel a batch job. Submit, - /// Register, delete, update, scale, enable and disable apps and batches; purge jobs. + /// Register, delete, update, scale, enable and disable apps and batches. Manage, /// Namespaces, quotas, tokens, allocations, preemption and nodes, in any namespace. Admin, diff --git a/crates/fleet-scheduler/src/workloads.rs b/crates/fleet-scheduler/src/workloads.rs index 5f90869..8930fe1 100644 --- a/crates/fleet-scheduler/src/workloads.rs +++ b/crates/fleet-scheduler/src/workloads.rs @@ -48,6 +48,41 @@ fn fail(code: u16, message: impl Into) -> anyhow::Error { message: message.into(), }) } +/// The files of the job records and commands a deletion dropped. +type RecordFiles = (Vec, Vec); + +/// Whether command `c` changes app or batch `name` of `namespace`, or one of its entries. +fn about(c: &CommandRecord, namespace: &str, name: &str) -> bool { + let p = &c.parameters; + if c.is(NodeOperation::SetApp) { + return c.entry().app() == name && c.entry().namespace() == namespace; + } + c.is(NodeOperation::App) + && (p["batch"]["name"] == name + || p["app"]["name"] == name && p["app"]["namespace"] == namespace + || c.created().is_some_and(|e| e.app() == name && e.namespace() == namespace)) +} + +/// An outcome of removing a file, gone already counting as done. +fn gone(result: std::io::Result<()>) -> std::io::Result<()> { + match result { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(e), + _ => Ok(()), + } +} + +/// Removes deleted job records with their logs, and deleted commands. +fn remove_records(dir: &Path, jobs: &[String], commands: &[String]) -> anyhow::Result<()> { + for id in jobs { + gone(fs::remove_file(dir.join("jobs").join(format!("{id}.json"))))?; + gone(fs::remove_dir_all(dir.join("artifacts").join(id)))?; + } + for id in commands { + gone(fs::remove_file(dir.join("commands").join(format!("{id}.json"))))?; + } + Ok(()) +} + fn response_error(e: anyhow::Error) -> Response { if e.downcast_ref::().is_some() { return (StatusCode::BAD_REQUEST, "invalid JSON request").into_response(); @@ -146,14 +181,16 @@ impl WorkloadController { .insert(format!("{}-{}", v.worker, v.entry().name()), v); } fleet.set_worker_owned_apps(worker_owned(&inner)); - Ok(Self { + let controller = Self { fleet, cluster, dir, inner: Mutex::new(inner), ops: tokio::sync::Mutex::new(()), notify: tokio::sync::Notify::new(), - }) + }; + controller.forget_deleted()?; + Ok(controller) } pub(crate) fn fleet(&self) -> &Arc { &self.fleet @@ -244,11 +281,158 @@ impl WorkloadController { workers.into_iter().map(|worker| self.command(worker, NodeOperation::App, message.clone())).collect() } - /// Tells every node that `app` was deleted from `namespace`; each removes the entries it - /// configures for it. - pub(crate) fn remove_app(&self, namespace: &str, app: &str) -> anyhow::Result<()> { - let workers: Vec = self.inner.lock().workers.keys().cloned().collect(); - self.tell_nodes(&workers, self.app_message(namespace, app), message::remove())?; + /// Tells every node that `app` was deleted from `namespace`. Each removes the entries it + /// configures for it with what they hold on it, and `files`, which the app's proof jobs + /// left in its storage, but keeps what the nodes' other apps and batches use. Returns the + /// nodes told. + pub(crate) fn remove_app(&self, namespace: &str, app: &str, files: &[String]) -> anyhow::Result> { + // Its nodes, and every other connected since the scheduler started: one that is offline + // gets the message when it is back. + let workers: Vec = { + let i = self.inner.lock(); + let mine = i.apps.get(app).filter(|a| a.namespace == namespace).map(|a| a.workers.clone()).unwrap_or_default(); + i.workers.keys().cloned().chain(mine).collect::>().into_iter().collect() + }; + let change = json!({"remove": true, "delete": {"keep": self.paths_in_use(app), "files": files}}); + self.tell_nodes(&workers, self.app_message(namespace, app), change)?; + Ok(workers) + } + + /// The host paths that the apps and batches its nodes report use, but `name`'s. Deleting + /// `name` keeps them on every node, and what holds or lies in one of them. + fn paths_in_use(&self, name: &str) -> Vec { + let i = self.inner.lock(); + let mut paths = std::collections::BTreeSet::new(); + for w in i.workers.values() { + for e in w.report.entries().filter(|e| e.app() != name) { + for key in ["workdir", "worker_workdir", "proof_storage_root", "worker_env_file"] { + paths.extend(e.0[key].as_str().map(str::to_owned)); + } + for volume in e.0["volumes"].as_array().into_iter().flatten().filter_map(Value::as_str) { + paths.insert(volume.split(':').next().unwrap_or(volume).to_owned()); + } + } + for (_, spec) in w.report.batches.iter().filter(|(batch, _)| batch.as_str() != name) { + paths.extend(spec["allow"].as_array().into_iter().flatten().filter_map(Value::as_str).map(str::to_owned)); + } + } + paths.into_iter().collect() + } + + /// Deletes app or batch `name` of `namespace`, once nothing of it runs. What the control + /// host holds of it goes at once: its job records with their logs, and the commands about + /// them and it. Its own record stays, marked deleted, until its nodes have removed it + /// (`forget_deleted`), so that a node report sent before then can't bring it back. Each + /// node removes its configuration and what it holds of it: a batch's job directories; an + /// app's replicas, their logs, its workers' directories and `files`, which its proof jobs + /// left in its storage. Returns what it deleted and the nodes it told, and the files of + /// the records, to remove once the operation lock is released. + fn delete(&self, namespace: &str, name: &str, files: Vec) -> anyhow::Result<(Value, RecordFiles)> { + let template = self.inner.lock().apps.get(name).filter(|a| a.namespace == namespace).cloned(); + let batch = template.as_ref().is_some_and(|a| !a.persistent); + if let Some(mut t) = template.clone() { + t.removed = true; + self.save("apps", name, &t)?; + self.inner.lock().apps.insert(name.into(), t); + } + // Its deployments go once their nodes report them no more, which needs them disabled. + let enabled: Vec<(String, AppDeployment)> = self.inner.lock().deployments.iter() + .filter(|(_, d)| !d.disabled && d.entry().app() == name && d.entry().namespace() == namespace) + .map(|(key, d)| (key.clone(), d.clone())) + .collect(); + for (key, mut d) in enabled { + d.disabled = true; + self.save("deployments", &key, &d)?; + self.inner.lock().deployments.insert(key, d); + } + let (jobs, commands, shown, on) = { + let mut i = self.inner.lock(); + let jobs: Vec = i.jobs.values().filter(|r| r.batch == name).map(|r| r.id.clone()).collect(); + let ids: HashSet<&str> = jobs.iter().map(String::as_str).collect(); + let commands: Vec = i.commands.values() + .filter(|c| c.job_id().is_some_and(|id| ids.contains(id)) || c.done && about(c, namespace, name)) + .map(|c| c.id.clone()) + .collect(); + // Where each job's directory is; an earlier release's purge removed it already. + let (mut shown, mut on) = (0, BTreeMap::>::new()); + for id in &jobs { + let r = i.jobs.remove(id).expect("listed above"); + if !r.purged { + shown += 1; + if let Some(worker) = r.worker { + on.entry(worker).or_default().push(r.id); + } + } + } + for id in &commands { + i.commands.remove(id); + } + (jobs, commands, shown, on) + }; + let nodes = if batch { + let mut workers: std::collections::BTreeSet = template.iter().flat_map(|t| t.workers.clone()).collect(); + workers.extend(on.keys().cloned()); + for worker in &workers { + let jobs = on.get(worker).cloned().unwrap_or_default(); + let change = json!({"remove": true, "delete": {"jobs": jobs}}); + self.command(worker, NodeOperation::App, message::with(message::batch(name), &change))?; + } + workers.into_iter().collect() + } else { + self.remove_app(namespace, name, &files)? + }; + tracing::info!(name, jobs = shown, ?nodes, "deleted; its nodes remove what they hold of it"); + Ok((json!({"name": name, "removed": true, "jobs_deleted": shown, "nodes": nodes}), (jobs, commands))) + } + + /// Removes the files of records `delete` dropped. + async fn remove_files(&self, (jobs, commands): RecordFiles) -> anyhow::Result<()> { + let dir = self.dir.clone(); + tokio::task::spawn_blocking(move || remove_records(&dir, &jobs, &commands)).await? + } + + /// Forgets the deleted apps and batches that their nodes have removed: the record of each, + /// and the commands about it. One that a node failed to remove stays until it is deleted + /// again, and so does a batch deleted by an earlier release, while the job records that + /// release left are there (it goes with the ones that release purged). + fn forget_deleted(&self) -> anyhow::Result<()> { + let (names, jobs, commands) = { + let mut i = self.inner.lock(); + let deleted: Vec<(String, String)> = i.apps.values().filter(|a| a.removed) + .map(|a| (a.namespace.clone(), a.name.clone())).collect(); + if deleted.is_empty() { + return Ok(()); + } + let shown: HashSet<&str> = i.jobs.values().filter(|r| !r.purged).map(|r| r.batch.as_str()).collect(); + let gone: Vec<(String, String)> = deleted.into_iter() + .filter(|(namespace, name)| !shown.contains(name.as_str()) + && !i.commands.values().any(|c| about(c, namespace, name) && (!c.done || c.deletes().is_some() && c.error.is_some()))) + .collect(); + let names: HashSet<&str> = gone.iter().map(|(_, name)| name.as_str()).collect(); + let jobs: Vec = i.jobs.values().filter(|r| names.contains(r.batch.as_str())).map(|r| r.id.clone()).collect(); + let ids: HashSet<&str> = jobs.iter().map(String::as_str).collect(); + let commands: Vec = i.commands.values() + .filter(|c| c.job_id().is_some_and(|id| ids.contains(id)) || gone.iter().any(|(namespace, name)| about(c, namespace, name))) + .map(|c| c.id.clone()) + .collect(); + for id in &jobs { + i.jobs.remove(id); + } + for id in &commands { + i.commands.remove(id); + } + for (_, name) in &gone { + i.apps.remove(name); + } + (gone.into_iter().map(|(_, name)| name).collect::>(), jobs, commands) + }; + for name in &names { + gone(fs::remove_file(self.dir.join("apps").join(format!("{name}.json"))))?; + } + remove_records(&self.dir, &jobs, &commands)?; + if !names.is_empty() { + tracing::info!(?names, "deleted apps and batches forgotten: their nodes removed them"); + } Ok(()) } @@ -515,7 +699,8 @@ impl WorkloadController { if !app.persistent { return running; } - if !app.disabled() { + // A deleted app's nodes stop it as they remove it. + if !app.disabled() && !app.removed { running.push(format!("it is enabled: disable it first (fleetctl app disable {name})")); return running; } @@ -531,6 +716,15 @@ impl WorkloadController { running } + /// Refuses `name` while a deleted app or batch of that name is not gone yet. + pub(super) fn ensure_name_free(&self, name: &str) -> anyhow::Result<()> { + if self.inner.lock().apps.get(name).is_some_and(|a| a.removed) { + return Err(fail(409, format!("{name} is deleted, but not gone yet: its nodes are still removing it, \ + or it keeps job records from an earlier release. Delete it again to finish, then reuse the name"))); + } + Ok(()) + } + /// Whether a batch still has job records (not purged). fn has_jobs(&self, batch: &str) -> bool { self.inner.lock().jobs.values().any(|r| r.batch == batch && !r.purged) @@ -800,6 +994,11 @@ impl WorkloadController { }; let key = format!("{worker_id}-{name}"); let existing = { self.inner.lock().deployments.get(&key).cloned() }; + // An entry of a deleted app, which its node is about to remove. + if existing.is_none() && self.inner.lock().apps.get(observed.app()) + .is_some_and(|a| a.removed && a.namespace == namespace.id) { + continue; + } let intent = if let Some(mut intent) = existing { // The app an entry runs, its program and its launch settings are the node's own // configuration, not resource-scale overrides. @@ -976,7 +1175,7 @@ impl WorkloadController { self.save("jobs", &r.id, &r)?; self.inner.lock().jobs.insert(r.id.clone(), r); } - let mut acknowledged = vec![]; + let (mut acknowledged, mut removals) = (vec![], false); for result in &report.results { let Some(mut c) = self.inner.lock().commands.get(&result.id).cloned() else { acknowledged.push(result.id.clone()); @@ -1067,11 +1266,24 @@ impl WorkloadController { } } } + if let Some(name) = c.deletes() { + removals = true; + let kept = c.result["kept"].as_array().filter(|k| !k.is_empty()); + if c.error.is_some() || kept.is_some() { + tracing::warn!(worker = worker_id, name, error = ?c.error, ?kept, + "a node did not remove all of a deleted app or batch; delete it again to retry"); + } else { + tracing::info!(worker = worker_id, name, deleted = %c.result["deleted"], "a node removed a deleted app or batch"); + } + } self.save("commands", &c.id, &c)?; self.inner.lock().commands.insert(c.id.clone(), c); } acknowledged.push(result.id.clone()); } + if removals { + self.forget_deleted()?; + } self.notify.notify_waiters(); Ok(acknowledged) } @@ -1112,26 +1324,6 @@ impl WorkloadController { (w.report.free_cpus - pending.1 - starting.1).max(0.0), )) } - fn purge(&self, mut r: Job) -> anyhow::Result { - if !terminal(r.state()) || r.quota_held { - return Err(fail(409, "wait for confirmed cleanup before purging")); - } - r.purged = true; - self.save("jobs", &r.id, &r)?; - self.inner.lock().jobs.insert(r.id.clone(), r.clone()); - if let Some(worker) = &r.worker { - self.command( - worker, - NodeOperation::PurgeJob, - message::job(&r.id, &r.lease), - )?; - } - let path = self.dir.join("artifacts").join(&r.id); - if path.exists() { - fs::remove_dir_all(path)?; - } - Ok(json!({"id":r.id,"purged":true})) - } async fn stop_job(&self, mut r: Job, preempt: bool) -> anyhow::Result { if terminal(r.state()) && (preempt || !r.quota_held) { return Err(fail(409, "job already finished")); @@ -1579,27 +1771,11 @@ impl WorkloadController { } } -/// The commands one heartbeat hands `worker`: at most 16, purges after everything else. A -/// purge only drops a finished job's node record, so thousands of them, as a bulk purge of -/// history queues, must not hold back the launches, stops and app changes behind them. +/// The commands one heartbeat hands `worker`: at most 16. fn heartbeat_commands(i: &WorkloadState, worker: &str) -> Vec { - const LIMIT: usize = 16; - let (mut work, mut purges) = (vec![], vec![]); - for c in i.commands.open().filter(|c| c.worker == worker && !c.done && c.admitted) { - if c.is(NodeOperation::PurgeJob) { - if purges.len() < LIMIT { - purges.push(c); - } - } else { - work.push(c); - if work.len() == LIMIT { - break; - } - } - } - work.into_iter() - .chain(purges) - .take(LIMIT) + i.commands.open() + .filter(|c| c.worker == worker && !c.done && c.admitted) + .take(16) .map(|c| NodeCommand { id: c.id.clone(), operation: c.operation, parameters: Some(c.parameters.clone()) }) .collect() } @@ -1954,7 +2130,8 @@ async fn handle_result(s: Arc, request: Request) -> anyhow:: body["namespace"] = json!(namespace); body.as_object_mut().unwrap().remove("name"); let _ops = s.ops.lock().await; - if s.inner.lock().apps.get(&name).is_some_and(|a| !a.removed) || s.fleet.app_known(&namespace, &name) { + s.ensure_name_free(&name)?; + if s.inner.lock().apps.contains_key(&name) || s.fleet.app_known(&namespace, &name) { return Err(fail(409, format!("an app or batch named {name} exists"))); } let app = WorkloadTemplate { @@ -1973,8 +2150,10 @@ async fn handle_result(s: Arc, request: Request) -> anyhow:: } if parts.first() == Some(&"apps") && parts.len() >= 2 { let name = parts[1]; - // A deleted batch stays readable while it has job records: they are its history. - let readable_deleted = batch && method == Method::GET && parts.len() == 2 && s.has_jobs(name); + // A batch an earlier release deleted stays readable while it has job records; deleting + // a deleted app or batch again removes those and retries its nodes' removal. + let readable_deleted = batch && method == Method::GET && parts.len() == 2 && s.has_jobs(name) + || method == Method::DELETE && parts.len() == 2; let mut app = s .inner .lock() @@ -1993,22 +2172,17 @@ async fn handle_result(s: Arc, request: Request) -> anyhow:: return Ok(Json(view(&s, &app)).into_response()); } if method == Method::DELETE && parts.len() == 2 { - let _ops = s.ops.lock().await; + let ops = s.ops.lock().await; s.ensure_not_scaling(&app.namespace, name)?; let running = s.still_running(&app); if !running.is_empty() { let kind = if batch { "batch" } else { "app" }; return Err(fail(409, format!("{kind} {name} still runs: {}; delete it once nothing of it does", running.join("; ")))); } - app.removed = true; - s.save("apps", name, &app)?; - s.inner.lock().apps.insert(name.into(), app.clone()); - if app.persistent { - s.remove_app(&app.namespace, name)?; - } else { - s.tell_nodes(&app.workers, json!({"batch":{"name":name}}), json!({"remove":true}))?; - } - return Ok(Json(json!({"name":name,"removed":true})).into_response()); + let (deleted, files) = s.delete(&app.namespace, name, vec![])?; + drop(ops); + s.remove_files(files).await?; + return Ok(Json(deleted).into_response()); } if batch && parts.len() == 3 && parts[2] == "settings" && method == Method::PUT { let set: Value = serde_json::from_slice(&to_bytes(request.into_body(), 1 << 20).await?)?; @@ -2115,37 +2289,6 @@ async fn handle_result(s: Arc, request: Request) -> anyhow:: ) .into_response()); } - if method == Method::DELETE && parts.len() == 3 { - let days = q - .get("older_than") - .and_then(|s| s.parse::().ok()) - .filter(|v| v.is_finite() && *v >= 0.0) - .ok_or_else(|| fail(400, "older_than must be non-negative days"))?; - let cutoff = now() - days * 86400.0; - let purgeable = |r: &Job| { - !r.purged - && r.batch == batch - && terminal(r.state()) - && !r.quota_held - && r.finished_at().is_some_and(|v| v < cutoff) - && s.visible(&r.namespace, r.tag.as_deref(), c, &q) - }; - let ids: Vec = s.inner.lock().jobs.values().filter(|r| purgeable(r)).map(|r| r.id.clone()).collect(); - // A purge writes a record and a node command per job, so a bulk one of history can - // take minutes. Holding the operation lock for a chunk at a time lets node reports - // and dispatch through; each job is checked again under the lock. - let mut purged = Vec::with_capacity(ids.len()); - for chunk in ids.chunks(64) { - let _ops = s.ops.lock().await; - for id in chunk { - let r = s.inner.lock().jobs.get(id).filter(|r| purgeable(r)).cloned(); - if let Some(r) = r { - purged.push(s.purge(r)?); - } - } - } - return Ok(Json(purged).into_response()); - } if method == Method::POST && parts.len() == 3 { let body = serde_json::from_slice(&to_bytes(request.into_body(), 1 << 20).await?)?; return Ok((StatusCode::CREATED, Json(s.submit(batch, body, c, &q).await?)).into_response()); @@ -2168,11 +2311,6 @@ async fn handle_result(s: Arc, request: Request) -> anyhow:: let r = job(&s)?; return Ok(Json(s.stop_job(r, preempt).await?.value()).into_response()); } - if method == Method::DELETE && parts.len() == 4 { - let _ops = s.ops.lock().await; - let r = job(&s)?; - return Ok(Json(vec![s.purge(r)?]).into_response()); - } if method == Method::GET && parts.len() == 5 && parts[4] == "logs" { return log_response(&s, &r, &q).await; } @@ -2288,15 +2426,12 @@ impl WorkloadController { match (method, rest) { (&Method::GET, []) => Ok(Json(self.stored_proof_app(&namespace, &name, tag.as_deref()).await?).into_response()), (&Method::DELETE, []) => { - let _guard = self.app_lifecycle_guard(&namespace, &name).await?; - let deleted = self.fleet.delete_app(&namespace, &name).await?; - let template = self.inner.lock().apps.get(&name).filter(|a| a.namespace == namespace).cloned(); - if let Some(mut template) = template { - template.removed = true; - self.save("apps", &name, &template)?; - self.inner.lock().apps.insert(name.clone(), template); - } - self.remove_app(&namespace, &name)?; + let guard = self.app_lifecycle_guard(&namespace, &name).await?; + let (mut deleted, files) = self.fleet.delete_app(&namespace, &name).await?; + let (removed, records) = self.delete(&namespace, &name, files)?; + drop(guard); + self.remove_files(records).await?; + deleted["nodes"] = removed["nodes"].clone(); Ok(Json(deleted).into_response()) } (&Method::PUT, ["priority"]) => { diff --git a/crates/fleet-scheduler/src/workloads_records.rs b/crates/fleet-scheduler/src/workloads_records.rs index 3cfc45c..d71c911 100644 --- a/crates/fleet-scheduler/src/workloads_records.rs +++ b/crates/fleet-scheduler/src/workloads_records.rs @@ -72,6 +72,12 @@ impl Records { self.all.get(id) } + pub(super) fn remove(&mut self, id: &str) -> Option { + let old = self.all.remove(id)?; + unindex(&mut self.open, &mut self.about, id, &old); + Some(old) + } + /// Every record, history included, in id order. pub(super) fn values(&self) -> btree_map::Values<'_, String, T> { self.all.values() diff --git a/crates/fleet-scheduler/src/workloads_scale.rs b/crates/fleet-scheduler/src/workloads_scale.rs index e9c4e11..a367286 100644 --- a/crates/fleet-scheduler/src/workloads_scale.rs +++ b/crates/fleet-scheduler/src/workloads_scale.rs @@ -764,9 +764,10 @@ impl WorkloadController { WorkloadController::require_grant(c, &name)?; WorkloadController::require_admin_settings(c, &spec, &Self::ADMIN_CREATE_SETTINGS)?; let _ops = self.ops.lock().await; + self.ensure_name_free(&name)?; { let i = self.inner.lock(); - if i.apps.get(&name).is_some_and(|a| !a.removed) + if i.apps.contains_key(&name) || i.deployments.values().any(|d| d.entry().name() == name) || self.fleet.app_known(&namespace, &name) { diff --git a/crates/fleet-scheduler/src/workloads_tests.rs b/crates/fleet-scheduler/src/workloads_tests.rs index 97fa90e..22d0b4c 100644 --- a/crates/fleet-scheduler/src/workloads_tests.rs +++ b/crates/fleet-scheduler/src/workloads_tests.rs @@ -320,7 +320,7 @@ async fn batch_policy_and_job_lifecycle_use_batch_routes() { assert_eq!(scale_call(c.clone(), "admin", "DELETE", "/batches/training", Value::Null) .await.status(), StatusCode::OK); assert_eq!(scale_call(c.clone(), "admin", "GET", &format!("/batches/training/jobs/{}", text(&job,"id")), Value::Null) - .await.status(), StatusCode::OK); + .await.status(), StatusCode::NOT_FOUND, "its jobs go with it"); } #[tokio::test] @@ -465,30 +465,6 @@ async fn a_job_its_quota_cannot_take_waits_without_rewriting_its_record() { assert_eq!(fs::metadata(&path).unwrap().ino(), written, "a pass that cannot place it leaves its record alone"); assert_eq!(placed(&c), 2); } -#[tokio::test] -async fn a_backlog_of_purges_does_not_hold_back_a_launch() { - let (_dir, c, ns) = setup().await; - c.report("gpu", "session", inventory(&ns)).await.unwrap(); - // A bulk purge of history: more purges than one heartbeat carries, all ahead in id order. - for n in 0..20 { - let id = format!("0000-{n:02}"); - c.inner.lock().commands.insert(id.clone(), CommandRecord { - id, worker: "gpu".into(), operation: NodeOperation::PurgeJob as i32, - parameters: json!({"id": format!("old-{n}"), "lease": "lease"}).into(), - done: false, result: Value::Null.into(), error: None, admitted: true, - }); - } - let job = submit(&c).await; - c.dispatch().await.unwrap(); - let launch = c.inner.lock().commands.about(&job.id).next().unwrap().id.clone(); - - let sent = heartbeat_commands(&c.inner.lock(), "gpu"); - assert_eq!(sent.len(), 16); - assert_eq!(sent[0].id, launch, "the launch goes first"); - assert!(sent[1..].iter().all(|c| c.operation == NodeOperation::PurgeJob as i32)); - assert_eq!(sent[1].id, "0000-00", "purges keep their order"); - assert!(heartbeat_commands(&c.inner.lock(), "other").is_empty()); -} /// Everything the tests log, from a process-wide subscriber: a scoped one can miss events /// while tests on other threads run the same log lines. fn test_logs() -> &'static parking_lot::Mutex> { @@ -512,76 +488,22 @@ fn test_logs() -> &'static parking_lot::Mutex> { } #[tokio::test] -async fn a_bulk_purge_takes_every_finished_job_in_chunks_and_records_who_asked() { +async fn a_change_is_on_record_with_who_asked_and_a_read_is_not() { let logs = test_logs(); let (_dir, c, ns) = setup().await; c.report("gpu", "session", inventory(&ns)).await.unwrap(); - let template = submit(&c).await; - // More finished jobs than one chunk holds, and one that finished just now. - for n in 0..150 { - let mut r = template.clone(); - r.id = format!("old-{n:03}"); - r.worker = Some("gpu".into()); - r.status = json!({"state":"succeeded","submitted_at":1,"finished_at":(now() - 7200.0)}).into(); - c.inner.lock().jobs.insert(r.id.clone(), r); - } - let mut recent = template.clone(); - recent.id = "recent".into(); - recent.status = json!({"state":"succeeded","submitted_at":1,"finished_at":now()}).into(); - c.inner.lock().jobs.insert(recent.id.clone(), recent); - - let response = scale_call(c.clone(), "admin", "DELETE", "/batches/training/jobs?older_than=0.0517", json!(null)).await; - assert_eq!(response.status(), StatusCode::OK); - let body: Value = serde_json::from_slice(&to_bytes(response.into_body(), 1 << 20).await.unwrap()).unwrap(); - assert_eq!(body.as_array().unwrap().len(), 150); - { - let i = c.inner.lock(); - assert!((0..150).all(|n| i.jobs[&format!("old-{n:03}")].purged)); - assert!(!i.jobs["recent"].purged && !i.jobs[&template.id].purged); - let purges: HashSet<&str> = i.commands.open() - .filter(|c| c.worker == "gpu" && c.operation == NodeOperation::PurgeJob as i32) - .filter_map(|c| c.parameters["id"].as_str()) - .collect(); - assert_eq!(purges.len(), 150, "each purged job's node drops its copy"); - } - - // The request is on record with its caller; a read is not. - scale_call(c.clone(), "admin", "GET", "/batches/training/jobs", json!(null)).await; + let uri = format!("/batches/training?namespace={ns}"); + assert_eq!(scale_call(c.clone(), "admin", "DELETE", &uri, json!(null)).await.status(), StatusCode::OK); + scale_call(c.clone(), "admin", "GET", "/batches", json!(null)).await; let logs = String::from_utf8(logs.lock().clone()).unwrap(); - let lines: Vec<&str> = logs.lines().filter(|l| l.contains("api request") && l.contains("older_than=0.0517")).collect(); + let lines: Vec<&str> = logs.lines().filter(|l| l.contains("api request") && l.contains(&format!("uri={uri}"))).collect(); assert_eq!(lines.len(), 1, "{logs}"); assert!(!logs.lines().any(|l| l.contains("api request") && l.contains("method=GET")), "reads are not logged"); - for part in ["method=DELETE", "uri=/batches/training/jobs?older_than=0.0517", "status=200", &format!("token=\"{}\"", "ad".repeat(16)), "label=\"admin\""] { + for part in ["method=DELETE", "status=200", &format!("token=\"{}\"", "ad".repeat(16)), "label=\"admin\""] { assert!(lines[0].contains(part), "{part} in {}", lines[0]); } assert!(!lines[0].contains(ADMIN), "the token itself is never logged"); } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn a_bulk_purge_lets_node_reports_in_between_its_chunks() { - let (_dir, c, ns) = setup().await; - c.report("gpu", "session", inventory(&ns)).await.unwrap(); - let template = submit(&c).await; - for n in 0..400 { - let mut r = template.clone(); - r.id = format!("old-{n:03}"); - r.worker = Some("gpu".into()); - r.status = json!({"state":"succeeded","submitted_at":1,"finished_at":1}).into(); - c.inner.lock().jobs.insert(r.id.clone(), r); - } - let purged = |c: &WorkloadController| c.inner.lock().jobs.values().filter(|r| r.purged).count(); - let purge = tokio::spawn(scale_call(c.clone(), "admin", "DELETE", "/batches/training/jobs?older_than=0", json!(null))); - while purged(&c) == 0 { - tokio::time::sleep(Duration::from_millis(1)).await; - } - // What a node report waits for: it gets the lock before the purge has finished. - let during = { - let _ops = c.ops.lock().await; - purged(&c) - }; - assert_eq!(purge.await.unwrap().status(), StatusCode::OK); - assert!(during < 400, "the report waited for the whole purge"); - assert_eq!(purged(&c), 400); -} #[tokio::test] async fn offline_worker_and_terminal_without_cleanup_keep_reservations() { let (_dir, c, ns) = setup().await; @@ -608,7 +530,7 @@ async fn offline_worker_and_terminal_without_cleanup_keep_reservations() { assert!(c.fleet.allocations(&ns).is_empty()); } #[tokio::test] -async fn stale_lease_is_ignored_and_purged_history_cannot_be_imported_again() { +async fn stale_lease_is_ignored_and_a_purged_record_is_not_imported_again() { let (_dir, c, ns) = setup().await; c.report("gpu", "session", inventory(&ns)).await.unwrap(); let job = submit(&c).await; @@ -622,8 +544,8 @@ async fn stale_lease_is_ignored_and_purged_history_cannot_be_imported_again() { assert_eq!(c.fleet.allocations(&ns).len(), 1); report.jobs[0] = incoming(&job, "succeeded", true); c.report("gpu", "session", report.clone()).await.unwrap(); - c.purge(c.job(&job.id, None, &BTreeMap::new()).unwrap()) - .unwrap(); + // A record an earlier release purged: its node's copy may be reported still. + c.inner.lock().jobs.update(&job.id, |r| r.purged = true); c.report("gpu", "session", report).await.unwrap(); assert!(c.job(&job.id, None, &BTreeMap::new()).is_err()); } @@ -2052,9 +1974,9 @@ async fn deleting_a_registered_app_tells_every_node_which_app_was_deleted() { c.report("gpu","session",inventory(&ns)).await.unwrap(); c.report("cpu","session",inventory(&ns)).await.unwrap(); let app=PROOF.to_string(); - c.remove_app(&ns,&app).unwrap(); - c.remove_app(&ns,&app).unwrap(); - let expected=json!({"app":{"namespace":ns,"namespace_name":"test","name":app},"remove":true}); + c.remove_app(&ns,&app,&[]).unwrap(); + c.remove_app(&ns,&app,&[]).unwrap(); + let expected=json!({"app":{"namespace":ns,"namespace_name":"test","name":app},"remove":true,"delete":{"keep":[],"files":[]}}); assert_eq!(remove_commands(&c,NodeOperation::App),vec![("cpu".to_string(),expected.clone()),("gpu".to_string(),expected)]); } @@ -2064,11 +1986,20 @@ async fn deleting_an_app_tells_its_nodes() { c.report("gpu","session",scale_inventory(&ns,false,true)).await.unwrap(); c.inner.lock().apps.get_mut("model").unwrap().spec["disabled"]=json!(true); assert_eq!(scale_call(c.clone(),"admin","DELETE",&format!("/apps/model?namespace={ns}"),Value::Null).await.status(),StatusCode::OK); - assert_eq!(remove_commands(&c,NodeOperation::App),vec![("gpu".to_string(),json!({"app":{"namespace":ns,"namespace_name":"test","name":"model"},"remove":true}))]); + assert_eq!(remove_commands(&c,NodeOperation::App),vec![("gpu".to_string(), + json!({"app":{"namespace":ns,"namespace_name":"test","name":"model"},"remove":true,"delete":{"keep":[],"files":[]}}))]); +} + +/// The body of a response, as JSON. +async fn body(response: Response) -> Value { + serde_json::from_slice(&to_bytes(response.into_body(), 1 << 20).await.unwrap()).unwrap() } +/// Deleting a batch deletes its job records with their logs, and the commands about them, at once, +/// and tells each node which of its jobs' directories to remove. Its own record stays until its node +/// has removed it: a report before then brings back neither it nor its jobs, and its name stays taken. #[tokio::test] -async fn deleting_a_batch_sends_remove_batch_and_keeps_its_jobs() { +async fn deleting_a_batch_deletes_its_jobs_and_goes_once_its_node_removed_it() { let (_dir,c,ns)=setup().await; c.report("gpu","session",inventory(&ns)).await.unwrap(); let job=submit(&c).await; @@ -2077,13 +2008,128 @@ async fn deleting_a_batch_sends_remove_batch_and_keeps_its_jobs() { let mut report=inventory(&ns); report.jobs.push(incoming(&job,"succeeded",true)); c.report("gpu","session",report).await.unwrap(); - assert_eq!(scale_call(c.clone(),"admin","DELETE",&format!("/batches/training?namespace={ns}"),Value::Null).await.status(),StatusCode::OK); - assert_eq!(remove_commands(&c,NodeOperation::App),vec![("gpu".to_string(),json!({"batch":{"name":"training"},"remove":true}))]); - assert!(c.job(&job.id,None,&BTreeMap::new()).is_ok(),"job history and outputs follow the job's own lifecycle"); + let logs=c.dir.join("artifacts").join(&job.id).join(&job.lease).join("logs"); + fs::create_dir_all(&logs).unwrap(); + let launch=c.inner.lock().commands.about(&job.id).next().unwrap().id.clone(); + let deleted=scale_call(c.clone(),"admin","DELETE",&format!("/batches/training?namespace={ns}"),Value::Null).await; + assert_eq!(deleted.status(),StatusCode::OK); + let deleted=body(deleted).await; + assert_eq!((&deleted["jobs_deleted"],&deleted["nodes"]),(&json!(1),&json!(["gpu"])),"{deleted}"); + assert_eq!(remove_commands(&c,NodeOperation::App),vec![("gpu".to_string(),json!({"batch":{"name":"training"},"remove":true,"delete":{"jobs":[job.id]}}))]); + assert!(c.job(&job.id,None,&BTreeMap::new()).is_err()); + assert!(!c.dir.join("jobs").join(format!("{}.json",job.id)).exists() && !c.dir.join("artifacts").join(&job.id).exists()); + assert!(c.inner.lock().commands.get(&launch).is_none() && !c.dir.join("commands").join(format!("{launch}.json")).exists()); + let mut late=incoming(&job,"succeeded",true);late.id="other-job".into(); - let mut report=inventory(&ns);report.jobs.push(late); + let mut report=inventory(&ns);report.jobs.extend([incoming(&job,"succeeded",true),late]); + c.report("gpu","session",report).await.unwrap(); + assert!(c.job(&job.id,None,&BTreeMap::new()).is_err() && c.job("other-job",None,&BTreeMap::new()).is_err(),"a deleted batch imports no jobs"); + assert_eq!(scale_call(c.clone(),"admin","GET",&format!("/batches/training?namespace={ns}"),Value::Null).await.status(),StatusCode::NOT_FOUND); + let batches=format!("/batches?namespace={ns}"); + let create=|c: Arc| scale_call(c,"admin","POST",&batches,json!({"name":"training","image":"test"})); + let refused=create(c.clone()).await; + assert_eq!(refused.status(),StatusCode::CONFLICT); + assert!(String::from_utf8_lossy(&to_bytes(refused.into_body(),1<<20).await.unwrap()).contains("training is deleted, but not gone yet")); + + let removal=c.inner.lock().commands.open().find(|v|v.removes()==Some("training")).unwrap().id.clone(); + let mut report=inventory(&ns);report.batches.clear(); + report.results=vec![CommandResult{id:removal.clone(),result:json!({"removed":["training"],"jobs_deleted":1,"kept":[]}).into(),error:None}]; c.report("gpu","session",report).await.unwrap(); - assert!(c.job("other-job",None,&BTreeMap::new()).is_err(),"a deleted batch imports no jobs"); + assert!(!c.inner.lock().apps.contains_key("training") && !c.dir.join("apps/training.json").exists(),"gone once its node removed it"); + assert!(c.inner.lock().commands.get(&removal).is_none() && !c.dir.join("commands").join(format!("{removal}.json")).exists()); + assert_eq!(create(c.clone()).await.status(),StatusCode::CREATED,"and its name is free"); +} + +/// A node that fails to remove a deleted batch leaves its record there; deleting it again retries. +#[tokio::test] +async fn a_deleted_batch_a_node_failed_to_remove_is_deleted_again() { + let (_dir,c,ns)=setup().await; + c.report("gpu","session",inventory(&ns)).await.unwrap(); + let path=format!("/batches/training?namespace={ns}"); + assert_eq!(scale_call(c.clone(),"admin","DELETE",&path,Value::Null).await.status(),StatusCode::OK); + let removal=c.inner.lock().commands.open().find(|v|v.removes()==Some("training")).unwrap().id.clone(); + let mut report=inventory(&ns); + report.results=vec![CommandResult{id:removal.clone(),result:json!({"cleaned":false}).into(),error:Some("batches.yaml: not a mapping".into())}]; + c.report("gpu","session",report).await.unwrap(); + assert!(c.inner.lock().apps["training"].removed,"stays while not removed"); + assert_eq!(scale_call(c.clone(),"admin","DELETE",&path,Value::Null).await.status(),StatusCode::OK); + let retry=c.inner.lock().commands.open().find(|v|v.removes()==Some("training")).unwrap().id.clone(); + assert_ne!(retry,removal); + assert!(c.inner.lock().commands.get(&removal).is_none(),"the failed one goes with the retry"); + let mut report=inventory(&ns);report.batches.clear(); + report.results=vec![CommandResult{id:retry,result:json!({"removed":["training"],"jobs_deleted":0,"kept":[]}).into(),error:None}]; + c.report("gpu","session",report).await.unwrap(); + assert!(!c.inner.lock().apps.contains_key("training")); +} + +/// A batch an earlier release deleted kept its job records: it stays listed as deleted until it is +/// deleted again, which deletes them. One whose jobs that release purged goes at the next start. +#[tokio::test] +async fn a_batch_an_earlier_release_deleted_goes_with_its_jobs() { + let (_dir,c,ns)=setup().await; + c.report("gpu","session",inventory(&ns)).await.unwrap(); + assert_eq!(scale_call(c.clone(),"admin","POST",&format!("/batches?namespace={ns}"),json!({"name":"render","image":"test"})).await.status(),StatusCode::CREATED); + let kept=submit(&c).await; + let purged=c.submit("render",json!({"command":["true"],"gpus":1,"cpus":1}),None,&BTreeMap::new()).await.unwrap(); + let purged=c.job(text(&purged,"id"),None,&BTreeMap::new()).unwrap(); + for (mut r,flag) in [(kept.clone(),false),(purged.clone(),true)] { + r.status=json!({"state":"succeeded","submitted_at":1,"finished_at":2}).into(); + r.quota_held=false; + r.purged=flag; + c.save("jobs",&r.id,&r).unwrap(); + } + for name in ["training","render"] { + let mut t=c.inner.lock().apps[name].clone(); + t.removed=true; + c.save("apps",name,&t).unwrap(); + } + let c=Arc::new(WorkloadController::new(c.fleet.clone(),"cluster".into()).unwrap()); + assert!(!c.inner.lock().apps.contains_key("render") && !c.dir.join("apps/render.json").exists()); + assert!(!c.inner.lock().jobs.values().any(|r|r.id==purged.id) && !c.dir.join("jobs").join(format!("{}.json",purged.id)).exists()); + let listed=body(scale_call(c.clone(),"admin","GET",&format!("/batches?namespace={ns}"),Value::Null).await).await; + assert_eq!(listed.as_array().unwrap().iter().map(|b|(b["name"].clone(),b["state"].clone())).collect::>(),vec![(json!("training"),json!("deleted"))]); + let deleted=body(scale_call(c.clone(),"admin","DELETE",&format!("/batches/training?namespace={ns}"),Value::Null).await).await; + assert_eq!(deleted["jobs_deleted"],1,"{deleted}"); + assert!(!c.dir.join("jobs").join(format!("{}.json",kept.id)).exists()); + assert_eq!(body(scale_call(c.clone(),"admin","GET",&format!("/batches?namespace={ns}"),Value::Null).await).await,json!([])); +} + +/// Deleting a proof app deletes its jobs and tells every node which files they left in its storage: +/// each job's output directory, and the inputs and ELF that no other job or app uses. +#[tokio::test] +async fn deleting_a_proof_app_tells_its_nodes_the_files_its_jobs_left() { + use crate::store::{ProofJob,ProofJobState}; + use fleet_proto::fleet::v1::{ArtifactRef,ProofInputs,ProofKind}; + use fleet_proto::stage::v1::Status as StageStatus; + let (_dir,c,ns)=setup().await; + let other_elf="c".repeat(64); + register(&c,&ns,PROOF,&proof_elf()).await; + register(&c,&ns,"other",&other_elf).await; + let second=c.fleet.create_namespace("second",Quota{gpus:Some(1.0),cpus:Some(1.0)}).await.unwrap().id; + register(&c,&second,"twin",&proof_elf()).await; + c.report("gpu","session",inventory(&ns)).await.unwrap(); + let file=|path:&str,sha:&str| ArtifactRef{uri:format!("file:///store/{path}"),sha256:sha.into(),size:1}; + for (id,app,elf,input) in [("p1",PROOF,proof_elf(),"inputs/a"),("p2",PROOF,proof_elf(),"inputs/shared"),("o1","other",other_elf.clone(),"inputs/shared")] { + c.fleet.store.upsert(&ProofJob{ + inputs:Some(ProofInputs{input:Some(file(input,&"1".repeat(64))),elf:Some(file(&format!("elf/{elf}"),&elf)),output_root:"file:///store/outputs/".into()}), + outputs:None,id:id.into(),app:app.into(),owner:"b".repeat(40),namespace:ns.clone(),tag:String::new(),client_id:id.into(),elf_id:elf.clone(), + kind:ProofKind::Compressed as i32,priority:0,state:ProofJobState::Failed as i32,fail_status:Some(StageStatus::InternalError as i32), + error:String::new(),node:None,attempts:1,failed_on:vec![],cycles:0,prove_ms:0,created_ms:1,started_ms:Some(1),finished_ms:Some(2), + }).await.unwrap(); + } + let files=|c: &WorkloadController, app: &str| { + let i=c.inner.lock(); + let removal=i.commands.open().find(|v|v.removes()==Some(app)).expect("removal"); + let mut files: Vec=removal.parameters["delete"]["files"].as_array().unwrap().iter().map(|f|f.as_str().unwrap().to_owned()).collect(); + files.sort(); + files + }; + let deleted=body(scale_call(c.clone(),"admin","DELETE",&format!("/apps/other?namespace={ns}"),Value::Null).await).await; + assert_eq!((&deleted["jobs_deleted"],&deleted["nodes"]),(&json!(1),&json!(["gpu"])),"{deleted}"); + assert_eq!(files(&c,"other"),vec![format!("/store/elf/{other_elf}"),"/store/outputs/o1".into()],"its ELF is its own; the input is p2's too"); + let deleted=body(scale_call(c.clone(),"admin","DELETE",&format!("/apps/{PROOF}?namespace={ns}"),Value::Null).await).await; + assert_eq!(deleted["jobs_deleted"],2,"{deleted}"); + assert_eq!(files(&c,PROOF),vec!["/store/inputs/a","/store/inputs/shared","/store/outputs/p1","/store/outputs/p2"],"twin proves its ELF"); } /// Deleting an app and deleting a batch are one operation: refused while any of its work diff --git a/crates/fleet-scheduler/tests/flow.rs b/crates/fleet-scheduler/tests/flow.rs index bd07584..ec8bd8f 100644 --- a/crates/fleet-scheduler/tests/flow.rs +++ b/crates/fleet-scheduler/tests/flow.rs @@ -602,8 +602,9 @@ async fn an_app_is_deleted_with_its_jobs_once_none_is_running() { let gone: serde_json::Value = serde_json::from_str(&gone).unwrap(); assert_eq!((gone["app"].as_str(), gone["jobs_deleted"].as_u64()), (Some(app_of(b"elf-d").as_str()), Some(1)), "{gone}"); assert_eq!(get_job(&h, USER, &app_of(b"elf-d"), "job-d").await.0, 404, "the deleted app's job is gone"); - let again = admin_curl(&h, &["-o", "/dev/null", "-w", "%{http_code}", "-X", "DELETE", &url]); - assert_eq!(again, "404"); + // Until its nodes have removed it, deleting it again retries that; nothing else is left. + let again: serde_json::Value = serde_json::from_str(&admin_curl(&h, &["-X", "DELETE", &url])).unwrap(); + assert_eq!((again["removed"].as_bool(), again["jobs_deleted"].as_u64()), (Some(true), Some(0)), "{again}"); } /// The API through curl as an administrator (what fleetctl does), as text. diff --git a/crates/fleet-scheduler/tests/pg.rs b/crates/fleet-scheduler/tests/pg.rs index 3fd1cb5..5379d22 100644 --- a/crates/fleet-scheduler/tests/pg.rs +++ b/crates/fleet-scheduler/tests/pg.rs @@ -160,6 +160,22 @@ async fn check_registered_app_inventory(store: &dyn Store) { assert_eq!((registered.jobs, registered.failed, registered.created_ms), (1, 1, 3000)); store.delete_app(SYSTEM_NAMESPACE, &rejected.app).await.unwrap(); assert!(!store.apps().await.unwrap().iter().any(|a| a.name == rejected.app)); + check_referenced_files(store).await; +} + +/// A job's input and ELF are the files `referenced` names; its outputs and other files aren't. +async fn check_referenced_files(store: &dyn Store) { + use fleet_proto::fleet::v1::{ArtifactRef, ProofInputs}; + let file = |path: &str, sha: &str| ArtifactRef { uri: format!("file:///store/{path}"), sha256: sha.into(), size: 1 }; + let mut j = job("with-files", ProofJobState::Failed); + j.inputs = Some(ProofInputs { input: Some(file("inputs/i1", &"1".repeat(64))), elf: Some(file("elf/e", &j.elf_id)), + output_root: "file:///store/outputs".into() }); + store.upsert(&j).await.unwrap(); + let asked = ["inputs/i1", "elf/e", "inputs/i2", "outputs"].map(|p| format!("file:///store/{p}")); + let mut found: Vec = store.referenced(&asked).await.unwrap().into_iter().collect(); + found.sort(); + assert_eq!(found, vec!["file:///store/elf/e", "file:///store/inputs/i1"]); + assert!(store.referenced(&[]).await.unwrap().is_empty()); } #[tokio::test] @@ -212,7 +228,7 @@ async fn fresh_schema_requires_complete_job_identity() { assert!(store.apps().await.unwrap().iter().find(|a| a.name == "goat").unwrap().disabled); store.set_app_disabled(SYSTEM_NAMESPACE, "goat", false, 7).await.unwrap(); assert!(!store.apps().await.unwrap().iter().find(|a| a.name == "goat").unwrap().disabled); - assert_eq!(store.delete_app(SYSTEM_NAMESPACE, "goat").await.unwrap(), vec!["g2".to_string()], "the running one stays"); + assert_eq!(store.delete_app(SYSTEM_NAMESPACE, "goat").await.unwrap().iter().map(|j| j.id.as_str()).collect::>(), vec!["g2"], "the running one stays"); assert!(!store.apps().await.unwrap().iter().any(|a| a.name == "goat"), "the app is gone"); assert_eq!(store.incomplete().await.unwrap().iter().filter(|j| j.id == "g1").count(), 1); } diff --git a/crates/fleetctl/src/main.rs b/crates/fleetctl/src/main.rs index c521172..8975c64 100644 --- a/crates/fleetctl/src/main.rs +++ b/crates/fleetctl/src/main.rs @@ -134,11 +134,12 @@ enum AppCmd { /// Select a machine agent when the app runs on several nodes. #[arg(long)] node: Option, }, - /// Delete an app and remove it from its nodes, like batch delete: refused while any of it - /// runs, so disable a running app first, and wait for a proof app's queued or running - /// jobs. Its past execution records stay; a proof app goes with its jobs' records, inputs - /// and proofs, its anchor, and its ELF when no other app uses it. - /// Without --yes it only says what it would delete. + /// Delete an app and everything of it, like batch delete: its configuration and replicas on + /// its nodes, their logs and its workers' directories; a proof app's job records, with their + /// inputs and proofs, its anchor, and its ELF unless another app uses it. Its volumes, env + /// files, storage roots and image stay, and so does what another app or batch uses. Refused + /// while any of it runs, so disable a running app first, and wait for a proof app's queued + /// or running jobs. Without --yes it only says what it would delete. Delete { name: String, #[arg(long)] @@ -448,15 +449,6 @@ enum BatchJobCmd { /// Stop a batch job as an app needing its cards would: it queues again (same id and /// out/, FLEET_ATTEMPT counting on) and spends no --retry. Preempt { id: String, #[arg(long)] batch: Option }, - /// Delete a finished batch job's record (out/ included), or every finished batch job older than --older-than days. - Purge { - id: Option, - /// Only this batch's jobs (with --older-than), or the job's batch (found from the id when omitted). - #[arg(long)] - batch: Option, - #[arg(long)] - older_than: Option, - }, } #[derive(Subcommand)] @@ -554,9 +546,10 @@ enum BatchCmd { /// K=V every batch job gets; repeat. Replaces the batch's environment. #[arg(long)] env: Vec, }, - /// Delete a batch and remove it from its nodes, like app delete: refused while any of its - /// jobs is queued or running, so cancel them or let them finish first. Its job records - /// stay until purged. Without --yes it only says what it would delete. + /// Delete a batch and everything of it, like app delete: its configuration on its nodes, + /// and its jobs' records, logs and outputs on the control host and its nodes. Its mounts + /// stay. Refused while any of its jobs is queued or running, so cancel them or let them + /// finish first. Without --yes it only says what it would delete. Delete { name: String, #[arg(long)] yes: bool }, #[command(flatten)] Job(BatchJobCmd), @@ -841,8 +834,6 @@ const NODE_COLS: &[out::TableColumn] = &[ ]; const TOKEN_COLS: &[out::TableColumn] = &[col("TOKEN ID", "id"), col("NAMESPACE","namespace"), col("TAG","tag"), col("PERMISSIONS", "permissions"), col("LABEL", "label"), col("STATE", "state"), col("CREATED", "created_ms")]; -const PURGE_COLS: &[out::TableColumn] = - &[col("RUN", "id"), col("PURGED", "purged"), col("STATE", "state"), col("SIZE", "bytes"), col("REASON", "reason")]; /// Tokens as rows: a STATE column (active / revoked); proof identity stays internal. fn token_rows(v: &Value) -> Vec { @@ -1212,17 +1203,24 @@ fn describe_app(name: &str, conn: &ConnectionOptions, out: &OutputWriter) -> any Ok(()) } +/// What a deletion's nodes still remove, after its record and what the control host held went. +fn removing(deleted: &Value) -> String { + let nodes: Vec<&str> = deleted["nodes"].as_array().into_iter().flatten().filter_map(Value::as_str).collect(); + if nodes.is_empty() { String::new() } else { format!("; its nodes remove what they hold of it: {}", nodes.join(", ")) } +} + /// Deletes a batch (after showing what goes with it, until --yes), as `delete_app` deletes an app. fn delete_batch(a: &FleetClient, name: &str, batch: &Value, yes: bool, out: &OutputWriter) -> anyhow::Result<()> { let (total, active) = (&batch["jobs"]["total"], &batch["jobs"]["active"]); if !yes { bail!( - "would delete batch {name} and remove it from its nodes; its {total} job record(s) stay until \ - purged. Refused while any job is queued or running ({active} now). Pass --yes to delete" + "would delete batch {name} with its {total} job(s): their records, logs and outputs, on the control \ + host and its nodes. Its mounts stay. Refused while any job is queued or running ({active} now). \ + Pass --yes to delete" ); } let v = a.send(a.req(reqwest::Method::DELETE, &format!("/batches/{name}")))?; - out.done(&format!("deleted batch {name}; its {total} job record(s) stay until purged"), v); + out.done(&format!("deleted batch {name} and its {} job(s){}", v["jobs_deleted"], removing(&v)), v); Ok(()) } @@ -1232,27 +1230,24 @@ fn delete_app(name: &str, yes: bool, conn: &ConnectionOptions, out: &OutputWrite let app = app_named(&a, name)?; let (total, active) = (&app["jobs"]["total"], &app["jobs"]["active"]); let proof = app["proof"].is_object(); + let nodes = "its configuration and replicas on its nodes, their logs and its workers' directories"; if !yes { if proof { bail!( - "would delete app {name}: {total} job record(s) with their inputs and proofs, its anchor, and its ELF {} \ - unless another app uses it ({active} queued or running: refused while any is). Pass --yes to delete", + "would delete app {name}: {total} job record(s) with their inputs and proofs, {nodes}, its anchor, \ + and its ELF {} unless another app uses it. Its volumes, env files, storage roots and image stay \ + ({active} job(s) queued or running: refused while any is). Pass --yes to delete", app["proof"]["elf_id"].as_str().unwrap_or("") ); } bail!( - "would delete app {name} and remove it from its nodes; its {total} past execution record(s) \ - stay. Refused while it runs: disable it first. Pass --yes to delete" + "would delete app {name}: {nodes}. Its volumes, env files and image stay. Refused while it runs: \ + disable it first. Pass --yes to delete" ); } let v = a.send(a.req(reqwest::Method::DELETE, &format!("/apps/{name}")))?; - let text = if proof { - format!("deleted app {name}: {} job(s), ELF {}", v["jobs_deleted"], - if v["elf_removed"].as_bool() == Some(true) { "removed" } else { "kept (in use or absent)" }) - } else { - format!("deleted app {name}; its {total} past execution record(s) stay") - }; - out.done(&text, v); + let jobs = if proof { format!(" and its {} job(s)", v["jobs_deleted"]) } else { String::new() }; + out.done(&format!("deleted app {name}{jobs}{}", removing(&v)), v); Ok(()) } @@ -1489,22 +1484,6 @@ fn batch_action(cmd: &BatchJobCmd, conn: &ConnectionOptions, out: &OutputWriter) let path = batch_job_path(&a, batch.as_deref(), id)?; out.record(&a.send(a.req(M::POST, &format!("{path}/preempt")))?) } - BatchJobCmd::Purge { id, batch, older_than } => { - let purged = match (id, older_than) { - (Some(id), _) => a.send(a.req(M::DELETE, &batch_job_path(&a, batch.as_deref(), id)?))?, - (None, Some(days)) => { - let batches = match batch { Some(batch) => vec![batch.clone()], None => batch_names(&a)? }; - let mut purged = Vec::new(); - for batch in batches { - let v = a.send(a.req(M::DELETE, &format!("/batches/{batch}/jobs?older_than={days}")))?; - purged.extend(v.as_array().cloned().unwrap_or_default()); - } - Value::Array(purged) - } - (None, None) => bail!("purge ID, or purge --older-than DAYS"), - }; - out.table(purged.as_array().map(Vec::as_slice).unwrap_or_default(), PURGE_COLS, Some(&purged)); - } BatchJobCmd::Logs { id, batch, follow, lines } => { let path = batch_job_path(&a, batch.as_deref(), id)?; let mut chunk = a.get(&format!("{path}/logs?lines={lines}"))?; diff --git a/crates/fleetctl/tests/apps.rs b/crates/fleetctl/tests/apps.rs index 131fb1d..bc4904a 100644 --- a/crates/fleetctl/tests/apps.rs +++ b/crates/fleetctl/tests/apps.rs @@ -263,17 +263,21 @@ fn app_delete_says_what_goes_until_confirmed() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let url = format!("http://{}", listener.local_addr().unwrap()); let token = token_file("app-delete"); - let server = stub(listener, 3, move |line, _| (200, if line.starts_with("GET") { app.into() } else { r#"{"jobs_deleted":4,"elf_removed":true}"#.into() })); + let server = stub(listener, 3, move |line, _| (200, if line.starts_with("GET") { app.into() } else { r#"{"app":"ethproofs","jobs_deleted":4,"nodes":["gpu-a"]}"#.into() })); let refused = cli().args(["--namespace", "alice", "--url", &url, "--token-file"]).arg(&token) .args(["app", "delete", "ethproofs"]).output().unwrap(); assert!(!refused.status.success()); - assert!(String::from_utf8_lossy(&refused.stderr).contains("would delete app ethproofs: 4 job record(s)")); + let preview = String::from_utf8_lossy(&refused.stderr); + assert!(preview.contains("would delete app ethproofs: 4 job record(s) with their inputs and proofs, its configuration and \ + replicas on its nodes, their logs and its workers' directories, its anchor, and its ELF aaaa unless another app uses it. \ + Its volumes, env files, storage roots and image stay"), "{preview}"); let deleted = cli().args(["--namespace", "alice", "--url", &url, "--token-file"]).arg(&token) .args(["app", "delete", "ethproofs", "--yes"]).output().unwrap(); let seen = server.join().unwrap(); std::fs::remove_file(token).unwrap(); assert!(deleted.status.success(), "{}", String::from_utf8_lossy(&deleted.stderr)); - assert!(String::from_utf8_lossy(&deleted.stdout).contains("deleted app ethproofs: 4 job(s), ELF removed")); + let said = String::from_utf8_lossy(&deleted.stdout); + assert!(said.contains("deleted app ethproofs and its 4 job(s); its nodes remove what they hold of it: gpu-a"), "{said}"); assert!(seen[2].line.starts_with("DELETE /apps/ethproofs?namespace=alice"), "{}", seen[2].line); } @@ -283,19 +287,22 @@ fn batch_delete_says_what_goes_until_confirmed_like_app_delete() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let url = format!("http://{}", listener.local_addr().unwrap()); let token = token_file("batch-delete"); - let server = stub(listener, 3, move |line, _| (200, if line.starts_with("GET") { batch.into() } else { r#"{"name":"training","removed":true}"#.into() })); + let server = stub(listener, 3, move |line, _| (200, if line.starts_with("GET") { batch.into() } else { + r#"{"name":"training","removed":true,"jobs_deleted":5,"nodes":["gpu-a","gpu-b"]}"#.into() })); let refused = cli().args(["--namespace", "alice", "--url", &url, "--token-file"]).arg(&token) .args(["batch", "delete", "training"]).output().unwrap(); assert!(!refused.status.success()); let preview = String::from_utf8_lossy(&refused.stderr); - assert!(preview.contains("would delete batch training and remove it from its nodes; its 5 job record(s) stay until purged"), "{preview}"); + assert!(preview.contains("would delete batch training with its 5 job(s): their records, logs and outputs, on the control host \ + and its nodes. Its mounts stay."), "{preview}"); assert!(preview.contains("(1 now). Pass --yes to delete"), "{preview}"); let deleted = cli().args(["--namespace", "alice", "--url", &url, "--token-file"]).arg(&token) .args(["batch", "delete", "training", "--yes"]).output().unwrap(); let seen = server.join().unwrap(); std::fs::remove_file(token).unwrap(); assert!(deleted.status.success(), "{}", String::from_utf8_lossy(&deleted.stderr)); - assert!(String::from_utf8_lossy(&deleted.stdout).contains("deleted batch training; its 5 job record(s) stay until purged")); + let said = String::from_utf8_lossy(&deleted.stdout); + assert!(said.contains("deleted batch training and its 5 job(s); its nodes remove what they hold of it: gpu-a, gpu-b"), "{said}"); assert!(seen[2].line.starts_with("DELETE /batches/training?namespace=alice"), "{}", seen[2].line); } diff --git a/docs/maintenance.md b/docs/maintenance.md index d5dda65..8b4c4ac 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -263,8 +263,8 @@ at a full core with sqlx `slow_acquire` warnings although Postgres is idle. - **Up to v0.1.8**, a bulk purge of a busy batch's history (`fleetctl job purge --older-than DAYS`) does the same: it holds that lock while it writes a record and a node command for every job, minutes for tens of thousands, and its node commands - then crowd out launches for as long again, at 16 per heartbeat. Later releases purge - in chunks and send a node's purges after its other commands. + then crowd out launches for as long again, at 16 per heartbeat. Later releases + purged in chunks; now there is no purge, and a batch's jobs go when it is deleted. ### Who changed what @@ -272,7 +272,7 @@ The scheduler logs an `api request` line for each API call that can change state method, path and query, status, duration, and the caller's token ID, label, namespace and tag (never the token itself). `docker logs zkm-fleet-scheduler-1 | grep 'api request'` shows who submitted, -cancelled, purged or reconfigured what. +cancelled, deleted or reconfigured what. ## Installed files and backups diff --git a/docs/reference.md b/docs/reference.md index fe74fe1..6e89d08 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -209,11 +209,28 @@ is and how to stop it: - A proof app's work also includes its queued or running proofs. - A batch's work is its queued or running jobs. Cancel them or let them finish. -The delete then removes the record and has every node remove the app's entries or the -batch from its configuration. The data on the nodes stays. A container app's past -execution records and a batch's job records stay until purged. A proof app's finished -proof records and control-host artifacts go with it, and the app's own storage stays -with its owner. +The app or batch then goes with everything Fleet holds of it: +- On the control host: its job records with their logs, and the commands about them. + A proof app's job records and anchor go from Postgres. +- On each node: its configuration, with what the node holds of it. A batch's jobs lose + their directories: record, logs and `out/`. An app's replicas' logs and its workers' + directories (`worker_workdir`) go, and so do its proof jobs' files in its proof + storage: each job's output directory, and the inputs and ELF that no other job or + app uses. An EthProofs worker's directory is its proof storage root, so its + checkpoints and proofs go with it. + +Volumes, batch mounts, env files, storage roots and images stay. So does whatever +holds or lies in a path that another app or batch on any node uses, and anything at +the top of the filesystem or around Fleet's own files; the node's result says what it +kept and why, and the scheduler logs it. A container's `workdir` is the container's +own: what it keeps on the host is in its volumes. Job history goes only with its +batch; there is no separate purge. + +The record stays, hidden, until every node has removed it, so the name can't be +reused before then. A node that is offline removes it once it's back; deleting it +again retries a node that failed. A batch deleted by a release before this one kept +its job records: it stays listed as `deleted` until it is deleted again, which removes +them. A node with an older bridge removes only the configuration. **Node entries.** Every `apps.yaml` entry has an `app`. A `ziren-gpu` entry binds its server to the one app it serves with `namespace`, `app: NAME` and @@ -246,7 +263,7 @@ fleetctl batch # the batches, with their active and last jobs fleetctl batch NAME # one batch and its newest jobs ``` -`fleetctl batch logs`, `cancel`, `preempt` and `purge` take a job ID and find its +`fleetctl batch logs`, `cancel` and `preempt` take a job ID and find its batch; `--batch NAME` skips that lookup. `batch submit --name` names the job. Nodes can also advertise batches in their `batches.yaml`. @@ -271,9 +288,7 @@ its `namespace` field; a name or an ID is accepted, and the ID is stored. A batc `apps.yaml` entry configured on a node may name `namespace: alice`, which is checked before launch. Disabling a batch blocks new submissions and lets accepted jobs finish. Deleting a batch works like deleting an app ([Deleting](#apps)): it is -refused while any of its jobs is queued or running. A deleted batch stays listed, as -`deleted`, and readable while it has job records; its jobs keep their own lifecycle -until they're purged. +refused while any of its jobs is queued or running, and its jobs go with it. **Submissions.** A job takes `command`, a non-empty list of strings, and optionally: @@ -515,7 +530,7 @@ accepted work: an administrator stops jobs explicitly. |---|---| | Read the own namespace, apps, batches, jobs, logs and scaling status | `read`, within the token's scope | | Submit proofs and batch jobs; cancel batch jobs | `submit`, within the token's scope | -| Create, remove, enable, disable, prioritize, scale or update apps and batches; purge batch jobs | `manage`, with any app grants. Removing, enabling, disabling or prioritizing a proof app also needs the tag it was created for and the tag of every node entry running it; scaling and updating it need the tag of the entry they change. | +| Create, remove, enable, disable, prioritize, scale or update apps and batches | `manage`, with any app grants. Removing, enabling, disabling or prioritizing a proof app also needs the tag it was created for and the tag of every node entry running it; scaling and updating it need the tag of the entry they change. | | Batch `allow` and `user`; app `user`, `volumes` and `proof_storage_root`; changing an app's image, environment, command, user, workdir or volumes | `admin` | | Preempt batch jobs | `admin` | | Namespaces, tokens, quotas, suspension, allocations, the preemption plan, nodes and status | `admin` | @@ -622,7 +637,7 @@ listener behind the HTTPS gateway. Administrators select a namespace with | Scaling | `PUT` / `GET /apps/NAME/scale` (the app's replicas over all its nodes; `GET ...?node=NODE` shows one node's part) | | Proof app jobs | `GET /apps/NAME/jobs?limit=N&active=true`, `POST /apps/NAME/jobs` (app-storage references), `GET /apps/NAME/jobs/ID`, `GET /apps/NAME/jobs/ID/output/proof.json` and `.../output/public_values.bin` | | Batches | `GET` / `POST /batches`, `GET` / `DELETE /batches/NAME`, `PUT /batches/NAME/priority`, `PUT /batches/NAME/settings`, `PUT` / `DELETE /batches/NAME/disabled` | -| Batch jobs | `GET` / `POST /batches/NAME/jobs`, `DELETE /batches/NAME/jobs?older_than=DAYS`, `GET` / `DELETE /batches/NAME/jobs/ID`, `POST /batches/NAME/jobs/ID/cancel`, `POST /batches/NAME/jobs/ID/preempt`, `GET /batches/NAME/jobs/ID/logs` | +| Batch jobs | `GET` / `POST /batches/NAME/jobs`, `GET /batches/NAME/jobs/ID`, `POST /batches/NAME/jobs/ID/cancel`, `POST /batches/NAME/jobs/ID/preempt`, `GET /batches/NAME/jobs/ID/logs` | | Namespaces and tokens | Action | |---|---|