diff --git a/bridge/fleet_bridge/node.py b/bridge/fleet_bridge/node.py index d98ee67..49c5628 100644 --- a/bridge/fleet_bridge/node.py +++ b/bridge/fleet_bridge/node.py @@ -8,6 +8,7 @@ import json import logging import os +import re import shutil import socket import subprocess @@ -183,6 +184,11 @@ def append_entry(path: Path, section: str, entry: dict[str, Any]) -> None: raise ValueError(f"{section}: must be a list to add an entry to") _replace(path, text) +def _holds(outer: str, inner: str) -> bool: + """Whether directory `outer` is `inner` or holds it.""" + return os.path.commonpath([outer, inner]) == outer + + class NodeExecutor: def __init__(self, apps: str, batches: str, jobs_dir: str) -> None: self.apps_file, self.batches, self.jobs_dir = Path(apps), batches, Path(jobs_dir) @@ -239,20 +245,11 @@ def execute(self, command: dict[str, Any]) -> None: elif operation == 4: with self.apps_lock: result = self.set_app(params) - elif operation == 5: - directory = self.jobs_dir / params["id"] - if directory.exists(): - current = read_json(directory / ".fleet-lease.json", {"lease": "imported"})["lease"] - if current != params["lease"]: - raise ValueError("stale purge lease") - job = jobs.JobDirectory(str(self.jobs_dir), params["id"]) - if not self.cleanup(job): - raise ValueError("job cleanup is not confirmed") - shutil.rmtree(job.path) - result = {"cleaned": True} elif operation == 7: with self.apps_lock: - result = self.app(params) + result, then = self.app(params) + # Deleting files can take long; other app changes needn't wait for it. + result.update(then()) else: raise ValueError("unsupported node operation") except BaseException as e: @@ -410,15 +407,24 @@ def _remove_replicas(names: list[str]) -> None: if not container_gone(actors.gpu_container(name)) or not container_gone(actors.app_container(name)): raise ValueError(f"{name}: app container cleanup is uncertain") - def app(self, p: dict[str, Any]) -> dict[str, Any]: + def app(self, p: dict[str, Any]) -> tuple[dict[str, Any], Any]: """An app-level change from the scheduler, for an app (`p["app"]`: the entries that run the app of that name, in a namespace given by id and by name), a batch (`p["batch"]`, by name), or the one entry a change of an app stopped (`p["entry"]`, by name, set only): `remove` it, or `set` some of its configuration. - A removed entry has what is left of its replicas cleaned up the way disabling does; - app data stays (storage roots, workdirs, env files and images belong to the operator). Settings are written to the configuration only: the scheduler drains and stops an app first and starts it again afterwards. `p["create"]` adds a new app's - entry to the configuration, disabled until the app is enabled.""" + entry to the configuration, disabled until the app is enabled. + + A removed entry has what is left of its replicas cleaned up the way disabling does. + Removing an app from this node alone (a scale left it no replicas here) keeps its + data. Deleting it (`p["delete"]`) removes what this node holds of it too: its + replicas' logs, its worker's directory and the files of its proof jobs + (`delete["files"]`) in its proof storage. A deleted batch's jobs (`delete["jobs"]`) + lose their directories: record, logs and outputs. Volumes, mounts, env files, + storage roots and images stay, and so does whatever holds or lies in a path another + app or batch uses, here or (by `delete["keep"]`) on another node. Returns the + result, and what deletes the files, for after the configuration lock.""" + nothing = lambda: {} if "create" in p: entry = p["create"] AppSpec.load(entry) @@ -426,26 +432,29 @@ def app(self, p: dict[str, Any]) -> dict[str, Any]: if any(e.get("name") == entry["name"] for e in entries): raise ValueError(f"{entry['name']} is already configured on this node") append_entry(self.apps_file, "apps", entry) - return {"created": [entry["name"]]} + return {"created": [entry["name"]]}, nothing if "batch" in p: name, path = p["batch"]["name"], Path(self.batches) if p.get("remove"): - return {"removed": [name] if remove_entry(path, "batches", name) else []} + removed = [name] if remove_entry(path, "batches", name) else [] + if "delete" not in p: + return {"removed": removed}, nothing + return {"removed": removed}, lambda: self._delete_jobs(name, p["delete"].get("jobs") or []) entry = (yaml.safe_load(path.read_text()) or {}).get("batches", {}).get(name) if entry is None: - return {"updated": []} + return {"updated": []}, nothing Batch.load({**entry, **p["set"]}) set_fields(path, "batches", name, p["set"]) - return {"updated": [name]} + return {"updated": [name]}, nothing if "entry" in p: # The one entry a change of an app stopped: its configuration only. name = p["entry"]["name"] entries = {e["name"]: e for e in (yaml.safe_load(self.apps_file.read_text()) or {}).get("apps", [])} if p.get("remove") or name not in entries: - return {"updated": []} + return {"updated": []}, nothing AppSpec.load({**entries[name], **p["set"]}) set_fields(self.apps_file, "apps", name, p["set"]) - return {"updated": [name]} + return {"updated": [name]}, nothing target = p["app"] namespaces = {target["namespace"], target.get("namespace_name") or target["namespace"]} entries = {e["name"]: e for e in (yaml.safe_load(self.apps_file.read_text()) or {}).get("apps", [])} @@ -457,13 +466,123 @@ def app(self, p: dict[str, Any]) -> dict[str, Any]: AppSpec.load({**entries[app.name], **p["set"]}) for app in apps: set_fields(self.apps_file, "apps", app.name, p["set"]) - return {"updated": [a.name for a in apps]} + return {"updated": [a.name for a in apps]}, nothing removed = [] for app in apps: self._remove_replicas([f"{app.name}-{index}" for index in range(app.replicas)]) if remove_entry(self.apps_file, "apps", app.name): - removed.append(app.name) - return {"removed": removed} + removed.append(app) + if "delete" not in p: + return {"removed": [a.name for a in removed]}, nothing + # What the node's other apps and batches use, read under the lock. + in_use = self._paths_in_use(p["delete"].get("keep") or [], removed) + log_dir = cfg.cluster.log_dir + return ({"removed": [a.name for a in removed]}, + lambda: self._delete_app_files(removed, log_dir, in_use, p["delete"].get("files") or [])) + + def _paths_in_use(self, keep: list[str], removed: list[AppSpec]) -> list[str]: + """The host paths that a deletion keeps, with what holds or lies in one: those of + the apps and batches this node still configures, `keep` (the ones on other nodes), + and the volumes and env files of the `removed` entries.""" + paths = list(keep) + for e in (yaml.safe_load(self.apps_file.read_text()) or {}).get("apps") or []: + paths += [e.get(k) for k in ("workdir", "worker_workdir", "proof_storage_root", "worker_env_file")] + paths += [str(v).split(":")[0] for v in e.get("volumes") or []] + for b in ((yaml.safe_load(Path(self.batches).read_text()) or {}).get("batches") or {}).values(): + paths += list((b or {}).get("allow") or []) + for a in removed: + paths += [v.split(":")[0] for v in a.volumes] + [a.worker_env_file] + return [os.path.realpath(p) for p in paths if isinstance(p, str) and os.path.isabs(p)] + + def _refusal(self, path: str, in_use: list[str], log_dir: str, *, inside_ok: bool = False) -> str | None: + """Why `path` must stay, or None when it can go. Fleet's own directories stay, with + what holds or lies in one; so do the filesystem's top level and what holds the home + directory, and what holds a path in use or (unless `inside_ok`) lies in one.""" + if not os.path.isabs(path) or ".." in Path(path).parts: + return "not an absolute path" + if os.path.islink(path): + return "a symbolic link" + real = os.path.realpath(path) + if len(Path(real).parts) < 3 or _holds(real, os.path.realpath(Path.home())): + return "too close to the root of the filesystem" + for own in (self.jobs_dir, self.apps_file.parent, Path(self.batches).parent, log_dir, Path.cwd()): + own = os.path.realpath(own) + if _holds(real, own) or _holds(own, real): + return f"holds or lies in fleet's own {own}" + for used in in_use: + if _holds(real, used) or not inside_ok and _holds(used, real): + return f"another app or batch uses {used}" + return None + + def _delete(self, path: str, reason: str | None, deleted: list[str], kept: list[dict[str, str]]) -> None: + if reason is None: + try: + if os.path.isdir(path) and not os.path.islink(path): + shutil.rmtree(path) + else: + os.unlink(path) + deleted.append(path) + return + except OSError as e: + reason = f"{type(e).__name__}: {e.strerror or e}" + kept.append({"path": path, "reason": reason}) + + def _delete_app_files(self, removed: list[AppSpec], log_dir: str, in_use: list[str], files: list[str]) -> dict[str, Any]: + """Deletes what the `removed` entries of a deleted app held: the files of its proof + jobs in their proof storage, their replicas' logs and their workers' directories.""" + deleted: list[str] = [] + kept: list[dict[str, str]] = [] + roots = [os.path.realpath(a.proof_storage_root) for a in removed if a.proof_storage_root] + for path in files if roots else []: + if not os.path.lexists(path): + continue + real = os.path.realpath(path) + if not any(real != root and _holds(root, real) for root in roots): + kept.append({"path": path, "reason": "not in the app's proof storage on this node"}) + continue + self._delete(path, self._refusal(path, in_use, log_dir, inside_ok=True), deleted, kept) + logs = sorted(os.listdir(log_dir)) if os.path.isdir(log_dir) else [] + for a in removed: + pattern = re.compile(rf"{re.escape(a.name)}-\d+\.node\.log") + for name in filter(pattern.fullmatch, logs): + self._delete(os.path.join(log_dir, name), None, deleted, kept) + if a.worker_workdir and os.path.lexists(a.worker_workdir): + self._delete(a.worker_workdir, self._refusal(a.worker_workdir, in_use, log_dir), deleted, kept) + if deleted or kept: + log.info("deleted app's files: %d deleted, kept %s", len(deleted), kept) + return {"deleted": deleted, "kept": kept} + + def _delete_jobs(self, batch: str, ids: list[str]) -> dict[str, Any]: + """Deletes the directories of the jobs `ids` of deleted batch `batch`: each one's + record, logs and outputs. One whose container may still be there stays.""" + deleted, kept = 0, [] + for ident in ids: + try: + job = jobs.JobDirectory(str(self.jobs_dir), ident) + except jobs.NotFound: + continue + directory = Path(job.path) + try: + try: + owner = job.spec().batch + except Exception: + owner = batch # a record half written: the scheduler lists it as the batch's + if owner != batch: + kept.append({"path": job.path, "reason": f"a job of batch {owner}"}) + continue + lease = read_json(directory / ".fleet-lease.json", {"lease": "imported"})["lease"] + if not (directory / f".fleet-cleaned-{lease}").exists() and not self.cleanup(job): + kept.append({"path": job.path, "reason": "its container may still be there"}) + continue + shutil.rmtree(directory) + for stop in (self.jobs_dir / ".node" / "stops").glob(f"{ident}-*.json"): + stop.unlink(missing_ok=True) + deleted += 1 + except Exception as e: + kept.append({"path": job.path, "reason": f"{type(e).__name__}: {e}"}) + if deleted or kept: + log.info("deleted batch %s: %d job directories deleted, kept %s", batch, deleted, kept) + return {"jobs_deleted": deleted, "kept": kept} def report(self) -> dict[str, Any]: nodes = alive_nodes() diff --git a/bridge/tests/test_node.py b/bridge/tests/test_node.py index 4e7d2d3..8030c1b 100644 --- a/bridge/tests/test_node.py +++ b/bridge/tests/test_node.py @@ -1,4 +1,5 @@ """Local execution fences are independent of user authentication and admission.""" +import os from unittest.mock import Mock import pytest import yaml @@ -93,14 +94,6 @@ def test_start_uses_local_executor_and_no_admin_token(executor,monkeypatch): assert all(not c.kwargs for c in start.call_args_list) -def test_purge_refuses_an_old_execution_lease(executor): - c=command(5);path=executor.jobs_dir/c["parameters"]["id"];path.mkdir(parents=True) - node.write_json(path/".fleet-lease.json",{"lease":"new-lease"}) - executor.execute(c) - assert path.exists() - assert "stale purge" in node.read_json(executor.journal/"command-1.json")["error"] - - def test_local_ray_must_contain_only_this_machine(executor,monkeypatch): monkeypatch.setattr(node.ray,"nodes",lambda:[{"Alive":True},{"Alive":True}]) with pytest.raises(ValueError,match="only this machine"): @@ -295,7 +288,7 @@ def test_app_removal_cleans_up_and_drops_the_entry(executor,monkeypatch): executor.apps_file.write_text(yaml.safe_dump(apps)) assert app_message(executor,"remove-1",{"remove":True},app="eth")==({"removed":["proof"]},None) assert [c.args[0] for c in stop.call_args_list]==["proof-0","proof-1"] - assert app_message(executor,"remove-2",{"remove":True},app="eth")==({"removed":[]},None) + assert app_message(executor,"remove-2",{"remove":True,"delete":{}},app="eth")==({"removed":[],"deleted":[],"kept":[]},None) assert app_message(executor,"remove-3",{"remove":True},app="web",ns="1"*32,ns_name="zkm")==({"removed":["web"]},None) assert [s["name"] for s in yaml.safe_load(executor.apps_file.read_text())["apps"]]==["other-ns"] (executor.apps_file.parent/"batches.yaml").write_text("batches:\n old: {image: a}\n kept: {image: b}\n") @@ -303,21 +296,120 @@ def test_app_removal_cleans_up_and_drops_the_entry(executor,monkeypatch): assert list(yaml.safe_load((executor.apps_file.parent/"batches.yaml").read_text())["batches"])==["kept"] -def test_removing_an_app_keeps_its_data(executor,monkeypatch,tmp_path): - """A start placed on another node has its own node remove the entry; the app's storage is its own.""" +def removable(monkeypatch): + """Replicas that stop at once and leave no container.""" monkeypatch.setattr(reconcile,"stop",Mock());monkeypatch.setattr(actors,"clean_after",Mock()) monkeypatch.setattr(node.ray,"get",lambda *a,**k:None);monkeypatch.setattr(node,"container_gone",lambda _:True) - data={name:tmp_path/"data"/name for name in ("storage","workdir","models")} - for d in data.values(): - d.mkdir(parents=True);(d/"cursor").write_text("42") - env=tmp_path/"data"/"worker.env";env.write_text("RPC=x\n") + + +def files(root, *paths): + for path in paths: + (root/path).parent.mkdir(parents=True,exist_ok=True);(root/path).write_text("x") + + +def configure(executor, *entries, log_dir=None): apps=yaml.safe_load(executor.apps_file.read_text()) - apps["apps"]=[{"name":"eth-worker","kind":"ziren-gpu","namespace":"a"*32,"app":"eth","elf_id":"b"*64,"image":"ziren-gpu:1", - "machine":"m1","worker_command":["/opt/eth-proofs"],"worker_env_file":str(env),"worker_workdir":str(data["workdir"]), - "proof_storage_root":str(data["storage"]),"volumes":[f"{data['models']}:ro"]}] + apps["apps"]=list(entries) + if log_dir: + apps["log_dir"]=str(log_dir) executor.apps_file.write_text(yaml.safe_dump(apps)) + + +def worker(name, app, data, **more): + """An entry with an integrated worker, which works in data//run and keeps proofs in data//store.""" + return {"name":name,"kind":"ziren-gpu","namespace":"a"*32,"app":app,"elf_id":"b"*64,"image":"ziren-gpu:1", + "machine":"m1","worker_command":["/opt/worker"],"worker_workdir":str(data/app/"run"), + "proof_storage_root":str(data/app/"store"),**more} + + +def test_removing_an_app_from_one_node_keeps_its_data(executor,monkeypatch,tmp_path_factory): + """A scale that leaves a node no replicas has it remove the app's entries; the app runs on elsewhere.""" + removable(monkeypatch) + data=tmp_path_factory.mktemp("data") + files(data,"eth/run/checkpoint","eth/store/outputs/job-1/lease/proof.json","logs/eth-worker-0.node.log") + configure(executor,worker("eth-worker","eth",data),log_dir=data/"logs") assert app_message(executor,"remove-1",{"remove":True},app="eth")==({"removed":["eth-worker"]},None) - assert all((d/"cursor").read_text()=="42" for d in data.values()) and env.read_text()=="RPC=x\n" + assert len([f for f in data.rglob("*") if f.is_file()])==3 + + +def test_deleting_an_app_deletes_what_its_entries_held(executor,monkeypatch,tmp_path_factory): + """A deleted app's node deletes its replicas' logs, its worker's directory and the files of its proof + jobs in its proof storage. Its storage root, volumes and env file stay, and so does what isn't its own.""" + removable(monkeypatch) + data=tmp_path_factory.mktemp("data") + files(data,"eth/run/checkpoint","eth/store/outputs/job-1/lease/proof.json","eth/store/outputs/job-2/lease/proof.json", + "eth/store/inputs/i1","eth/store/elf/e1","models/weights","worker.env","elsewhere/proof.json", + "logs/eth-worker-0.node.log","logs/eth-worker-11.node.log","logs/eth-worker-db-0.node.log","logs/node.log") + configure(executor,worker("eth-worker","eth",data,worker_env_file=str(data/"worker.env"),volumes=[f"{data/'models'}:ro"]), + log_dir=data/"logs") + listed=[str(data/p) for p in ("eth/store/outputs/job-1","eth/store/inputs/i1","eth/store/elf/e1","eth/store/inputs/gone", + "eth/store","elsewhere/proof.json")] + result,error=app_message(executor,"remove-1",{"remove":True,"delete":{"keep":[],"files":listed}},app="eth") + assert error is None and result["removed"]==["eth-worker"] + assert sorted(result["deleted"])==sorted(str(data/p) for p in ("eth/store/outputs/job-1","eth/store/inputs/i1","eth/store/elf/e1", + "logs/eth-worker-0.node.log","logs/eth-worker-11.node.log","eth/run")) + assert result["kept"]==[{"path":str(data/p),"reason":"not in the app's proof storage on this node"} for p in ("eth/store","elsewhere/proof.json")] + left=sorted(str(f.relative_to(data)) for f in data.rglob("*") if f.is_file()) + assert left==["elsewhere/proof.json","eth/store/outputs/job-2/lease/proof.json","logs/eth-worker-db-0.node.log","logs/node.log", + "models/weights","worker.env"] + + +def test_deleting_an_app_keeps_what_another_app_or_batch_uses(executor,monkeypatch,tmp_path_factory): + """Its worker's directory stays when it holds or lies in a path that another app or batch uses: on this + node, as its configuration says, or on another, as the scheduler says. So does one that is too close to + the root or holds fleet's own files.""" + removable(monkeypatch) + data=tmp_path_factory.mktemp("data") + files(data,"web/run/state","shared/run/state") + cases=[ + ({}, [worker("web-db","db",data,volumes=[str(data/"web")])], "batches: {}\n", f"another app or batch uses {data/'web'}"), + ({"keep":[str(data/"web/run/state")]}, [], "batches: {}\n", f"another app or batch uses {data/'web/run/state'}"), + ({}, [], f"batches:\n render: {{image: a, allow: [{data}]}}\n", f"another app or batch uses {data}"), + ] + for n,(change,others,batches,reason) in enumerate(cases): + configure(executor,worker("web-gpu","web",data),*others) + (executor.apps_file.parent/"batches.yaml").write_text(batches) + result,error=app_message(executor,f"remove-{n}",{"remove":True,"delete":change},app="web") + assert error is None and result["kept"]==[{"path":str(data/"web/run"),"reason":reason}], result + assert (data/"web/run/state").exists() + (executor.apps_file.parent/"batches.yaml").write_text("batches: {}\n") + for n,(workdir,reason) in enumerate([("/opt","too close to the root of the filesystem"), + (str(executor.jobs_dir.parent),f"holds or lies in fleet's own {executor.jobs_dir}")]): + configure(executor,{**worker("web-gpu","web",data),"worker_workdir":workdir}) + executor.jobs_dir.mkdir(exist_ok=True) + result,error=app_message(executor,f"root-{n}",{"remove":True,"delete":{}},app="web") + assert result["kept"]==[{"path":workdir,"reason":reason}], result + assert os.path.isdir("/opt") and executor.apps_file.exists() + + +def job_directory(executor, ident, batch, *, cleaned=True): + directory=executor.jobs_dir/ident + (directory/"out").mkdir(parents=True) + (directory/"out/frame.png").write_text("output") + JobSpec(id=ident, batch=batch, policy=Batch(image="render:1"), command=["true"], gpus=0, cpus=1, + out=str(directory/"out")).write(str(directory/"spec.json")) + node.write_json(directory/".fleet-lease.json", {"lease": "lease-1"}) + if cleaned: + (directory/".fleet-cleaned-lease-1").touch() + stops=executor.jobs_dir/".node"/"stops";stops.mkdir(parents=True,exist_ok=True) + node.write_json(stops/f"{ident}-lease-1.json", {"operation": 2}) + return directory + + +def test_deleting_a_batch_deletes_its_jobs_directories(executor,monkeypatch): + """Each listed job of the deleted batch loses its directory: record, logs and outputs. One of another + batch stays, and so does one whose container may still be there.""" + (executor.apps_file.parent/"batches.yaml").write_text("batches:\n render: {image: a}\n") + done=job_directory(executor,"job-done","render") + unsure=job_directory(executor,"job-unsure","render",cleaned=False) + other=job_directory(executor,"job-other","train") + monkeypatch.setattr(jobs,"_going",lambda _:False);monkeypatch.setattr(node,"container_gone",lambda _:False) + result,error=app_message(executor,"remove-1",{"remove":True,"delete":{"jobs":["job-done","job-unsure","job-other","job-gone","../etc"]}},batch="render") + assert error is None + assert result=={"removed":["render"],"jobs_deleted":1,"kept":[ + {"path":str(unsure),"reason":"its container may still be there"},{"path":str(other),"reason":"a job of batch train"}]} + assert not done.exists() and unsure.exists() and other.exists() + assert sorted(p.name for p in (executor.jobs_dir/".node"/"stops").iterdir())==["job-other-lease-1.json","job-unsure-lease-1.json"] def test_an_app_whose_cleanup_is_uncertain_stays_configured(executor,monkeypatch): diff --git a/crates/fleet-node/src/main.rs b/crates/fleet-node/src/main.rs index 055a5c4..10a7476 100644 --- a/crates/fleet-node/src/main.rs +++ b/crates/fleet-node/src/main.rs @@ -3,7 +3,8 @@ mod logs; use anyhow::{Context, Result}; use clap::Parser; use fleet_proto::fleet::v1::{ - worker_service_client::WorkerServiceClient, LogRequest, ProofFetch, WorkerHeartbeatRequest, + worker_service_client::WorkerServiceClient, LogRequest, ProofFetch, WorkerEndSessionRequest, + WorkerHeartbeatRequest, }; use serde_json::{json, Value}; use std::{ @@ -75,6 +76,24 @@ fn outbound(mut report: Value, stale: bool, version: Option<&str>) -> Value { } report } +/// Tells the control plane this agent's session is over, so the agent that replaces it (a +/// restart, an upgrade) is admitted at once instead of after the session goes quiet. Best +/// effort: a scheduler without the call, or one out of reach, waits it out as before. +async fn end_session(endpoint: Endpoint, auth: tonic::metadata::MetadataValue, worker: &str, session: &str) { + let ended = async { + let mut client = connect(endpoint).await?; + let mut request = tonic::Request::new(WorkerEndSessionRequest { worker_id: worker.into(), session: session.into() }); + request.metadata_mut().insert("authorization", auth); + request.set_timeout(Duration::from_secs(3)); + client.end_session(request).await?; + anyhow::Ok(()) + }; + match tokio::time::timeout(Duration::from_secs(5), ended).await { + Ok(Ok(())) => tracing::info!(worker, "agent session ended"), + Ok(Err(error)) => tracing::warn!(%error, "could not end the agent session; the next agent waits until it goes quiet"), + Err(_) => tracing::warn!("ending the agent session timed out; the next agent waits until it goes quiet"), + } +} async fn connect(endpoint: Endpoint) -> Result> { Ok(WorkerServiceClient::new(endpoint.connect().await?) .max_decoding_message_size(16 << 20) @@ -298,6 +317,8 @@ async fn main() -> Result<()> { log_task.abort(); reader.abort(); let _ = child.kill().await; + // Stopped: nothing of this agent reports any more. + end_session(endpoint, auth, &a.name, &session).await; Ok(()) } @@ -548,4 +569,46 @@ mod tests { let unnamed = outbound(json!({"version":3,"machine":"m","ready":true}), false, None); assert!(unnamed.get("agent_version").is_none()); } + + /// A control plane that records the sessions agents end, with the token they sent. + struct Ends(Arc>>); + #[tonic::async_trait] + impl fleet_proto::fleet::v1::worker_service_server::WorkerService for Ends { + async fn heartbeat(&self, _: tonic::Request, + ) -> Result, tonic::Status> { + Err(tonic::Status::unimplemented("not in this test")) + } + async fn end_session(&self, request: tonic::Request, + ) -> Result, tonic::Status> { + let token = request.metadata().get("authorization").and_then(|v| v.to_str().ok()).unwrap_or("").to_owned(); + let r = request.into_inner(); + self.0.lock().await.push((r.worker_id, r.session, token)); + Ok(tonic::Response::new(fleet_proto::fleet::v1::WorkerEndSessionResponse {})) + } + } + + #[tokio::test] + async fn a_stopping_agent_ends_its_own_session_with_its_token() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let ends = Arc::new(Mutex::new(Vec::new())); + let incoming = tonic::transport::server::TcpIncoming::from_listener(listener, true, None).unwrap(); + tokio::spawn(tonic::transport::Server::builder() + .add_service(fleet_proto::fleet::v1::worker_service_server::WorkerServiceServer::new(Ends(ends.clone()))) + .serve_with_incoming(incoming)); + let endpoint = Endpoint::from_shared(format!("http://{address}")).unwrap(); + end_session(endpoint, "Bearer secret".parse().unwrap(), "gpu-1", "session-1").await; + assert_eq!(*ends.lock().await, [("gpu-1".to_string(), "session-1".to_string(), "Bearer secret".to_string())]); + } + + #[tokio::test] + async fn an_agent_stops_promptly_when_the_control_plane_is_out_of_reach() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + drop(listener); // nothing listens there + let endpoint = Endpoint::from_shared(format!("http://{address}")).unwrap().connect_timeout(Duration::from_secs(5)); + let started = Instant::now(); + end_session(endpoint, "Bearer secret".parse().unwrap(), "gpu-1", "session-1").await; + assert!(started.elapsed() < Duration::from_secs(6), "{:?}", started.elapsed()); + } } diff --git a/crates/fleet-proto/proto/fleet/model/v1/model.proto b/crates/fleet-proto/proto/fleet/model/v1/model.proto index 5816de3..7fcbc80 100644 --- a/crates/fleet-proto/proto/fleet/model/v1/model.proto +++ b/crates/fleet-proto/proto/fleet/model/v1/model.proto @@ -168,6 +168,7 @@ message Job { required bool quota_held = 11; optional string stop = 12; required bool logs_ready = 13; + // Purged by an earlier release: hidden, and gone with its batch. required bool purged = 14; } diff --git a/crates/fleet-proto/proto/fleet/v1/fleet.proto b/crates/fleet-proto/proto/fleet/v1/fleet.proto index 9d5d400..c756453 100644 --- a/crates/fleet-proto/proto/fleet/v1/fleet.proto +++ b/crates/fleet-proto/proto/fleet/v1/fleet.proto @@ -124,6 +124,8 @@ message DeregisterResponse {} // Machine agents use their own local Ray instance. No Ray cluster crosses machines. service WorkerService { rpc Heartbeat(WorkerHeartbeatRequest) returns (WorkerHeartbeatResponse); + // A stopping agent ends its session, so the agent that replaces it is admitted at once. + rpc EndSession(WorkerEndSessionRequest) returns (WorkerEndSessionResponse); } message NodeCommand { @@ -148,6 +150,14 @@ message WorkerHeartbeatRequest { fleet.model.v1.NodeReport report = 5; } +// Only the session itself can end: another agent's session stays until it is quiet. +message WorkerEndSessionRequest { + string worker_id = 1; + string session = 2; +} + +message WorkerEndSessionResponse {} + message WorkerHeartbeatResponse { uint32 heartbeat_secs = 1; repeated NodeCommand commands = 2; diff --git a/crates/fleet-proto/proto/fleet/v1/types.proto b/crates/fleet-proto/proto/fleet/v1/types.proto index 00f8cd4..35baa29 100644 --- a/crates/fleet-proto/proto/fleet/v1/types.proto +++ b/crates/fleet-proto/proto/fleet/v1/types.proto @@ -55,6 +55,7 @@ enum NodeOperation { // Start or stop an app's replicas as the node's apps.yaml entry configures them // (runtime only; NODE_OPERATION_APP writes that configuration). NODE_OPERATION_SET_APP = 4; + // Sent by releases that purged a finished job; deleting its batch removes it now. NODE_OPERATION_PURGE_JOB = 5; reserved 6; reserved "NODE_OPERATION_STAGE_MODEL"; @@ -62,5 +63,9 @@ enum NodeOperation { // configuration (enabled state, resources, settings). Parameters: {"app": {namespace, // namespace_name, name}} or {"batch": {name}} with {"remove": true} or {"set": {...}}, or // {"entry": {name}} with {"set": {...}} for the one apps.yaml entry a change of an app stops. + // A deletion's removal, with {"delete": {...}}, also deletes what the node holds of it: a + // batch's jobs' directories ("jobs"); an app's replicas' logs, its workers' directories and + // its proof jobs' "files", but nothing that holds or lies in one of the paths to "keep". A + // removal without it, which takes an app off one node, keeps that. NODE_OPERATION_APP = 7; } diff --git a/crates/fleet-proto/src/workload.rs b/crates/fleet-proto/src/workload.rs index a2840b2..413720a 100644 --- a/crates/fleet-proto/src/workload.rs +++ b/crates/fleet-proto/src/workload.rs @@ -468,6 +468,18 @@ impl CommandRecord { pub fn removes_app(&self, app: &str) -> bool { self.removal().is_some_and(|(_, name)| name == app) } + /// The app or batch an app message removes, by name. + pub fn removes(&self) -> Option<&str> { + let p = &self.parameters; + (self.is(NodeOperation::App) && p["remove"] == json!(true)) + .then(|| p["app"]["name"].as_str().or(p["batch"]["name"].as_str())) + .flatten() + } + /// The app or batch an app message deletes, by name: it is removed with what the node + /// holds of it. A removal that only takes it off one node keeps that. + pub fn deletes(&self) -> Option<&str> { + self.removes().filter(|_| self.parameters.get("delete").is_some()) + } /// In its result: nothing of what it stopped is left running. pub fn cleaned(&self) -> bool { self.result["cleaned"].as_bool() == Some(true) @@ -503,7 +515,7 @@ pub mod command { json!({"id": job.id, "lease": job.lease, "namespace": job.namespace, "tag": job.tag, "owner": job.owner, "batch": job.batch, "policy": job.policy, "request": job.request, "submitted_at": job.status["submitted_at"]}) } - /// Cancels, preempts or purges a job's execution `lease`. + /// Cancels or preempts a job's execution `lease`. pub fn job(id: &str, lease: &str) -> Value { json!({"id": id, "lease": lease}) } diff --git a/crates/fleet-scheduler/src/fleet.rs b/crates/fleet-scheduler/src/fleet.rs index f533a3f..db27b12 100644 --- a/crates/fleet-scheduler/src/fleet.rs +++ b/crates/fleet-scheduler/src/fleet.rs @@ -754,11 +754,12 @@ impl Scheduler { }).collect()) } - /// Deletes an app, its terminal job records and their inputs and outputs, and its - /// anchor. Removes the ELF only when no other app or job uses it. Refused while any of - /// its jobs is queued or running. + /// Deletes an app with every job record of it, and its anchor. Refused while any of its + /// jobs is queued or running. Also returns the files its jobs left in its nodes' app + /// storage, for them to remove: each job's output directory, and the inputs and ELF no + /// other job or app uses. pub async fn delete_app(&self, namespace: &str, name: &str, - ) -> anyhow::Result { + ) -> anyhow::Result<(serde_json::Value, Vec)> { let _ops = self.app_ops.write().await; let apps = self.store.apps().await?; let app = apps @@ -782,12 +783,12 @@ impl Scheduler { app.active.max(live) ))); } - let ids = self.store.delete_app(namespace, name).await?; + let jobs = self.store.delete_app(namespace, name).await?; self.apps.lock().remove(&(namespace.to_string(), name.to_string())); { let mut inner = self.inner.lock(); - for id in &ids { - inner.jobs.remove(id); + for j in &jobs { + inner.jobs.remove(&j.id); } if let Some(node) = inner.anchors.remove(&(namespace.to_string(), name.to_string())) { if inner.reserved.get(&node) == Some(&placement::app_key(namespace, name)) { @@ -795,8 +796,23 @@ impl Scheduler { } } } - tracing::info!(app = %name, jobs = ids.len(), "app deleted"); - Ok(serde_json::json!({"app": name, "jobs_deleted": ids.len()})) + let elf_shared = apps.iter().any(|a| a.elf_id == app.elf_id && !(a.namespace == namespace && a.name == name)); + let inputs: Vec<&fleet_proto::fleet::v1::ProofInputs> = jobs.iter().filter_map(|j| j.inputs.as_ref()).collect(); + let mut shared: Vec = inputs.iter() + .flat_map(|i| [i.input.as_ref(), i.elf.as_ref().filter(|_| !elf_shared)]) + .flatten() + .map(|r| r.uri.clone()) + .collect(); + shared.sort(); + shared.dedup(); + let used = self.store.referenced(&shared).await?; + let files = jobs.iter() + .filter_map(|j| Some(format!("{}/{}", j.inputs.as_ref()?.output_root.trim_end_matches('/'), j.id))) + .chain(shared.into_iter().filter(|uri| !used.contains(uri))) + .filter_map(|uri| uri.strip_prefix("file://").map(str::to_owned)) + .collect(); + tracing::info!(app = %name, jobs = jobs.len(), "app deleted"); + Ok((serde_json::json!({"app": name, "jobs_deleted": jobs.len()}), files)) } /// The newest jobs (the store has every one) that the filter selects. diff --git a/crates/fleet-scheduler/src/http.rs b/crates/fleet-scheduler/src/http.rs index a37de79..c049745 100644 --- a/crates/fleet-scheduler/src/http.rs +++ b/crates/fleet-scheduler/src/http.rs @@ -55,7 +55,7 @@ pub fn router(state: HttpState) -> Router { /// The permission an API route needs. Reading apps, batches, their jobs and the caller's own /// namespace needs read; submitting a job (or cancelling a batch job) submit; changing an app -/// or batch, or purging its jobs, manage; everything else (namespaces and their quotas, +/// or batch manage; everything else (namespaces and their quotas, /// tokens, allocations, preemption, nodes, the fleet's status) admin. pub fn required(method: &Method, path: &str) -> Permission { let parts: Vec<&str> = path.trim_matches('/').split('/').collect(); diff --git a/crates/fleet-scheduler/src/store.rs b/crates/fleet-scheduler/src/store.rs index 5fff6a1..2aaa4c2 100644 --- a/crates/fleet-scheduler/src/store.rs +++ b/crates/fleet-scheduler/src/store.rs @@ -1,6 +1,6 @@ //! ProofJob metadata. A proof job references its inputs and outputs in app-owned storage. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use async_trait::async_trait; use parking_lot::Mutex; @@ -39,8 +39,10 @@ pub trait Store: Send + Sync { async fn recent(&self, f: &ProofJobFilter, limit: usize) -> anyhow::Result>; /// Every registered app with its job counts, most recently used first. async fn apps(&self) -> anyhow::Result>; - /// Deletes an app and its finished jobs; returns the jobs' ids. - async fn delete_app(&self, namespace: &str, name: &str) -> anyhow::Result>; + /// Deletes an app and its finished jobs; returns the jobs. + async fn delete_app(&self, namespace: &str, name: &str) -> anyhow::Result>; + /// Which of `uris` a job's input or ELF still references. + async fn referenced(&self, uris: &[String]) -> anyhow::Result>; /// Registers an app; false when its name or program is one of the namespace's already. async fn add_app(&self, app: &App) -> anyhow::Result; async fn set_app_priority(&self, namespace: &str, name: &str, priority: i32) -> anyhow::Result<()>; @@ -153,18 +155,26 @@ impl Store for MemoryStore { Ok(v) } - async fn delete_app(&self, namespace: &str, name: &str) -> anyhow::Result> { + async fn delete_app(&self, namespace: &str, name: &str) -> anyhow::Result> { let mut jobs = self.jobs.lock(); let ids: Vec = jobs .values() .filter(|j| j.namespace == namespace && j.app == name && j.state().is_terminal()) .map(|j| j.id.clone()) .collect(); - for id in &ids { - jobs.remove(id); - } + let gone = ids.iter().filter_map(|id| jobs.remove(id)).collect(); self.apps.lock().remove(&(namespace.to_string(), name.to_string())); - Ok(ids) + Ok(gone) + } + + async fn referenced(&self, uris: &[String]) -> anyhow::Result> { + Ok(self.jobs.lock().values() + .filter_map(|j| j.inputs.as_ref()) + .flat_map(|i| [i.input.as_ref(), i.elf.as_ref()]) + .flatten() + .filter(|r| uris.contains(&r.uri)) + .map(|r| r.uri.clone()) + .collect()) } async fn add_app(&self, app: &App) -> anyhow::Result { @@ -659,17 +669,29 @@ impl Store for PgStore { .collect() } - async fn delete_app(&self, namespace: &str, name: &str) -> anyhow::Result> { + async fn delete_app(&self, namespace: &str, name: &str) -> anyhow::Result> { let rows = sqlx::query( - "WITH gone AS (DELETE FROM fleet_jobs WHERE namespace = $1 AND app = $2 AND state IN ('succeeded', 'failed') RETURNING id), \ + "WITH gone AS (DELETE FROM fleet_jobs WHERE namespace = $1 AND app = $2 AND state IN ('succeeded', 'failed') RETURNING *), \ app AS (DELETE FROM fleet_apps WHERE namespace = $1 AND name = $2) \ - SELECT id FROM gone", + SELECT * FROM gone", ) .bind(namespace) .bind(name) .fetch_all(&self.pool) .await?; - rows.iter().map(|r| Ok(r.try_get("id")?)).collect() + rows.iter().map(Self::row_to_job).collect() + } + + async fn referenced(&self, uris: &[String]) -> anyhow::Result> { + let rows: Vec = sqlx::query_scalar( + "SELECT DISTINCT uri FROM fleet_jobs, \ + LATERAL (VALUES (inputs->'input'->>'uri'), (inputs->'elf'->>'uri')) AS r(uri) \ + WHERE uri = ANY($1)", + ) + .bind(uris) + .fetch_all(&self.pool) + .await?; + Ok(rows.into_iter().collect()) } async fn add_app(&self, a: &App) -> anyhow::Result { diff --git a/crates/fleet-scheduler/src/token.rs b/crates/fleet-scheduler/src/token.rs index cc4237d..9381ef5 100644 --- a/crates/fleet-scheduler/src/token.rs +++ b/crates/fleet-scheduler/src/token.rs @@ -15,7 +15,7 @@ pub enum Permission { Read, /// Submit proof and batch jobs; cancel a batch job. Submit, - /// Register, delete, update, scale, enable and disable apps and batches; purge jobs. + /// Register, delete, update, scale, enable and disable apps and batches. Manage, /// Namespaces, quotas, tokens, allocations, preemption and nodes, in any namespace. Admin, diff --git a/crates/fleet-scheduler/src/workloads.rs b/crates/fleet-scheduler/src/workloads.rs index cfc4f8d..8930fe1 100644 --- a/crates/fleet-scheduler/src/workloads.rs +++ b/crates/fleet-scheduler/src/workloads.rs @@ -16,8 +16,8 @@ use axum::{ Json, Router, }; use fleet_proto::fleet::v1::{ - worker_service_server::WorkerService, NodeCommand, NodeOperation, WorkerHeartbeatRequest, - WorkerHeartbeatResponse, + worker_service_server::WorkerService, NodeCommand, NodeOperation, WorkerEndSessionRequest, + WorkerEndSessionResponse, WorkerHeartbeatRequest, WorkerHeartbeatResponse, }; use parking_lot::Mutex; use serde::Serialize; @@ -48,6 +48,41 @@ fn fail(code: u16, message: impl Into) -> anyhow::Error { message: message.into(), }) } +/// The files of the job records and commands a deletion dropped. +type RecordFiles = (Vec, Vec); + +/// Whether command `c` changes app or batch `name` of `namespace`, or one of its entries. +fn about(c: &CommandRecord, namespace: &str, name: &str) -> bool { + let p = &c.parameters; + if c.is(NodeOperation::SetApp) { + return c.entry().app() == name && c.entry().namespace() == namespace; + } + c.is(NodeOperation::App) + && (p["batch"]["name"] == name + || p["app"]["name"] == name && p["app"]["namespace"] == namespace + || c.created().is_some_and(|e| e.app() == name && e.namespace() == namespace)) +} + +/// An outcome of removing a file, gone already counting as done. +fn gone(result: std::io::Result<()>) -> std::io::Result<()> { + match result { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(e), + _ => Ok(()), + } +} + +/// Removes deleted job records with their logs, and deleted commands. +fn remove_records(dir: &Path, jobs: &[String], commands: &[String]) -> anyhow::Result<()> { + for id in jobs { + gone(fs::remove_file(dir.join("jobs").join(format!("{id}.json"))))?; + gone(fs::remove_dir_all(dir.join("artifacts").join(id)))?; + } + for id in commands { + gone(fs::remove_file(dir.join("commands").join(format!("{id}.json"))))?; + } + Ok(()) +} + fn response_error(e: anyhow::Error) -> Response { if e.downcast_ref::().is_some() { return (StatusCode::BAD_REQUEST, "invalid JSON request").into_response(); @@ -95,6 +130,9 @@ struct NodeSession { seen: f64, report: NodeReport, draining: bool, + /// Its agent stopped and said so: the next agent's session takes over at once, without + /// waiting for this one to go quiet. The node counts as connected until then, as before. + ended: bool, } #[derive(Default)] struct WorkloadState { @@ -143,14 +181,16 @@ impl WorkloadController { .insert(format!("{}-{}", v.worker, v.entry().name()), v); } fleet.set_worker_owned_apps(worker_owned(&inner)); - Ok(Self { + let controller = Self { fleet, cluster, dir, inner: Mutex::new(inner), ops: tokio::sync::Mutex::new(()), notify: tokio::sync::Notify::new(), - }) + }; + controller.forget_deleted()?; + Ok(controller) } pub(crate) fn fleet(&self) -> &Arc { &self.fleet @@ -241,11 +281,158 @@ impl WorkloadController { workers.into_iter().map(|worker| self.command(worker, NodeOperation::App, message.clone())).collect() } - /// Tells every node that `app` was deleted from `namespace`; each removes the entries it - /// configures for it. - pub(crate) fn remove_app(&self, namespace: &str, app: &str) -> anyhow::Result<()> { - let workers: Vec = self.inner.lock().workers.keys().cloned().collect(); - self.tell_nodes(&workers, self.app_message(namespace, app), message::remove())?; + /// Tells every node that `app` was deleted from `namespace`. Each removes the entries it + /// configures for it with what they hold on it, and `files`, which the app's proof jobs + /// left in its storage, but keeps what the nodes' other apps and batches use. Returns the + /// nodes told. + pub(crate) fn remove_app(&self, namespace: &str, app: &str, files: &[String]) -> anyhow::Result> { + // Its nodes, and every other connected since the scheduler started: one that is offline + // gets the message when it is back. + let workers: Vec = { + let i = self.inner.lock(); + let mine = i.apps.get(app).filter(|a| a.namespace == namespace).map(|a| a.workers.clone()).unwrap_or_default(); + i.workers.keys().cloned().chain(mine).collect::>().into_iter().collect() + }; + let change = json!({"remove": true, "delete": {"keep": self.paths_in_use(app), "files": files}}); + self.tell_nodes(&workers, self.app_message(namespace, app), change)?; + Ok(workers) + } + + /// The host paths that the apps and batches its nodes report use, but `name`'s. Deleting + /// `name` keeps them on every node, and what holds or lies in one of them. + fn paths_in_use(&self, name: &str) -> Vec { + let i = self.inner.lock(); + let mut paths = std::collections::BTreeSet::new(); + for w in i.workers.values() { + for e in w.report.entries().filter(|e| e.app() != name) { + for key in ["workdir", "worker_workdir", "proof_storage_root", "worker_env_file"] { + paths.extend(e.0[key].as_str().map(str::to_owned)); + } + for volume in e.0["volumes"].as_array().into_iter().flatten().filter_map(Value::as_str) { + paths.insert(volume.split(':').next().unwrap_or(volume).to_owned()); + } + } + for (_, spec) in w.report.batches.iter().filter(|(batch, _)| batch.as_str() != name) { + paths.extend(spec["allow"].as_array().into_iter().flatten().filter_map(Value::as_str).map(str::to_owned)); + } + } + paths.into_iter().collect() + } + + /// Deletes app or batch `name` of `namespace`, once nothing of it runs. What the control + /// host holds of it goes at once: its job records with their logs, and the commands about + /// them and it. Its own record stays, marked deleted, until its nodes have removed it + /// (`forget_deleted`), so that a node report sent before then can't bring it back. Each + /// node removes its configuration and what it holds of it: a batch's job directories; an + /// app's replicas, their logs, its workers' directories and `files`, which its proof jobs + /// left in its storage. Returns what it deleted and the nodes it told, and the files of + /// the records, to remove once the operation lock is released. + fn delete(&self, namespace: &str, name: &str, files: Vec) -> anyhow::Result<(Value, RecordFiles)> { + let template = self.inner.lock().apps.get(name).filter(|a| a.namespace == namespace).cloned(); + let batch = template.as_ref().is_some_and(|a| !a.persistent); + if let Some(mut t) = template.clone() { + t.removed = true; + self.save("apps", name, &t)?; + self.inner.lock().apps.insert(name.into(), t); + } + // Its deployments go once their nodes report them no more, which needs them disabled. + let enabled: Vec<(String, AppDeployment)> = self.inner.lock().deployments.iter() + .filter(|(_, d)| !d.disabled && d.entry().app() == name && d.entry().namespace() == namespace) + .map(|(key, d)| (key.clone(), d.clone())) + .collect(); + for (key, mut d) in enabled { + d.disabled = true; + self.save("deployments", &key, &d)?; + self.inner.lock().deployments.insert(key, d); + } + let (jobs, commands, shown, on) = { + let mut i = self.inner.lock(); + let jobs: Vec = i.jobs.values().filter(|r| r.batch == name).map(|r| r.id.clone()).collect(); + let ids: HashSet<&str> = jobs.iter().map(String::as_str).collect(); + let commands: Vec = i.commands.values() + .filter(|c| c.job_id().is_some_and(|id| ids.contains(id)) || c.done && about(c, namespace, name)) + .map(|c| c.id.clone()) + .collect(); + // Where each job's directory is; an earlier release's purge removed it already. + let (mut shown, mut on) = (0, BTreeMap::>::new()); + for id in &jobs { + let r = i.jobs.remove(id).expect("listed above"); + if !r.purged { + shown += 1; + if let Some(worker) = r.worker { + on.entry(worker).or_default().push(r.id); + } + } + } + for id in &commands { + i.commands.remove(id); + } + (jobs, commands, shown, on) + }; + let nodes = if batch { + let mut workers: std::collections::BTreeSet = template.iter().flat_map(|t| t.workers.clone()).collect(); + workers.extend(on.keys().cloned()); + for worker in &workers { + let jobs = on.get(worker).cloned().unwrap_or_default(); + let change = json!({"remove": true, "delete": {"jobs": jobs}}); + self.command(worker, NodeOperation::App, message::with(message::batch(name), &change))?; + } + workers.into_iter().collect() + } else { + self.remove_app(namespace, name, &files)? + }; + tracing::info!(name, jobs = shown, ?nodes, "deleted; its nodes remove what they hold of it"); + Ok((json!({"name": name, "removed": true, "jobs_deleted": shown, "nodes": nodes}), (jobs, commands))) + } + + /// Removes the files of records `delete` dropped. + async fn remove_files(&self, (jobs, commands): RecordFiles) -> anyhow::Result<()> { + let dir = self.dir.clone(); + tokio::task::spawn_blocking(move || remove_records(&dir, &jobs, &commands)).await? + } + + /// Forgets the deleted apps and batches that their nodes have removed: the record of each, + /// and the commands about it. One that a node failed to remove stays until it is deleted + /// again, and so does a batch deleted by an earlier release, while the job records that + /// release left are there (it goes with the ones that release purged). + fn forget_deleted(&self) -> anyhow::Result<()> { + let (names, jobs, commands) = { + let mut i = self.inner.lock(); + let deleted: Vec<(String, String)> = i.apps.values().filter(|a| a.removed) + .map(|a| (a.namespace.clone(), a.name.clone())).collect(); + if deleted.is_empty() { + return Ok(()); + } + let shown: HashSet<&str> = i.jobs.values().filter(|r| !r.purged).map(|r| r.batch.as_str()).collect(); + let gone: Vec<(String, String)> = deleted.into_iter() + .filter(|(namespace, name)| !shown.contains(name.as_str()) + && !i.commands.values().any(|c| about(c, namespace, name) && (!c.done || c.deletes().is_some() && c.error.is_some()))) + .collect(); + let names: HashSet<&str> = gone.iter().map(|(_, name)| name.as_str()).collect(); + let jobs: Vec = i.jobs.values().filter(|r| names.contains(r.batch.as_str())).map(|r| r.id.clone()).collect(); + let ids: HashSet<&str> = jobs.iter().map(String::as_str).collect(); + let commands: Vec = i.commands.values() + .filter(|c| c.job_id().is_some_and(|id| ids.contains(id)) || gone.iter().any(|(namespace, name)| about(c, namespace, name))) + .map(|c| c.id.clone()) + .collect(); + for id in &jobs { + i.jobs.remove(id); + } + for id in &commands { + i.commands.remove(id); + } + for (_, name) in &gone { + i.apps.remove(name); + } + (gone.into_iter().map(|(_, name)| name).collect::>(), jobs, commands) + }; + for name in &names { + gone(fs::remove_file(self.dir.join("apps").join(format!("{name}.json"))))?; + } + remove_records(&self.dir, &jobs, &commands)?; + if !names.is_empty() { + tracing::info!(?names, "deleted apps and batches forgotten: their nodes removed them"); + } Ok(()) } @@ -496,6 +683,48 @@ impl WorkloadController { keys.iter().filter(|k| i.deployments.get(*k).and_then(|d| i.workers.get(&d.worker)).is_some_and(|w| now() - w.seen < 30.0)).cloned().collect() } + /// What of app or batch `app` may still run, each with how to stop it; nothing when it can + /// be deleted. Deleting an app or a batch is the same operation: it is refused while any + /// of its work runs or may run. A batch's work is its jobs, queued or running. An app's + /// is its replicas: it is disabled first, and every node that configures it confirms them + /// stopped. An offline node can't, so it holds the delete until it's back. + fn still_running(&self, app: &WorkloadTemplate) -> Vec { + let name = app.name.as_str(); + let i = self.inner.lock(); + let mut running = Vec::new(); + let jobs = i.jobs.open().filter(|r| r.batch == name && (!terminal(r.state()) || r.quota_held)).count(); + if jobs > 0 { + running.push(format!("{jobs} job(s) queued or running: cancel them or let them finish")); + } + if !app.persistent { + return running; + } + // A deleted app's nodes stop it as they remove it. + if !app.disabled() && !app.removed { + running.push(format!("it is enabled: disable it first (fleetctl app disable {name})")); + return running; + } + for worker in &app.workers { + match i.workers.get(worker).filter(|w| now() - w.seen < 30.0) { + None => running.push(format!("node {worker} is offline, so it can't confirm its replicas stopped")), + // A node that no longer configures it has nothing of it left. + Some(w) => if w.report.entries().any(|e| e.name() == name && !e.cleaned()) { + running.push(format!("its replicas on node {worker} are still stopping")); + }, + } + } + running + } + + /// Refuses `name` while a deleted app or batch of that name is not gone yet. + pub(super) fn ensure_name_free(&self, name: &str) -> anyhow::Result<()> { + if self.inner.lock().apps.get(name).is_some_and(|a| a.removed) { + return Err(fail(409, format!("{name} is deleted, but not gone yet: its nodes are still removing it, \ + or it keeps job records from an earlier release. Delete it again to finish, then reuse the name"))); + } + Ok(()) + } + /// Whether a batch still has job records (not purged). fn has_jobs(&self, batch: &str) -> bool { self.inner.lock().jobs.values().any(|r| r.batch == batch && !r.purged) @@ -622,6 +851,18 @@ impl WorkloadController { self.notify.notify_waiters(); Ok(v) } + /// Ends `worker`'s agent session `session`, which its agent sends as it stops: the agent + /// that replaces it is admitted at once. Another session than the current one changes + /// nothing, so one agent can't end another's. + pub(crate) fn end_session(&self, worker: &str, session: &str) -> bool { + let mut i = self.inner.lock(); + let Some(w) = i.workers.get_mut(worker).filter(|w| w.session == session) else { + return false; + }; + w.ended = true; + tracing::info!(worker, "agent session ended; the next agent session takes over at once"); + true + } async fn report( &self, worker_id: &str, @@ -677,7 +918,7 @@ impl WorkloadController { let mut i = self.inner.lock(); if i.workers .get(worker_id) - .is_some_and(|w| w.session != session && now() - w.seen < 30.0) + .is_some_and(|w| w.session != session && !w.ended && now() - w.seen < 30.0) { return Err(fail(409, "another agent session is active for this worker")); } @@ -692,6 +933,7 @@ impl WorkloadController { seen: now(), report: report.clone(), draining, + ended: false, }, ); } @@ -752,6 +994,11 @@ impl WorkloadController { }; let key = format!("{worker_id}-{name}"); let existing = { self.inner.lock().deployments.get(&key).cloned() }; + // An entry of a deleted app, which its node is about to remove. + if existing.is_none() && self.inner.lock().apps.get(observed.app()) + .is_some_and(|a| a.removed && a.namespace == namespace.id) { + continue; + } let intent = if let Some(mut intent) = existing { // The app an entry runs, its program and its launch settings are the node's own // configuration, not resource-scale overrides. @@ -928,7 +1175,7 @@ impl WorkloadController { self.save("jobs", &r.id, &r)?; self.inner.lock().jobs.insert(r.id.clone(), r); } - let mut acknowledged = vec![]; + let (mut acknowledged, mut removals) = (vec![], false); for result in &report.results { let Some(mut c) = self.inner.lock().commands.get(&result.id).cloned() else { acknowledged.push(result.id.clone()); @@ -1019,11 +1266,24 @@ impl WorkloadController { } } } + if let Some(name) = c.deletes() { + removals = true; + let kept = c.result["kept"].as_array().filter(|k| !k.is_empty()); + if c.error.is_some() || kept.is_some() { + tracing::warn!(worker = worker_id, name, error = ?c.error, ?kept, + "a node did not remove all of a deleted app or batch; delete it again to retry"); + } else { + tracing::info!(worker = worker_id, name, deleted = %c.result["deleted"], "a node removed a deleted app or batch"); + } + } self.save("commands", &c.id, &c)?; self.inner.lock().commands.insert(c.id.clone(), c); } acknowledged.push(result.id.clone()); } + if removals { + self.forget_deleted()?; + } self.notify.notify_waiters(); Ok(acknowledged) } @@ -1064,26 +1324,6 @@ impl WorkloadController { (w.report.free_cpus - pending.1 - starting.1).max(0.0), )) } - fn purge(&self, mut r: Job) -> anyhow::Result { - if !terminal(r.state()) || r.quota_held { - return Err(fail(409, "wait for confirmed cleanup before purging")); - } - r.purged = true; - self.save("jobs", &r.id, &r)?; - self.inner.lock().jobs.insert(r.id.clone(), r.clone()); - if let Some(worker) = &r.worker { - self.command( - worker, - NodeOperation::PurgeJob, - message::job(&r.id, &r.lease), - )?; - } - let path = self.dir.join("artifacts").join(&r.id); - if path.exists() { - fs::remove_dir_all(path)?; - } - Ok(json!({"id":r.id,"purged":true})) - } async fn stop_job(&self, mut r: Job, preempt: bool) -> anyhow::Result { if terminal(r.state()) && (preempt || !r.quota_held) { return Err(fail(409, "job already finished")); @@ -1531,27 +1771,11 @@ impl WorkloadController { } } -/// The commands one heartbeat hands `worker`: at most 16, purges after everything else. A -/// purge only drops a finished job's node record, so thousands of them, as a bulk purge of -/// history queues, must not hold back the launches, stops and app changes behind them. +/// The commands one heartbeat hands `worker`: at most 16. fn heartbeat_commands(i: &WorkloadState, worker: &str) -> Vec { - const LIMIT: usize = 16; - let (mut work, mut purges) = (vec![], vec![]); - for c in i.commands.open().filter(|c| c.worker == worker && !c.done && c.admitted) { - if c.is(NodeOperation::PurgeJob) { - if purges.len() < LIMIT { - purges.push(c); - } - } else { - work.push(c); - if work.len() == LIMIT { - break; - } - } - } - work.into_iter() - .chain(purges) - .take(LIMIT) + i.commands.open() + .filter(|c| c.worker == worker && !c.done && c.admitted) + .take(16) .map(|c| NodeCommand { id: c.id.clone(), operation: c.operation, parameters: Some(c.parameters.clone()) }) .collect() } @@ -1559,6 +1783,14 @@ fn heartbeat_commands(i: &WorkloadState, worker: &str) -> Vec { pub struct NodeControlService(pub Arc); #[tonic::async_trait] impl WorkerService for NodeControlService { + async fn end_session( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let r = request.into_inner(); + self.0.end_session(&r.worker_id, &r.session); + Ok(tonic::Response::new(WorkerEndSessionResponse {})) + } async fn heartbeat( &self, request: tonic::Request, @@ -1898,7 +2130,8 @@ async fn handle_result(s: Arc, request: Request) -> anyhow:: body["namespace"] = json!(namespace); body.as_object_mut().unwrap().remove("name"); let _ops = s.ops.lock().await; - if s.inner.lock().apps.get(&name).is_some_and(|a| !a.removed) || s.fleet.app_known(&namespace, &name) { + s.ensure_name_free(&name)?; + if s.inner.lock().apps.contains_key(&name) || s.fleet.app_known(&namespace, &name) { return Err(fail(409, format!("an app or batch named {name} exists"))); } let app = WorkloadTemplate { @@ -1917,8 +2150,10 @@ async fn handle_result(s: Arc, request: Request) -> anyhow:: } if parts.first() == Some(&"apps") && parts.len() >= 2 { let name = parts[1]; - // A deleted batch stays readable while it has job records: they are its history. - let readable_deleted = batch && method == Method::GET && parts.len() == 2 && s.has_jobs(name); + // A batch an earlier release deleted stays readable while it has job records; deleting + // a deleted app or batch again removes those and retries its nodes' removal. + let readable_deleted = batch && method == Method::GET && parts.len() == 2 && s.has_jobs(name) + || method == Method::DELETE && parts.len() == 2; let mut app = s .inner .lock() @@ -1937,36 +2172,17 @@ async fn handle_result(s: Arc, request: Request) -> anyhow:: return Ok(Json(view(&s, &app)).into_response()); } if method == Method::DELETE && parts.len() == 2 { - let _ops = s.ops.lock().await; + let ops = s.ops.lock().await; s.ensure_not_scaling(&app.namespace, name)?; - let persistent_ready_to_remove = !app.persistent - || (app.disabled() - && app.workers.iter().all(|id| { - s.inner.lock().workers.get(id).is_some_and(|w| { - now() - w.seen < 30.0 && w.report.entries().any(|v| v.name() == name && v.cleaned()) - }) - })); - if !persistent_ready_to_remove - || s.inner - .lock() - .jobs - .open() - .any(|r| r.batch == name && (!terminal(r.state()) || r.quota_held)) - { - return Err(fail( - 409, - "disable the app or finish/cancel jobs before deleting", - )); - } - app.removed = true; - s.save("apps", name, &app)?; - s.inner.lock().apps.insert(name.into(), app.clone()); - if app.persistent { - s.remove_app(&app.namespace, name)?; - } else { - s.tell_nodes(&app.workers, json!({"batch":{"name":name}}), json!({"remove":true}))?; + let running = s.still_running(&app); + if !running.is_empty() { + let kind = if batch { "batch" } else { "app" }; + return Err(fail(409, format!("{kind} {name} still runs: {}; delete it once nothing of it does", running.join("; ")))); } - return Ok(Json(json!({"name":name,"removed":true})).into_response()); + let (deleted, files) = s.delete(&app.namespace, name, vec![])?; + drop(ops); + s.remove_files(files).await?; + return Ok(Json(deleted).into_response()); } if batch && parts.len() == 3 && parts[2] == "settings" && method == Method::PUT { let set: Value = serde_json::from_slice(&to_bytes(request.into_body(), 1 << 20).await?)?; @@ -2073,37 +2289,6 @@ async fn handle_result(s: Arc, request: Request) -> anyhow:: ) .into_response()); } - if method == Method::DELETE && parts.len() == 3 { - let days = q - .get("older_than") - .and_then(|s| s.parse::().ok()) - .filter(|v| v.is_finite() && *v >= 0.0) - .ok_or_else(|| fail(400, "older_than must be non-negative days"))?; - let cutoff = now() - days * 86400.0; - let purgeable = |r: &Job| { - !r.purged - && r.batch == batch - && terminal(r.state()) - && !r.quota_held - && r.finished_at().is_some_and(|v| v < cutoff) - && s.visible(&r.namespace, r.tag.as_deref(), c, &q) - }; - let ids: Vec = s.inner.lock().jobs.values().filter(|r| purgeable(r)).map(|r| r.id.clone()).collect(); - // A purge writes a record and a node command per job, so a bulk one of history can - // take minutes. Holding the operation lock for a chunk at a time lets node reports - // and dispatch through; each job is checked again under the lock. - let mut purged = Vec::with_capacity(ids.len()); - for chunk in ids.chunks(64) { - let _ops = s.ops.lock().await; - for id in chunk { - let r = s.inner.lock().jobs.get(id).filter(|r| purgeable(r)).cloned(); - if let Some(r) = r { - purged.push(s.purge(r)?); - } - } - } - return Ok(Json(purged).into_response()); - } if method == Method::POST && parts.len() == 3 { let body = serde_json::from_slice(&to_bytes(request.into_body(), 1 << 20).await?)?; return Ok((StatusCode::CREATED, Json(s.submit(batch, body, c, &q).await?)).into_response()); @@ -2126,11 +2311,6 @@ async fn handle_result(s: Arc, request: Request) -> anyhow:: let r = job(&s)?; return Ok(Json(s.stop_job(r, preempt).await?.value()).into_response()); } - if method == Method::DELETE && parts.len() == 4 { - let _ops = s.ops.lock().await; - let r = job(&s)?; - return Ok(Json(vec![s.purge(r)?]).into_response()); - } if method == Method::GET && parts.len() == 5 && parts[4] == "logs" { return log_response(&s, &r, &q).await; } @@ -2246,15 +2426,12 @@ impl WorkloadController { match (method, rest) { (&Method::GET, []) => Ok(Json(self.stored_proof_app(&namespace, &name, tag.as_deref()).await?).into_response()), (&Method::DELETE, []) => { - let _guard = self.app_lifecycle_guard(&namespace, &name).await?; - let deleted = self.fleet.delete_app(&namespace, &name).await?; - let template = self.inner.lock().apps.get(&name).filter(|a| a.namespace == namespace).cloned(); - if let Some(mut template) = template { - template.removed = true; - self.save("apps", &name, &template)?; - self.inner.lock().apps.insert(name.clone(), template); - } - self.remove_app(&namespace, &name)?; + let guard = self.app_lifecycle_guard(&namespace, &name).await?; + let (mut deleted, files) = self.fleet.delete_app(&namespace, &name).await?; + let (removed, records) = self.delete(&namespace, &name, files)?; + drop(guard); + self.remove_files(records).await?; + deleted["nodes"] = removed["nodes"].clone(); Ok(Json(deleted).into_response()) } (&Method::PUT, ["priority"]) => { diff --git a/crates/fleet-scheduler/src/workloads_records.rs b/crates/fleet-scheduler/src/workloads_records.rs index 3cfc45c..d71c911 100644 --- a/crates/fleet-scheduler/src/workloads_records.rs +++ b/crates/fleet-scheduler/src/workloads_records.rs @@ -72,6 +72,12 @@ impl Records { self.all.get(id) } + pub(super) fn remove(&mut self, id: &str) -> Option { + let old = self.all.remove(id)?; + unindex(&mut self.open, &mut self.about, id, &old); + Some(old) + } + /// Every record, history included, in id order. pub(super) fn values(&self) -> btree_map::Values<'_, String, T> { self.all.values() diff --git a/crates/fleet-scheduler/src/workloads_scale.rs b/crates/fleet-scheduler/src/workloads_scale.rs index e9c4e11..a367286 100644 --- a/crates/fleet-scheduler/src/workloads_scale.rs +++ b/crates/fleet-scheduler/src/workloads_scale.rs @@ -764,9 +764,10 @@ impl WorkloadController { WorkloadController::require_grant(c, &name)?; WorkloadController::require_admin_settings(c, &spec, &Self::ADMIN_CREATE_SETTINGS)?; let _ops = self.ops.lock().await; + self.ensure_name_free(&name)?; { let i = self.inner.lock(); - if i.apps.get(&name).is_some_and(|a| !a.removed) + if i.apps.contains_key(&name) || i.deployments.values().any(|d| d.entry().name() == name) || self.fleet.app_known(&namespace, &name) { diff --git a/crates/fleet-scheduler/src/workloads_tests.rs b/crates/fleet-scheduler/src/workloads_tests.rs index 125124d..22d0b4c 100644 --- a/crates/fleet-scheduler/src/workloads_tests.rs +++ b/crates/fleet-scheduler/src/workloads_tests.rs @@ -320,7 +320,7 @@ async fn batch_policy_and_job_lifecycle_use_batch_routes() { assert_eq!(scale_call(c.clone(), "admin", "DELETE", "/batches/training", Value::Null) .await.status(), StatusCode::OK); assert_eq!(scale_call(c.clone(), "admin", "GET", &format!("/batches/training/jobs/{}", text(&job,"id")), Value::Null) - .await.status(), StatusCode::OK); + .await.status(), StatusCode::NOT_FOUND, "its jobs go with it"); } #[tokio::test] @@ -465,30 +465,6 @@ async fn a_job_its_quota_cannot_take_waits_without_rewriting_its_record() { assert_eq!(fs::metadata(&path).unwrap().ino(), written, "a pass that cannot place it leaves its record alone"); assert_eq!(placed(&c), 2); } -#[tokio::test] -async fn a_backlog_of_purges_does_not_hold_back_a_launch() { - let (_dir, c, ns) = setup().await; - c.report("gpu", "session", inventory(&ns)).await.unwrap(); - // A bulk purge of history: more purges than one heartbeat carries, all ahead in id order. - for n in 0..20 { - let id = format!("0000-{n:02}"); - c.inner.lock().commands.insert(id.clone(), CommandRecord { - id, worker: "gpu".into(), operation: NodeOperation::PurgeJob as i32, - parameters: json!({"id": format!("old-{n}"), "lease": "lease"}).into(), - done: false, result: Value::Null.into(), error: None, admitted: true, - }); - } - let job = submit(&c).await; - c.dispatch().await.unwrap(); - let launch = c.inner.lock().commands.about(&job.id).next().unwrap().id.clone(); - - let sent = heartbeat_commands(&c.inner.lock(), "gpu"); - assert_eq!(sent.len(), 16); - assert_eq!(sent[0].id, launch, "the launch goes first"); - assert!(sent[1..].iter().all(|c| c.operation == NodeOperation::PurgeJob as i32)); - assert_eq!(sent[1].id, "0000-00", "purges keep their order"); - assert!(heartbeat_commands(&c.inner.lock(), "other").is_empty()); -} /// Everything the tests log, from a process-wide subscriber: a scoped one can miss events /// while tests on other threads run the same log lines. fn test_logs() -> &'static parking_lot::Mutex> { @@ -512,76 +488,22 @@ fn test_logs() -> &'static parking_lot::Mutex> { } #[tokio::test] -async fn a_bulk_purge_takes_every_finished_job_in_chunks_and_records_who_asked() { +async fn a_change_is_on_record_with_who_asked_and_a_read_is_not() { let logs = test_logs(); let (_dir, c, ns) = setup().await; c.report("gpu", "session", inventory(&ns)).await.unwrap(); - let template = submit(&c).await; - // More finished jobs than one chunk holds, and one that finished just now. - for n in 0..150 { - let mut r = template.clone(); - r.id = format!("old-{n:03}"); - r.worker = Some("gpu".into()); - r.status = json!({"state":"succeeded","submitted_at":1,"finished_at":(now() - 7200.0)}).into(); - c.inner.lock().jobs.insert(r.id.clone(), r); - } - let mut recent = template.clone(); - recent.id = "recent".into(); - recent.status = json!({"state":"succeeded","submitted_at":1,"finished_at":now()}).into(); - c.inner.lock().jobs.insert(recent.id.clone(), recent); - - let response = scale_call(c.clone(), "admin", "DELETE", "/batches/training/jobs?older_than=0.0517", json!(null)).await; - assert_eq!(response.status(), StatusCode::OK); - let body: Value = serde_json::from_slice(&to_bytes(response.into_body(), 1 << 20).await.unwrap()).unwrap(); - assert_eq!(body.as_array().unwrap().len(), 150); - { - let i = c.inner.lock(); - assert!((0..150).all(|n| i.jobs[&format!("old-{n:03}")].purged)); - assert!(!i.jobs["recent"].purged && !i.jobs[&template.id].purged); - let purges: HashSet<&str> = i.commands.open() - .filter(|c| c.worker == "gpu" && c.operation == NodeOperation::PurgeJob as i32) - .filter_map(|c| c.parameters["id"].as_str()) - .collect(); - assert_eq!(purges.len(), 150, "each purged job's node drops its copy"); - } - - // The request is on record with its caller; a read is not. - scale_call(c.clone(), "admin", "GET", "/batches/training/jobs", json!(null)).await; + let uri = format!("/batches/training?namespace={ns}"); + assert_eq!(scale_call(c.clone(), "admin", "DELETE", &uri, json!(null)).await.status(), StatusCode::OK); + scale_call(c.clone(), "admin", "GET", "/batches", json!(null)).await; let logs = String::from_utf8(logs.lock().clone()).unwrap(); - let lines: Vec<&str> = logs.lines().filter(|l| l.contains("api request") && l.contains("older_than=0.0517")).collect(); + let lines: Vec<&str> = logs.lines().filter(|l| l.contains("api request") && l.contains(&format!("uri={uri}"))).collect(); assert_eq!(lines.len(), 1, "{logs}"); assert!(!logs.lines().any(|l| l.contains("api request") && l.contains("method=GET")), "reads are not logged"); - for part in ["method=DELETE", "uri=/batches/training/jobs?older_than=0.0517", "status=200", &format!("token=\"{}\"", "ad".repeat(16)), "label=\"admin\""] { + for part in ["method=DELETE", "status=200", &format!("token=\"{}\"", "ad".repeat(16)), "label=\"admin\""] { assert!(lines[0].contains(part), "{part} in {}", lines[0]); } assert!(!lines[0].contains(ADMIN), "the token itself is never logged"); } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn a_bulk_purge_lets_node_reports_in_between_its_chunks() { - let (_dir, c, ns) = setup().await; - c.report("gpu", "session", inventory(&ns)).await.unwrap(); - let template = submit(&c).await; - for n in 0..400 { - let mut r = template.clone(); - r.id = format!("old-{n:03}"); - r.worker = Some("gpu".into()); - r.status = json!({"state":"succeeded","submitted_at":1,"finished_at":1}).into(); - c.inner.lock().jobs.insert(r.id.clone(), r); - } - let purged = |c: &WorkloadController| c.inner.lock().jobs.values().filter(|r| r.purged).count(); - let purge = tokio::spawn(scale_call(c.clone(), "admin", "DELETE", "/batches/training/jobs?older_than=0", json!(null))); - while purged(&c) == 0 { - tokio::time::sleep(Duration::from_millis(1)).await; - } - // What a node report waits for: it gets the lock before the purge has finished. - let during = { - let _ops = c.ops.lock().await; - purged(&c) - }; - assert_eq!(purge.await.unwrap().status(), StatusCode::OK); - assert!(during < 400, "the report waited for the whole purge"); - assert_eq!(purged(&c), 400); -} #[tokio::test] async fn offline_worker_and_terminal_without_cleanup_keep_reservations() { let (_dir, c, ns) = setup().await; @@ -608,7 +530,7 @@ async fn offline_worker_and_terminal_without_cleanup_keep_reservations() { assert!(c.fleet.allocations(&ns).is_empty()); } #[tokio::test] -async fn stale_lease_is_ignored_and_purged_history_cannot_be_imported_again() { +async fn stale_lease_is_ignored_and_a_purged_record_is_not_imported_again() { let (_dir, c, ns) = setup().await; c.report("gpu", "session", inventory(&ns)).await.unwrap(); let job = submit(&c).await; @@ -622,8 +544,8 @@ async fn stale_lease_is_ignored_and_purged_history_cannot_be_imported_again() { assert_eq!(c.fleet.allocations(&ns).len(), 1); report.jobs[0] = incoming(&job, "succeeded", true); c.report("gpu", "session", report.clone()).await.unwrap(); - c.purge(c.job(&job.id, None, &BTreeMap::new()).unwrap()) - .unwrap(); + // A record an earlier release purged: its node's copy may be reported still. + c.inner.lock().jobs.update(&job.id, |r| r.purged = true); c.report("gpu", "session", report).await.unwrap(); assert!(c.job(&job.id, None, &BTreeMap::new()).is_err()); } @@ -2052,9 +1974,9 @@ async fn deleting_a_registered_app_tells_every_node_which_app_was_deleted() { c.report("gpu","session",inventory(&ns)).await.unwrap(); c.report("cpu","session",inventory(&ns)).await.unwrap(); let app=PROOF.to_string(); - c.remove_app(&ns,&app).unwrap(); - c.remove_app(&ns,&app).unwrap(); - let expected=json!({"app":{"namespace":ns,"namespace_name":"test","name":app},"remove":true}); + c.remove_app(&ns,&app,&[]).unwrap(); + c.remove_app(&ns,&app,&[]).unwrap(); + let expected=json!({"app":{"namespace":ns,"namespace_name":"test","name":app},"remove":true,"delete":{"keep":[],"files":[]}}); assert_eq!(remove_commands(&c,NodeOperation::App),vec![("cpu".to_string(),expected.clone()),("gpu".to_string(),expected)]); } @@ -2064,11 +1986,20 @@ async fn deleting_an_app_tells_its_nodes() { c.report("gpu","session",scale_inventory(&ns,false,true)).await.unwrap(); c.inner.lock().apps.get_mut("model").unwrap().spec["disabled"]=json!(true); assert_eq!(scale_call(c.clone(),"admin","DELETE",&format!("/apps/model?namespace={ns}"),Value::Null).await.status(),StatusCode::OK); - assert_eq!(remove_commands(&c,NodeOperation::App),vec![("gpu".to_string(),json!({"app":{"namespace":ns,"namespace_name":"test","name":"model"},"remove":true}))]); + assert_eq!(remove_commands(&c,NodeOperation::App),vec![("gpu".to_string(), + json!({"app":{"namespace":ns,"namespace_name":"test","name":"model"},"remove":true,"delete":{"keep":[],"files":[]}}))]); } +/// The body of a response, as JSON. +async fn body(response: Response) -> Value { + serde_json::from_slice(&to_bytes(response.into_body(), 1 << 20).await.unwrap()).unwrap() +} + +/// Deleting a batch deletes its job records with their logs, and the commands about them, at once, +/// and tells each node which of its jobs' directories to remove. Its own record stays until its node +/// has removed it: a report before then brings back neither it nor its jobs, and its name stays taken. #[tokio::test] -async fn deleting_a_batch_sends_remove_batch_and_keeps_its_jobs() { +async fn deleting_a_batch_deletes_its_jobs_and_goes_once_its_node_removed_it() { let (_dir,c,ns)=setup().await; c.report("gpu","session",inventory(&ns)).await.unwrap(); let job=submit(&c).await; @@ -2077,13 +2008,171 @@ async fn deleting_a_batch_sends_remove_batch_and_keeps_its_jobs() { let mut report=inventory(&ns); report.jobs.push(incoming(&job,"succeeded",true)); c.report("gpu","session",report).await.unwrap(); - assert_eq!(scale_call(c.clone(),"admin","DELETE",&format!("/batches/training?namespace={ns}"),Value::Null).await.status(),StatusCode::OK); - assert_eq!(remove_commands(&c,NodeOperation::App),vec![("gpu".to_string(),json!({"batch":{"name":"training"},"remove":true}))]); - assert!(c.job(&job.id,None,&BTreeMap::new()).is_ok(),"job history and outputs follow the job's own lifecycle"); + let logs=c.dir.join("artifacts").join(&job.id).join(&job.lease).join("logs"); + fs::create_dir_all(&logs).unwrap(); + let launch=c.inner.lock().commands.about(&job.id).next().unwrap().id.clone(); + let deleted=scale_call(c.clone(),"admin","DELETE",&format!("/batches/training?namespace={ns}"),Value::Null).await; + assert_eq!(deleted.status(),StatusCode::OK); + let deleted=body(deleted).await; + assert_eq!((&deleted["jobs_deleted"],&deleted["nodes"]),(&json!(1),&json!(["gpu"])),"{deleted}"); + assert_eq!(remove_commands(&c,NodeOperation::App),vec![("gpu".to_string(),json!({"batch":{"name":"training"},"remove":true,"delete":{"jobs":[job.id]}}))]); + assert!(c.job(&job.id,None,&BTreeMap::new()).is_err()); + assert!(!c.dir.join("jobs").join(format!("{}.json",job.id)).exists() && !c.dir.join("artifacts").join(&job.id).exists()); + assert!(c.inner.lock().commands.get(&launch).is_none() && !c.dir.join("commands").join(format!("{launch}.json")).exists()); + let mut late=incoming(&job,"succeeded",true);late.id="other-job".into(); - let mut report=inventory(&ns);report.jobs.push(late); + let mut report=inventory(&ns);report.jobs.extend([incoming(&job,"succeeded",true),late]); + c.report("gpu","session",report).await.unwrap(); + assert!(c.job(&job.id,None,&BTreeMap::new()).is_err() && c.job("other-job",None,&BTreeMap::new()).is_err(),"a deleted batch imports no jobs"); + assert_eq!(scale_call(c.clone(),"admin","GET",&format!("/batches/training?namespace={ns}"),Value::Null).await.status(),StatusCode::NOT_FOUND); + let batches=format!("/batches?namespace={ns}"); + let create=|c: Arc| scale_call(c,"admin","POST",&batches,json!({"name":"training","image":"test"})); + let refused=create(c.clone()).await; + assert_eq!(refused.status(),StatusCode::CONFLICT); + assert!(String::from_utf8_lossy(&to_bytes(refused.into_body(),1<<20).await.unwrap()).contains("training is deleted, but not gone yet")); + + let removal=c.inner.lock().commands.open().find(|v|v.removes()==Some("training")).unwrap().id.clone(); + let mut report=inventory(&ns);report.batches.clear(); + report.results=vec![CommandResult{id:removal.clone(),result:json!({"removed":["training"],"jobs_deleted":1,"kept":[]}).into(),error:None}]; + c.report("gpu","session",report).await.unwrap(); + assert!(!c.inner.lock().apps.contains_key("training") && !c.dir.join("apps/training.json").exists(),"gone once its node removed it"); + assert!(c.inner.lock().commands.get(&removal).is_none() && !c.dir.join("commands").join(format!("{removal}.json")).exists()); + assert_eq!(create(c.clone()).await.status(),StatusCode::CREATED,"and its name is free"); +} + +/// A node that fails to remove a deleted batch leaves its record there; deleting it again retries. +#[tokio::test] +async fn a_deleted_batch_a_node_failed_to_remove_is_deleted_again() { + let (_dir,c,ns)=setup().await; + c.report("gpu","session",inventory(&ns)).await.unwrap(); + let path=format!("/batches/training?namespace={ns}"); + assert_eq!(scale_call(c.clone(),"admin","DELETE",&path,Value::Null).await.status(),StatusCode::OK); + let removal=c.inner.lock().commands.open().find(|v|v.removes()==Some("training")).unwrap().id.clone(); + let mut report=inventory(&ns); + report.results=vec![CommandResult{id:removal.clone(),result:json!({"cleaned":false}).into(),error:Some("batches.yaml: not a mapping".into())}]; c.report("gpu","session",report).await.unwrap(); - assert!(c.job("other-job",None,&BTreeMap::new()).is_err(),"a deleted batch imports no jobs"); + assert!(c.inner.lock().apps["training"].removed,"stays while not removed"); + assert_eq!(scale_call(c.clone(),"admin","DELETE",&path,Value::Null).await.status(),StatusCode::OK); + let retry=c.inner.lock().commands.open().find(|v|v.removes()==Some("training")).unwrap().id.clone(); + assert_ne!(retry,removal); + assert!(c.inner.lock().commands.get(&removal).is_none(),"the failed one goes with the retry"); + let mut report=inventory(&ns);report.batches.clear(); + report.results=vec![CommandResult{id:retry,result:json!({"removed":["training"],"jobs_deleted":0,"kept":[]}).into(),error:None}]; + c.report("gpu","session",report).await.unwrap(); + assert!(!c.inner.lock().apps.contains_key("training")); +} + +/// A batch an earlier release deleted kept its job records: it stays listed as deleted until it is +/// deleted again, which deletes them. One whose jobs that release purged goes at the next start. +#[tokio::test] +async fn a_batch_an_earlier_release_deleted_goes_with_its_jobs() { + let (_dir,c,ns)=setup().await; + c.report("gpu","session",inventory(&ns)).await.unwrap(); + assert_eq!(scale_call(c.clone(),"admin","POST",&format!("/batches?namespace={ns}"),json!({"name":"render","image":"test"})).await.status(),StatusCode::CREATED); + let kept=submit(&c).await; + let purged=c.submit("render",json!({"command":["true"],"gpus":1,"cpus":1}),None,&BTreeMap::new()).await.unwrap(); + let purged=c.job(text(&purged,"id"),None,&BTreeMap::new()).unwrap(); + for (mut r,flag) in [(kept.clone(),false),(purged.clone(),true)] { + r.status=json!({"state":"succeeded","submitted_at":1,"finished_at":2}).into(); + r.quota_held=false; + r.purged=flag; + c.save("jobs",&r.id,&r).unwrap(); + } + for name in ["training","render"] { + let mut t=c.inner.lock().apps[name].clone(); + t.removed=true; + c.save("apps",name,&t).unwrap(); + } + let c=Arc::new(WorkloadController::new(c.fleet.clone(),"cluster".into()).unwrap()); + assert!(!c.inner.lock().apps.contains_key("render") && !c.dir.join("apps/render.json").exists()); + assert!(!c.inner.lock().jobs.values().any(|r|r.id==purged.id) && !c.dir.join("jobs").join(format!("{}.json",purged.id)).exists()); + let listed=body(scale_call(c.clone(),"admin","GET",&format!("/batches?namespace={ns}"),Value::Null).await).await; + assert_eq!(listed.as_array().unwrap().iter().map(|b|(b["name"].clone(),b["state"].clone())).collect::>(),vec![(json!("training"),json!("deleted"))]); + let deleted=body(scale_call(c.clone(),"admin","DELETE",&format!("/batches/training?namespace={ns}"),Value::Null).await).await; + assert_eq!(deleted["jobs_deleted"],1,"{deleted}"); + assert!(!c.dir.join("jobs").join(format!("{}.json",kept.id)).exists()); + assert_eq!(body(scale_call(c.clone(),"admin","GET",&format!("/batches?namespace={ns}"),Value::Null).await).await,json!([])); +} + +/// Deleting a proof app deletes its jobs and tells every node which files they left in its storage: +/// each job's output directory, and the inputs and ELF that no other job or app uses. +#[tokio::test] +async fn deleting_a_proof_app_tells_its_nodes_the_files_its_jobs_left() { + use crate::store::{ProofJob,ProofJobState}; + use fleet_proto::fleet::v1::{ArtifactRef,ProofInputs,ProofKind}; + use fleet_proto::stage::v1::Status as StageStatus; + let (_dir,c,ns)=setup().await; + let other_elf="c".repeat(64); + register(&c,&ns,PROOF,&proof_elf()).await; + register(&c,&ns,"other",&other_elf).await; + let second=c.fleet.create_namespace("second",Quota{gpus:Some(1.0),cpus:Some(1.0)}).await.unwrap().id; + register(&c,&second,"twin",&proof_elf()).await; + c.report("gpu","session",inventory(&ns)).await.unwrap(); + let file=|path:&str,sha:&str| ArtifactRef{uri:format!("file:///store/{path}"),sha256:sha.into(),size:1}; + for (id,app,elf,input) in [("p1",PROOF,proof_elf(),"inputs/a"),("p2",PROOF,proof_elf(),"inputs/shared"),("o1","other",other_elf.clone(),"inputs/shared")] { + c.fleet.store.upsert(&ProofJob{ + inputs:Some(ProofInputs{input:Some(file(input,&"1".repeat(64))),elf:Some(file(&format!("elf/{elf}"),&elf)),output_root:"file:///store/outputs/".into()}), + outputs:None,id:id.into(),app:app.into(),owner:"b".repeat(40),namespace:ns.clone(),tag:String::new(),client_id:id.into(),elf_id:elf.clone(), + kind:ProofKind::Compressed as i32,priority:0,state:ProofJobState::Failed as i32,fail_status:Some(StageStatus::InternalError as i32), + error:String::new(),node:None,attempts:1,failed_on:vec![],cycles:0,prove_ms:0,created_ms:1,started_ms:Some(1),finished_ms:Some(2), + }).await.unwrap(); + } + let files=|c: &WorkloadController, app: &str| { + let i=c.inner.lock(); + let removal=i.commands.open().find(|v|v.removes()==Some(app)).expect("removal"); + let mut files: Vec=removal.parameters["delete"]["files"].as_array().unwrap().iter().map(|f|f.as_str().unwrap().to_owned()).collect(); + files.sort(); + files + }; + let deleted=body(scale_call(c.clone(),"admin","DELETE",&format!("/apps/other?namespace={ns}"),Value::Null).await).await; + assert_eq!((&deleted["jobs_deleted"],&deleted["nodes"]),(&json!(1),&json!(["gpu"])),"{deleted}"); + assert_eq!(files(&c,"other"),vec![format!("/store/elf/{other_elf}"),"/store/outputs/o1".into()],"its ELF is its own; the input is p2's too"); + let deleted=body(scale_call(c.clone(),"admin","DELETE",&format!("/apps/{PROOF}?namespace={ns}"),Value::Null).await).await; + assert_eq!(deleted["jobs_deleted"],2,"{deleted}"); + assert_eq!(files(&c,PROOF),vec!["/store/inputs/a","/store/inputs/shared","/store/outputs/p1","/store/outputs/p2"],"twin proves its ELF"); +} + +/// Deleting an app and deleting a batch are one operation: refused while any of its work +/// runs or may run, with what that is and how to stop it. A batch's work is its jobs; an app's +/// is its replicas, which a node that no longer configures it doesn't have. +#[tokio::test] +async fn deleting_an_app_or_a_batch_is_refused_while_any_of_it_runs() { + let (_dir, c, ns) = setup().await; + let refusal = |c: Arc, path: String| async move { + let response = scale_call(c, "admin", "DELETE", &path, Value::Null).await; + let status = response.status(); + (status, String::from_utf8_lossy(&to_bytes(response.into_body(), 1 << 20).await.unwrap()).into_owned()) + }; + let (batch, app) = (format!("/batches/training?namespace={ns}"), format!("/apps/model?namespace={ns}")); + + c.report("gpu", "session", scale_inventory(&ns, false, false)).await.unwrap(); + let job = submit(&c).await; + let (status, body) = refusal(c.clone(), batch.clone()).await; + assert_eq!(status, StatusCode::CONFLICT); + assert!(body.contains("batch training still runs: 1 job(s) queued or running: cancel them or let them finish"), "{body}"); + let (status, body) = refusal(c.clone(), app.clone()).await; + assert_eq!(status, StatusCode::CONFLICT); + assert!(body.contains("app model still runs: it is enabled: disable it first (fleetctl app disable model)"), "{body}"); + + c.inner.lock().apps.get_mut("model").unwrap().spec["disabled"] = json!(true); + let (_, body) = refusal(c.clone(), app.clone()).await; + assert!(body.contains("its replicas on node gpu are still stopping"), "{body}"); + c.inner.lock().workers.get_mut("gpu").unwrap().seen = 0.0; + let (_, body) = refusal(c.clone(), app.clone()).await; + assert!(body.contains("node gpu is offline, so it can't confirm its replicas stopped"), "{body}"); + + c.inner.lock().jobs.update(&job.id, |j| j.status["state"] = json!("cancelled")); + assert_eq!(refusal(c.clone(), batch).await.0, StatusCode::OK); + + // A node the app was on that no longer configures it holds nothing of it: only a node + // that configures it does, until it has stopped it. + let (_dir, c, ns) = setup().await; + c.report("gpu", "session", scale_inventory(&ns, false, true)).await.unwrap(); + c.report("cpu", "session", inventory(&ns)).await.unwrap(); + c.inner.lock().apps.get_mut("model").unwrap().workers.push("cpu".into()); + c.inner.lock().apps.get_mut("model").unwrap().spec["disabled"] = json!(true); + let (status, body) = refusal(c.clone(), format!("/apps/model?namespace={ns}")).await; + assert_eq!(status, StatusCode::OK, "{body}"); } fn command_where(c: &WorkloadController, pick: impl Fn(&CommandRecord) -> bool) -> Option { diff --git a/crates/fleet-scheduler/tests/flow.rs b/crates/fleet-scheduler/tests/flow.rs index a8c6c37..ec8bd8f 100644 --- a/crates/fleet-scheduler/tests/flow.rs +++ b/crates/fleet-scheduler/tests/flow.rs @@ -602,8 +602,9 @@ async fn an_app_is_deleted_with_its_jobs_once_none_is_running() { let gone: serde_json::Value = serde_json::from_str(&gone).unwrap(); assert_eq!((gone["app"].as_str(), gone["jobs_deleted"].as_u64()), (Some(app_of(b"elf-d").as_str()), Some(1)), "{gone}"); assert_eq!(get_job(&h, USER, &app_of(b"elf-d"), "job-d").await.0, 404, "the deleted app's job is gone"); - let again = admin_curl(&h, &["-o", "/dev/null", "-w", "%{http_code}", "-X", "DELETE", &url]); - assert_eq!(again, "404"); + // Until its nodes have removed it, deleting it again retries that; nothing else is left. + let again: serde_json::Value = serde_json::from_str(&admin_curl(&h, &["-X", "DELETE", &url])).unwrap(); + assert_eq!((again["removed"].as_bool(), again["jobs_deleted"].as_u64()), (Some(true), Some(0)), "{again}"); } /// The API through curl as an administrator (what fleetctl does), as text. @@ -1474,6 +1475,42 @@ async fn machine_heartbeat_requires_cluster_token_and_populates_control_plane_no assert!(status["nodes"].as_array().unwrap().iter().any(|n| n["name"]=="gpu-1" && n["backend"]=="ray-local")); } +/// One agent session at a time drives a worker. An agent that stops ends its own session, and +/// the agent that replaces it is admitted at once; no one else can end it. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_agent_that_ends_its_session_hands_its_worker_to_the_next_at_once() { + use fleet_proto::fleet::v1::{worker_service_client::WorkerServiceClient, WorkerEndSessionRequest, + WorkerHeartbeatRequest, + }; + let h = start().await; + let channel = Channel::from_shared(format!("http://127.0.0.1:{}", h.control_port)).unwrap().connect().await.unwrap(); + let mut agent = WorkerServiceClient::with_interceptor(channel.clone(), + fleet_scheduler::token::BearerTokenInterceptor::new(TOKEN).unwrap()); + let report = serde_json::from_value::(serde_json::json!({ + "version": 3, "machine": "gpu-machine", "ready": true, + "total_gpus": 8, "total_cpus": 48, "free_gpus": 8, "free_cpus": 48, + })).unwrap(); + let heartbeat = |session: &str| WorkerHeartbeatRequest { worker_id: "gpu-1".into(), session: session.into(), + report: Some(report.clone()), proof_fetch_ready: false, + }; + let end = |session: &str| WorkerEndSessionRequest { worker_id: "gpu-1".into(), session: session.into() }; + agent.heartbeat(heartbeat("old")).await.unwrap(); + let refused = agent.heartbeat(heartbeat("new")).await.unwrap_err(); + assert_eq!(refused.code(), tonic::Code::FailedPrecondition); + assert!(refused.message().contains("another agent session is active"), "{}", refused.message()); + // Another session than the active one ends nothing, and the call needs the cluster token. + agent.end_session(end("new")).await.unwrap(); + assert_eq!(agent.heartbeat(heartbeat("new")).await.unwrap_err().code(), tonic::Code::FailedPrecondition); + let mut anonymous = WorkerServiceClient::new(channel); + assert_eq!(anonymous.end_session(end("old")).await.unwrap_err().code(), tonic::Code::Unauthenticated); + assert_eq!(agent.heartbeat(heartbeat("new")).await.unwrap_err().code(), tonic::Code::FailedPrecondition); + + agent.end_session(end("old")).await.unwrap(); + agent.heartbeat(heartbeat("new")).await.unwrap(); + assert_eq!(agent.heartbeat(heartbeat("old")).await.unwrap_err().code(), tonic::Code::FailedPrecondition, + "the ended session doesn't come back while the new one is active"); +} + #[tokio::test(flavor = "multi_thread")] async fn integrated_app_worker_rejects_sdk_jobs_in_its_own_namespace() { use fleet_proto::fleet::v1::{worker_service_client::WorkerServiceClient, WorkerHeartbeatRequest, diff --git a/crates/fleet-scheduler/tests/pg.rs b/crates/fleet-scheduler/tests/pg.rs index 3fd1cb5..5379d22 100644 --- a/crates/fleet-scheduler/tests/pg.rs +++ b/crates/fleet-scheduler/tests/pg.rs @@ -160,6 +160,22 @@ async fn check_registered_app_inventory(store: &dyn Store) { assert_eq!((registered.jobs, registered.failed, registered.created_ms), (1, 1, 3000)); store.delete_app(SYSTEM_NAMESPACE, &rejected.app).await.unwrap(); assert!(!store.apps().await.unwrap().iter().any(|a| a.name == rejected.app)); + check_referenced_files(store).await; +} + +/// A job's input and ELF are the files `referenced` names; its outputs and other files aren't. +async fn check_referenced_files(store: &dyn Store) { + use fleet_proto::fleet::v1::{ArtifactRef, ProofInputs}; + let file = |path: &str, sha: &str| ArtifactRef { uri: format!("file:///store/{path}"), sha256: sha.into(), size: 1 }; + let mut j = job("with-files", ProofJobState::Failed); + j.inputs = Some(ProofInputs { input: Some(file("inputs/i1", &"1".repeat(64))), elf: Some(file("elf/e", &j.elf_id)), + output_root: "file:///store/outputs".into() }); + store.upsert(&j).await.unwrap(); + let asked = ["inputs/i1", "elf/e", "inputs/i2", "outputs"].map(|p| format!("file:///store/{p}")); + let mut found: Vec = store.referenced(&asked).await.unwrap().into_iter().collect(); + found.sort(); + assert_eq!(found, vec!["file:///store/elf/e", "file:///store/inputs/i1"]); + assert!(store.referenced(&[]).await.unwrap().is_empty()); } #[tokio::test] @@ -212,7 +228,7 @@ async fn fresh_schema_requires_complete_job_identity() { assert!(store.apps().await.unwrap().iter().find(|a| a.name == "goat").unwrap().disabled); store.set_app_disabled(SYSTEM_NAMESPACE, "goat", false, 7).await.unwrap(); assert!(!store.apps().await.unwrap().iter().find(|a| a.name == "goat").unwrap().disabled); - assert_eq!(store.delete_app(SYSTEM_NAMESPACE, "goat").await.unwrap(), vec!["g2".to_string()], "the running one stays"); + assert_eq!(store.delete_app(SYSTEM_NAMESPACE, "goat").await.unwrap().iter().map(|j| j.id.as_str()).collect::>(), vec!["g2"], "the running one stays"); assert!(!store.apps().await.unwrap().iter().any(|a| a.name == "goat"), "the app is gone"); assert_eq!(store.incomplete().await.unwrap().iter().filter(|j| j.id == "g1").count(), 1); } diff --git a/crates/fleetctl/src/main.rs b/crates/fleetctl/src/main.rs index 1a2e9ff..8975c64 100644 --- a/crates/fleetctl/src/main.rs +++ b/crates/fleetctl/src/main.rs @@ -134,10 +134,12 @@ enum AppCmd { /// Select a machine agent when the app runs on several nodes. #[arg(long)] node: Option, }, - /// Delete an app, refused while any of its jobs is queued or running. A proof app goes - /// with its jobs' records, inputs and proofs, its anchor, and its ELF when no other app - /// uses it; a running app must be disabled first. - /// Without --yes it only says what it would delete. + /// Delete an app and everything of it, like batch delete: its configuration and replicas on + /// its nodes, their logs and its workers' directories; a proof app's job records, with their + /// inputs and proofs, its anchor, and its ELF unless another app uses it. Its volumes, env + /// files, storage roots and image stay, and so does what another app or batch uses. Refused + /// while any of it runs, so disable a running app first, and wait for a proof app's queued + /// or running jobs. Without --yes it only says what it would delete. Delete { name: String, #[arg(long)] @@ -447,15 +449,6 @@ enum BatchJobCmd { /// Stop a batch job as an app needing its cards would: it queues again (same id and /// out/, FLEET_ATTEMPT counting on) and spends no --retry. Preempt { id: String, #[arg(long)] batch: Option }, - /// Delete a finished batch job's record (out/ included), or every finished batch job older than --older-than days. - Purge { - id: Option, - /// Only this batch's jobs (with --older-than), or the job's batch (found from the id when omitted). - #[arg(long)] - batch: Option, - #[arg(long)] - older_than: Option, - }, } #[derive(Subcommand)] @@ -553,7 +546,10 @@ enum BatchCmd { /// K=V every batch job gets; repeat. Replaces the batch's environment. #[arg(long)] env: Vec, }, - /// Delete a batch after all accepted jobs finish; its job history stays until purged. + /// Delete a batch and everything of it, like app delete: its configuration on its nodes, + /// and its jobs' records, logs and outputs on the control host and its nodes. Its mounts + /// stay. Refused while any of its jobs is queued or running, so cancel them or let them + /// finish first. Without --yes it only says what it would delete. Delete { name: String, #[arg(long)] yes: bool }, #[command(flatten)] Job(BatchJobCmd), @@ -838,8 +834,6 @@ const NODE_COLS: &[out::TableColumn] = &[ ]; const TOKEN_COLS: &[out::TableColumn] = &[col("TOKEN ID", "id"), col("NAMESPACE","namespace"), col("TAG","tag"), col("PERMISSIONS", "permissions"), col("LABEL", "label"), col("STATE", "state"), col("CREATED", "created_ms")]; -const PURGE_COLS: &[out::TableColumn] = - &[col("RUN", "id"), col("PURGED", "purged"), col("STATE", "state"), col("SIZE", "bytes"), col("REASON", "reason")]; /// Tokens as rows: a STATE column (active / revoked); proof identity stays internal. fn token_rows(v: &Value) -> Vec { @@ -1209,33 +1203,51 @@ fn describe_app(name: &str, conn: &ConnectionOptions, out: &OutputWriter) -> any Ok(()) } +/// What a deletion's nodes still remove, after its record and what the control host held went. +fn removing(deleted: &Value) -> String { + let nodes: Vec<&str> = deleted["nodes"].as_array().into_iter().flatten().filter_map(Value::as_str).collect(); + if nodes.is_empty() { String::new() } else { format!("; its nodes remove what they hold of it: {}", nodes.join(", ")) } +} + +/// Deletes a batch (after showing what goes with it, until --yes), as `delete_app` deletes an app. +fn delete_batch(a: &FleetClient, name: &str, batch: &Value, yes: bool, out: &OutputWriter) -> anyhow::Result<()> { + let (total, active) = (&batch["jobs"]["total"], &batch["jobs"]["active"]); + if !yes { + bail!( + "would delete batch {name} with its {total} job(s): their records, logs and outputs, on the control \ + host and its nodes. Its mounts stay. Refused while any job is queued or running ({active} now). \ + Pass --yes to delete" + ); + } + let v = a.send(a.req(reqwest::Method::DELETE, &format!("/batches/{name}")))?; + out.done(&format!("deleted batch {name} and its {} job(s){}", v["jobs_deleted"], removing(&v)), v); + Ok(()) +} + /// Deletes an app (after showing what goes with it, until --yes). fn delete_app(name: &str, yes: bool, conn: &ConnectionOptions, out: &OutputWriter) -> anyhow::Result<()> { let a = FleetClient::new(conn)?; let app = app_named(&a, name)?; let (total, active) = (&app["jobs"]["total"], &app["jobs"]["active"]); let proof = app["proof"].is_object(); + let nodes = "its configuration and replicas on its nodes, their logs and its workers' directories"; if !yes { if proof { bail!( - "would delete app {name}: {total} job record(s) with their inputs and proofs, its anchor, and its ELF {} \ - unless another app uses it ({active} queued or running: refused while any is). Pass --yes to delete", + "would delete app {name}: {total} job record(s) with their inputs and proofs, {nodes}, its anchor, \ + and its ELF {} unless another app uses it. Its volumes, env files, storage roots and image stay \ + ({active} job(s) queued or running: refused while any is). Pass --yes to delete", app["proof"]["elf_id"].as_str().unwrap_or("") ); } bail!( - "would remove app {name}; its {total} past execution record(s) \ - stay ({active} queued or running: refused while any is). Pass --yes to delete" + "would delete app {name}: {nodes}. Its volumes, env files and image stay. Refused while it runs: \ + disable it first. Pass --yes to delete" ); } let v = a.send(a.req(reqwest::Method::DELETE, &format!("/apps/{name}")))?; - let text = if proof { - format!("deleted app {name}: {} job(s), ELF {}", v["jobs_deleted"], - if v["elf_removed"].as_bool() == Some(true) { "removed" } else { "kept (in use or absent)" }) - } else { - format!("deleted app {name}; its {total} past execution record(s) stay") - }; - out.done(&text, v); + let jobs = if proof { format!(" and its {} job(s)", v["jobs_deleted"]) } else { String::new() }; + out.done(&format!("deleted app {name}{jobs}{}", removing(&v)), v); Ok(()) } @@ -1338,17 +1350,14 @@ fn batch(p: &BatchArgs, out: &OutputWriter) -> anyhow::Result<()> { | BatchCmd::Priority { name, .. } | BatchCmd::Delete { name, .. } => name, _ => unreachable!(), }; - batch_named(&a, name)?; + let batch = batch_named(&a, name)?; let path = format!("/batches/{name}"); let request = match cmd { BatchCmd::Disable { .. } => a.req(reqwest::Method::PUT, &format!("{path}/disabled")), BatchCmd::Enable { .. } => a.req(reqwest::Method::DELETE, &format!("{path}/disabled")), BatchCmd::Priority { priority, .. } => a.req(reqwest::Method::PUT, &format!("{path}/priority")) .json(&json!({"priority": priority})), - BatchCmd::Delete { yes, .. } => { - if !yes { bail!("would remove batch {name}; accepted jobs must finish first. Pass --yes to delete"); } - a.req(reqwest::Method::DELETE, &path) - } + BatchCmd::Delete { yes, .. } => return delete_batch(&a, name, &batch, *yes, out), _ => unreachable!(), }; let value = a.send(request)?; @@ -1475,22 +1484,6 @@ fn batch_action(cmd: &BatchJobCmd, conn: &ConnectionOptions, out: &OutputWriter) let path = batch_job_path(&a, batch.as_deref(), id)?; out.record(&a.send(a.req(M::POST, &format!("{path}/preempt")))?) } - BatchJobCmd::Purge { id, batch, older_than } => { - let purged = match (id, older_than) { - (Some(id), _) => a.send(a.req(M::DELETE, &batch_job_path(&a, batch.as_deref(), id)?))?, - (None, Some(days)) => { - let batches = match batch { Some(batch) => vec![batch.clone()], None => batch_names(&a)? }; - let mut purged = Vec::new(); - for batch in batches { - let v = a.send(a.req(M::DELETE, &format!("/batches/{batch}/jobs?older_than={days}")))?; - purged.extend(v.as_array().cloned().unwrap_or_default()); - } - Value::Array(purged) - } - (None, None) => bail!("purge ID, or purge --older-than DAYS"), - }; - out.table(purged.as_array().map(Vec::as_slice).unwrap_or_default(), PURGE_COLS, Some(&purged)); - } BatchJobCmd::Logs { id, batch, follow, lines } => { let path = batch_job_path(&a, batch.as_deref(), id)?; let mut chunk = a.get(&format!("{path}/logs?lines={lines}"))?; diff --git a/crates/fleetctl/tests/apps.rs b/crates/fleetctl/tests/apps.rs index 98fd7bb..bc4904a 100644 --- a/crates/fleetctl/tests/apps.rs +++ b/crates/fleetctl/tests/apps.rs @@ -263,20 +263,49 @@ fn app_delete_says_what_goes_until_confirmed() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let url = format!("http://{}", listener.local_addr().unwrap()); let token = token_file("app-delete"); - let server = stub(listener, 3, move |line, _| (200, if line.starts_with("GET") { app.into() } else { r#"{"jobs_deleted":4,"elf_removed":true}"#.into() })); + let server = stub(listener, 3, move |line, _| (200, if line.starts_with("GET") { app.into() } else { r#"{"app":"ethproofs","jobs_deleted":4,"nodes":["gpu-a"]}"#.into() })); let refused = cli().args(["--namespace", "alice", "--url", &url, "--token-file"]).arg(&token) .args(["app", "delete", "ethproofs"]).output().unwrap(); assert!(!refused.status.success()); - assert!(String::from_utf8_lossy(&refused.stderr).contains("would delete app ethproofs: 4 job record(s)")); + let preview = String::from_utf8_lossy(&refused.stderr); + assert!(preview.contains("would delete app ethproofs: 4 job record(s) with their inputs and proofs, its configuration and \ + replicas on its nodes, their logs and its workers' directories, its anchor, and its ELF aaaa unless another app uses it. \ + Its volumes, env files, storage roots and image stay"), "{preview}"); let deleted = cli().args(["--namespace", "alice", "--url", &url, "--token-file"]).arg(&token) .args(["app", "delete", "ethproofs", "--yes"]).output().unwrap(); let seen = server.join().unwrap(); std::fs::remove_file(token).unwrap(); assert!(deleted.status.success(), "{}", String::from_utf8_lossy(&deleted.stderr)); - assert!(String::from_utf8_lossy(&deleted.stdout).contains("deleted app ethproofs: 4 job(s), ELF removed")); + let said = String::from_utf8_lossy(&deleted.stdout); + assert!(said.contains("deleted app ethproofs and its 4 job(s); its nodes remove what they hold of it: gpu-a"), "{said}"); assert!(seen[2].line.starts_with("DELETE /apps/ethproofs?namespace=alice"), "{}", seen[2].line); } +#[test] +fn batch_delete_says_what_goes_until_confirmed_like_app_delete() { + let batch = r#"{"type":"batch","name":"training","persistent":false,"jobs":{"total":5,"active":1}}"#; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let token = token_file("batch-delete"); + let server = stub(listener, 3, move |line, _| (200, if line.starts_with("GET") { batch.into() } else { + r#"{"name":"training","removed":true,"jobs_deleted":5,"nodes":["gpu-a","gpu-b"]}"#.into() })); + let refused = cli().args(["--namespace", "alice", "--url", &url, "--token-file"]).arg(&token) + .args(["batch", "delete", "training"]).output().unwrap(); + assert!(!refused.status.success()); + let preview = String::from_utf8_lossy(&refused.stderr); + assert!(preview.contains("would delete batch training with its 5 job(s): their records, logs and outputs, on the control host \ + and its nodes. Its mounts stay."), "{preview}"); + assert!(preview.contains("(1 now). Pass --yes to delete"), "{preview}"); + let deleted = cli().args(["--namespace", "alice", "--url", &url, "--token-file"]).arg(&token) + .args(["batch", "delete", "training", "--yes"]).output().unwrap(); + let seen = server.join().unwrap(); + std::fs::remove_file(token).unwrap(); + assert!(deleted.status.success(), "{}", String::from_utf8_lossy(&deleted.stderr)); + let said = String::from_utf8_lossy(&deleted.stdout); + assert!(said.contains("deleted batch training and its 5 job(s); its nodes remove what they hold of it: gpu-a, gpu-b"), "{said}"); + assert!(seen[2].line.starts_with("DELETE /batches/training?namespace=alice"), "{}", seen[2].line); +} + /// Two batches as `GET /batches` returns them: the app record's shape, with the policy under `batch`. fn batch_list() -> String { serde_json::json!([ diff --git a/deploy/fleet-ctl.sh b/deploy/fleet-ctl.sh index 0fb003b..27e0911 100755 --- a/deploy/fleet-ctl.sh +++ b/deploy/fleet-ctl.sh @@ -156,7 +156,9 @@ scheduler_rollback() { node_upgrade() { # node_upgrade FLEET_NODE BRIDGE_DIR [FLEET_PROOF_APP] [DEPLOY_DIR], or node upgrade RELEASE_TAG local bin=${1:-} bridge=${2:-} proof_app=${3:-} scripts=${4:-} - if [ -n "$bin" ] && [ ! -f "$bin" ]; then + # A release tag is a bare name. A path names local files, even ones a dry run never + # shipped to the node. + if [ -n "$bin" ] && [ ! -f "$bin" ] && [[ $bin != */* ]]; then local stage; stage=$(mktemp -d) prepare_release "$bin" "$stage" bin=$RELEASE_ROOT/bin/fleet-node diff --git a/deploy/tests/test_fleet_ctl.py b/deploy/tests/test_fleet_ctl.py index 51b00b0..fc8d66f 100644 --- a/deploy/tests/test_fleet_ctl.py +++ b/deploy/tests/test_fleet_ctl.py @@ -180,6 +180,25 @@ def test_watchdog_repairs_a_dead_ray_link(tmp_path): assert (tmp_path / "ray.log").exists(), "the repaired link let the watchdog start Ray" +def test_a_dry_run_on_the_node_takes_the_files_it_never_shipped_for_local_files(tmp_path): + """A remote dry run ships nothing, so the node sees paths to no files: it prints the local + upgrade of just those files, not a download of a release named after them.""" + prefix, commands = node_fixture(tmp_path) + executable(commands / "curl", f"echo called >> {tmp_path / 'curl.log'}\nexit 1\n") + stage = tmp_path / "fleet-ctl-stage" # never created + environment = {**os.environ, "PATH": f"{commands}:{os.environ['PATH']}", "FLEET_PREFIX": str(prefix), + "FLEET_CTL_ENV": str(tmp_path / "none.env"), "FLEET_CTL_ON_NODE": "1", "FLEET_NODE_SSH": ""} + result = subprocess.run( + ["bash", str(ROOT / "deploy/fleet-ctl.sh"), "-n", "node", "upgrade", f"{stage}/fleet-node", f"{stage}/bridge"], + env=environment, capture_output=True, text=True, timeout=20) + assert result.returncode == 0, result.stderr + assert not (tmp_path / "curl.log").exists() and "curl" not in result.stdout, result.stdout + assert f"+ install -m 755 {stage}/fleet-node {prefix}/bin/fleet-node.new" in result.stdout, result.stdout + assert f"+ cp -a {stage}/bridge/fleet_bridge/. {prefix}/bridge/fleet_bridge/" in result.stdout, result.stdout + assert "fleet-proof-app" not in result.stdout and "ray-start.sh" not in result.stdout, result.stdout + assert (prefix / "bin/fleet-node").read_text() == "#!/bin/bash\necho old-node\n", "a dry run changes nothing" + + def test_remote_upgrade_preserves_scripts_position_without_a_proof_app(tmp_path): prefix, commands = node_fixture(tmp_path) newbin = tmp_path / "fleet-node" diff --git a/docs/maintenance.md b/docs/maintenance.md index 1e73e69..8b4c4ac 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -65,6 +65,14 @@ Ray keeps running, and so do apps and batches, through the agent restart. The up touches neither `etc/` (watchdog.env, apps.yaml, batches.yaml, tokens) nor `work/`; leave app endpoints running when their programs are unchanged. +One agent session at a time drives a node. A stopping agent ends its session, so the +scheduler admits the restarted agent at once. An agent that crashed, or one stopped +while the scheduler was out of reach, leaves its session open: the next agent is +refused ("another agent session is active for this worker") until the open session +has been quiet for 30 seconds. Ending a session needs a scheduler at least as new as +the agent, so upgrade the control host first. A remote `-n` run ships nothing to the node, +and the node's part of it prints the steps for the files it would have received. + #### Cron-supervised nodes `node upgrade` and `node rollback` need a systemd unit. On a cron-supervised node, @@ -255,8 +263,8 @@ at a full core with sqlx `slow_acquire` warnings although Postgres is idle. - **Up to v0.1.8**, a bulk purge of a busy batch's history (`fleetctl job purge --older-than DAYS`) does the same: it holds that lock while it writes a record and a node command for every job, minutes for tens of thousands, and its node commands - then crowd out launches for as long again, at 16 per heartbeat. Later releases purge - in chunks and send a node's purges after its other commands. + then crowd out launches for as long again, at 16 per heartbeat. Later releases + purged in chunks; now there is no purge, and a batch's jobs go when it is deleted. ### Who changed what @@ -264,7 +272,7 @@ The scheduler logs an `api request` line for each API call that can change state method, path and query, status, duration, and the caller's token ID, label, namespace and tag (never the token itself). `docker logs zkm-fleet-scheduler-1 | grep 'api request'` shows who submitted, -cancelled, purged or reconfigured what. +cancelled, deleted or reconfigured what. ## Installed files and backups diff --git a/docs/reference.md b/docs/reference.md index 2b82702..6e89d08 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -200,9 +200,37 @@ administrator, since an app can reach host networking, storage and prover credentials; an update mixing them with other fields is refused whole. Changing priority alone doesn't restart replicas. -**Removing.** A proof app can't be removed while any of its proofs is queued or -running. Removing it deletes its finished proof records and control-host artifacts; -the app's own storage stays with its owner. +**Deleting.** `fleetctl app delete NAME --yes` and `fleetctl batch delete NAME --yes` +work the same way. Without `--yes` they only say what would go. A delete is refused +while any of the app's or batch's work runs or may run, and the refusal says what that +is and how to stop it: +- An app's work is its replicas. Disable it first, and wait until every node that + configures it has stopped them; an offline node holds the delete until it's back. +- A proof app's work also includes its queued or running proofs. +- A batch's work is its queued or running jobs. Cancel them or let them finish. + +The app or batch then goes with everything Fleet holds of it: +- On the control host: its job records with their logs, and the commands about them. + A proof app's job records and anchor go from Postgres. +- On each node: its configuration, with what the node holds of it. A batch's jobs lose + their directories: record, logs and `out/`. An app's replicas' logs and its workers' + directories (`worker_workdir`) go, and so do its proof jobs' files in its proof + storage: each job's output directory, and the inputs and ELF that no other job or + app uses. An EthProofs worker's directory is its proof storage root, so its + checkpoints and proofs go with it. + +Volumes, batch mounts, env files, storage roots and images stay. So does whatever +holds or lies in a path that another app or batch on any node uses, and anything at +the top of the filesystem or around Fleet's own files; the node's result says what it +kept and why, and the scheduler logs it. A container's `workdir` is the container's +own: what it keeps on the host is in its volumes. Job history goes only with its +batch; there is no separate purge. + +The record stays, hidden, until every node has removed it, so the name can't be +reused before then. A node that is offline removes it once it's back; deleting it +again retries a node that failed. A batch deleted by a release before this one kept +its job records: it stays listed as `deleted` until it is deleted again, which removes +them. A node with an older bridge removes only the configuration. **Node entries.** Every `apps.yaml` entry has an `app`. A `ziren-gpu` entry binds its server to the one app it serves with `namespace`, `app: NAME` and @@ -235,7 +263,7 @@ fleetctl batch # the batches, with their active and last jobs fleetctl batch NAME # one batch and its newest jobs ``` -`fleetctl batch logs`, `cancel`, `preempt` and `purge` take a job ID and find its +`fleetctl batch logs`, `cancel` and `preempt` take a job ID and find its batch; `--batch NAME` skips that lookup. `batch submit --name` names the job. Nodes can also advertise batches in their `batches.yaml`. @@ -259,8 +287,8 @@ In the API, `POST /batches?namespace=alice` takes the batch JSON, or the namespa its `namespace` field; a name or an ID is accepted, and the ID is stored. A batch or `apps.yaml` entry configured on a node may name `namespace: alice`, which is checked before launch. Disabling a batch blocks new submissions and lets accepted jobs -finish. A deleted batch stays listed, as `deleted`, and readable while it has job -records; its jobs keep their own lifecycle until they're purged. +finish. Deleting a batch works like deleting an app ([Deleting](#apps)): it is +refused while any of its jobs is queued or running, and its jobs go with it. **Submissions.** A job takes `command`, a non-empty list of strings, and optionally: @@ -502,7 +530,7 @@ accepted work: an administrator stops jobs explicitly. |---|---| | Read the own namespace, apps, batches, jobs, logs and scaling status | `read`, within the token's scope | | Submit proofs and batch jobs; cancel batch jobs | `submit`, within the token's scope | -| Create, remove, enable, disable, prioritize, scale or update apps and batches; purge batch jobs | `manage`, with any app grants. Removing, enabling, disabling or prioritizing a proof app also needs the tag it was created for and the tag of every node entry running it; scaling and updating it need the tag of the entry they change. | +| Create, remove, enable, disable, prioritize, scale or update apps and batches | `manage`, with any app grants. Removing, enabling, disabling or prioritizing a proof app also needs the tag it was created for and the tag of every node entry running it; scaling and updating it need the tag of the entry they change. | | Batch `allow` and `user`; app `user`, `volumes` and `proof_storage_root`; changing an app's image, environment, command, user, workdir or volumes | `admin` | | Preempt batch jobs | `admin` | | Namespaces, tokens, quotas, suspension, allocations, the preemption plan, nodes and status | `admin` | @@ -609,7 +637,7 @@ listener behind the HTTPS gateway. Administrators select a namespace with | Scaling | `PUT` / `GET /apps/NAME/scale` (the app's replicas over all its nodes; `GET ...?node=NODE` shows one node's part) | | Proof app jobs | `GET /apps/NAME/jobs?limit=N&active=true`, `POST /apps/NAME/jobs` (app-storage references), `GET /apps/NAME/jobs/ID`, `GET /apps/NAME/jobs/ID/output/proof.json` and `.../output/public_values.bin` | | Batches | `GET` / `POST /batches`, `GET` / `DELETE /batches/NAME`, `PUT /batches/NAME/priority`, `PUT /batches/NAME/settings`, `PUT` / `DELETE /batches/NAME/disabled` | -| Batch jobs | `GET` / `POST /batches/NAME/jobs`, `DELETE /batches/NAME/jobs?older_than=DAYS`, `GET` / `DELETE /batches/NAME/jobs/ID`, `POST /batches/NAME/jobs/ID/cancel`, `POST /batches/NAME/jobs/ID/preempt`, `GET /batches/NAME/jobs/ID/logs` | +| Batch jobs | `GET` / `POST /batches/NAME/jobs`, `GET /batches/NAME/jobs/ID`, `POST /batches/NAME/jobs/ID/cancel`, `POST /batches/NAME/jobs/ID/preempt`, `GET /batches/NAME/jobs/ID/logs` | | Namespaces and tokens | Action | |---|---|