diff --git a/README.md b/README.md index a5068e7..4e64a66 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,11 @@ Kubernetes' jobs, deployments and controllers, without needing Kubernetes. Namespace owners create apps and batches and manage their priority, enabled state and scaling, within their namespace's and tag's quotas; the scheduler picks each app's -node. Administrators manage namespaces, quotas, access, host storage for batches and -apps' execution settings. +node, and moves an app's start to a node with room when its own has none. Fleet copies +nothing an app keeps on a node's disks, so an app that keeps data there, or runs from +it, is anchored to that node with `machine` in its entry in the node's `apps.yaml`. +Administrators manage namespaces, quotas, access, host storage for batches and apps' +execution settings. Two integrated app workers prove on their node's own GPU server: diff --git a/bridge/fleet_bridge/node.py b/bridge/fleet_bridge/node.py index 49c5628..15b91e8 100644 --- a/bridge/fleet_bridge/node.py +++ b/bridge/fleet_bridge/node.py @@ -69,7 +69,7 @@ def set_fields(path: Path, section: str, name: str, fields: dict[str, Any]) -> N entry = _entry(yaml.compose(text), section, name) if not isinstance(entry, yaml.MappingNode): raise ValueError(f"{section}: no entry {name}") - value = next((v for k, v in entry.value if k.value == key), None) + name_node, value = next(((k, v) for k, v in entry.value if k.value == key), (None, None)) written = json.dumps(setting) if value is not None: # A block value runs on to the next key; it ends with its last line of content. @@ -78,13 +78,30 @@ def set_fields(path: Path, section: str, name: str, fields: dict[str, Any]) -> N offset += len(line) if _is_content(line): end = offset - (1 if line.endswith("\n") else 0) - text = text[:value.start_mark.index] + written + text[end:] + start = value.start_mark.index + if isinstance(value, yaml.CollectionNode) and not value.flow_style: + # A block list or map starts on the next line, and a list's items may sit at + # the key's own indentation, so the new value goes right after the key's colon. + colon = text.index(":", name_node.end_mark.index) + 1 + rest = text[colon:text.index("\n", colon)].strip() + written = " " + written + (f" {rest}" if rest else "") + start = colon + text = text[:start] + written + text[end:] elif entry.flow_style: position = entry.end_mark.index - 1 text = text[:position] + f", {key}: {written}" + text[position:] else: position = entry.start_mark.index text = text[:position] + f"{key}: {written}\n" + " " * entry.start_mark.column + text[position:] + # The file is replaced only with text that reads back as the values asked for. + try: + data = (yaml.safe_load(text) or {}).get(section) + except yaml.YAMLError as error: + raise ValueError(f"{section}: writing {', '.join(fields)} into {name} would break {path.name}: {error}") + got = (next((e for e in data if isinstance(e, dict) and e.get("name") == name), None) + if isinstance(data, list) else (data or {}).get(name)) + if not isinstance(got, dict) or any(got.get(k) != v for k, v in fields.items()): + raise ValueError(f"{section}: {name} doesn't read back {', '.join(fields)} as written") _replace(path, text) @@ -585,6 +602,11 @@ def _delete_jobs(self, batch: str, ids: list[str]) -> dict[str, Any]: return {"jobs_deleted": deleted, "kept": kept} def report(self) -> dict[str, Any]: + # The results first: a command finished by then has changed the configuration read + # below, so the scheduler never sees a create done before the node reports its entry. + with self.lock: + results = [{"id": p.stem, "result": v["result"], "error": v["error"]} + for p in self.journal.glob("*.json") if (v := read_json(p)).get("done")] nodes = alive_nodes() ready = len(nodes) == 1 total, free = resources() @@ -635,9 +657,6 @@ def report(self) -> dict[str, Any]: "logs_ready": (entry / f".fleet-logs-{lease}").exists()}) except (OSError, ValueError, SystemExit): log.exception("reading local job %s", entry.name) - with self.lock: - results = [{"id": p.stem, "result": v["result"], "error": v["error"]} - for p in self.journal.glob("*.json") if (v := read_json(p)).get("done")] return {"version": 3, "machine": socket.gethostname(), "ready": ready, "total_gpus": total.get("GPU", 0), "total_cpus": total.get("CPU", 0), "free_gpus": free.get("GPU", 0), "free_cpus": free.get("CPU", 0), diff --git a/bridge/tests/test_node.py b/bridge/tests/test_node.py index 8030c1b..4aaad83 100644 --- a/bridge/tests/test_node.py +++ b/bridge/tests/test_node.py @@ -222,6 +222,30 @@ def test_resource_update_preserves_comments_and_handles_flow_yaml(tmp_path): assert data["image"]=="image:latest" and "# machine config" in p.read_text() +@pytest.mark.parametrize("lists",["- "," - "]) +def test_settings_update_replaces_block_lists_and_maps_however_indented(tmp_path,lists): + p=tmp_path/"apps.yaml" + p.write_text(f"apps:\n- name: app\n kind: app\n volumes: # mounts\n {lists}/old/a\n {lists}/old/b:ro\n" + f" env:\n HOME: /home/old\n command:\n {lists}python\n {lists}main.py\n image: image:latest\n" + "- name: other\n kind: app\n volumes:\n - /keep\n") + for fields in [{"volumes":["/new/a"]},{"command":["/new/python","main.py"]},{"env":{"HOME":"/home/new"}}, + {"volumes":["/new/b"],"command":["/new/b/python"],"workdir":"/new/b"}]: + node.set_fields(p,"apps","app",fields) + apps=yaml.safe_load(p.read_text())["apps"] + assert {k:apps[0][k] for k in fields}==fields and apps[0]["image"]=="image:latest" + assert apps[1]=={"name":"other","kind":"app","volumes":["/keep"]} + assert "# mounts" in p.read_text() + + +def test_settings_update_never_writes_a_file_that_does_not_read_back(tmp_path,monkeypatch): + p=tmp_path/"apps.yaml";p.write_text("apps:\n- name: app\n kind: app\n volumes:\n - /old\n") + before=p.read_bytes() + monkeypatch.setattr(node.json,"dumps",lambda value:"[unclosed") + with pytest.raises(ValueError,match="would break"): + node.set_fields(p,"apps","app",{"volumes":["/new"]}) + assert p.read_bytes()==before and not list(tmp_path.glob("*.node-tmp")) + + APPS = """# machine config @@ -568,6 +592,24 @@ def test_create_adds_a_disabled_app_and_refuses_a_taken_name(executor): assert error and "already configured" in str(error) +def test_a_report_carries_a_create_done_only_with_its_entry(executor,monkeypatch): + """A create that finishes while a report is read is reported with its entry, in the next one.""" + monkeypatch.setattr(node.ray,"cluster_resources",lambda:{"GPU":8,"CPU":48}) + monkeypatch.setattr(node.ray,"available_resources",lambda:{"GPU":8,"CPU":48}) + entry={"name":"web","kind":"app","namespace":"a"*32,"disabled":True,"image":"web:1","port_base":50100} + def running(): + # The create runs after this report read the configuration. + if not (executor.journal/"create-1.json").exists(): + app_message(executor,"create-1",{"create":entry},app="web",ns="a"*32) + return {} + monkeypatch.setattr(reconcile,"running",running) + for report in [executor.report(),executor.report()]: + done=[r["id"] for r in report["results"]] + if "create-1" in done: + assert "web" in [a["name"] for a in report["apps"]], report + assert "create-1" in done + + def test_create_adds_a_prover_and_refuses_an_invalid_entry(executor): prover={"name":"proofs","kind":"ziren-gpu","namespace":"a"*32,"disabled":True,"image":"ziren-gpu:1","elf_id":"b"*64} assert app_message(executor,"create-1",{"create":prover},app="proofs",ns="a"*32)==({"created":["proofs"]},None) diff --git a/docs/reference.md b/docs/reference.md index 6e89d08..2ea5638 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -175,7 +175,9 @@ anywhere, the start preempts lower-priority work on its own node and keeps the r that frees. A start whose entry is anchored with `machine` never moves; every integrated worker's entry is. It waits for room on its node, preempting lower-priority batch jobs of its namespace when it can. Fleet copies nothing an app -keeps on a node's disks: the app looks after its own storage. +keeps on a node's disks: the app looks after its own storage. Anchor an app that keeps +data there, or runs from it, by setting `machine` in its entry in that node's +`apps.yaml` to the node's machine, as `fleetctl node` lists it. **States.** An app is `configured`, `starting`, `running`, `draining`, `disabled`, `offline` or `failed`; a proof app without provers is `registered`. The table shows