Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,11 @@ Kubernetes' jobs, deployments and controllers, without needing Kubernetes.

Namespace owners create apps and batches and manage their priority, enabled state and
scaling, within their namespace's and tag's quotas; the scheduler picks each app's
node. Administrators manage namespaces, quotas, access, host storage for batches and
apps' execution settings.
node, and moves an app's start to a node with room when its own has none. Fleet copies
nothing an app keeps on a node's disks, so an app that keeps data there, or runs from
it, is anchored to that node with `machine` in its entry in the node's `apps.yaml`.
Administrators manage namespaces, quotas, access, host storage for batches and apps'
execution settings.

Two integrated app workers prove on their node's own GPU server:

Expand Down
29 changes: 24 additions & 5 deletions bridge/fleet_bridge/node.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ def set_fields(path: Path, section: str, name: str, fields: dict[str, Any]) -> N
entry = _entry(yaml.compose(text), section, name)
if not isinstance(entry, yaml.MappingNode):
raise ValueError(f"{section}: no entry {name}")
value = next((v for k, v in entry.value if k.value == key), None)
name_node, value = next(((k, v) for k, v in entry.value if k.value == key), (None, None))
written = json.dumps(setting)
if value is not None:
# A block value runs on to the next key; it ends with its last line of content.
Expand All @@ -78,13 +78,30 @@ def set_fields(path: Path, section: str, name: str, fields: dict[str, Any]) -> N
offset += len(line)
if _is_content(line):
end = offset - (1 if line.endswith("\n") else 0)
text = text[:value.start_mark.index] + written + text[end:]
start = value.start_mark.index
if isinstance(value, yaml.CollectionNode) and not value.flow_style:
# A block list or map starts on the next line, and a list's items may sit at
# the key's own indentation, so the new value goes right after the key's colon.
colon = text.index(":", name_node.end_mark.index) + 1
rest = text[colon:text.index("\n", colon)].strip()
written = " " + written + (f" {rest}" if rest else "")
start = colon
text = text[:start] + written + text[end:]
elif entry.flow_style:
position = entry.end_mark.index - 1
text = text[:position] + f", {key}: {written}" + text[position:]
else:
position = entry.start_mark.index
text = text[:position] + f"{key}: {written}\n" + " " * entry.start_mark.column + text[position:]
# The file is replaced only with text that reads back as the values asked for.
try:
data = (yaml.safe_load(text) or {}).get(section)
except yaml.YAMLError as error:
raise ValueError(f"{section}: writing {', '.join(fields)} into {name} would break {path.name}: {error}")
got = (next((e for e in data if isinstance(e, dict) and e.get("name") == name), None)
if isinstance(data, list) else (data or {}).get(name))
if not isinstance(got, dict) or any(got.get(k) != v for k, v in fields.items()):
raise ValueError(f"{section}: {name} doesn't read back {', '.join(fields)} as written")
_replace(path, text)


Expand Down Expand Up @@ -585,6 +602,11 @@ def _delete_jobs(self, batch: str, ids: list[str]) -> dict[str, Any]:
return {"jobs_deleted": deleted, "kept": kept}

def report(self) -> dict[str, Any]:
# The results first: a command finished by then has changed the configuration read
# below, so the scheduler never sees a create done before the node reports its entry.
with self.lock:
results = [{"id": p.stem, "result": v["result"], "error": v["error"]}
for p in self.journal.glob("*.json") if (v := read_json(p)).get("done")]
nodes = alive_nodes()
ready = len(nodes) == 1
total, free = resources()
Expand Down Expand Up @@ -635,9 +657,6 @@ def report(self) -> dict[str, Any]:
"logs_ready": (entry / f".fleet-logs-{lease}").exists()})
except (OSError, ValueError, SystemExit):
log.exception("reading local job %s", entry.name)
with self.lock:
results = [{"id": p.stem, "result": v["result"], "error": v["error"]}
for p in self.journal.glob("*.json") if (v := read_json(p)).get("done")]
return {"version": 3, "machine": socket.gethostname(), "ready": ready,
"total_gpus": total.get("GPU", 0), "total_cpus": total.get("CPU", 0),
"free_gpus": free.get("GPU", 0), "free_cpus": free.get("CPU", 0),
Expand Down
42 changes: 42 additions & 0 deletions bridge/tests/test_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,30 @@ def test_resource_update_preserves_comments_and_handles_flow_yaml(tmp_path):
assert data["image"]=="image:latest" and "# machine config" in p.read_text()


@pytest.mark.parametrize("lists",["- "," - "])
def test_settings_update_replaces_block_lists_and_maps_however_indented(tmp_path,lists):
p=tmp_path/"apps.yaml"
p.write_text(f"apps:\n- name: app\n kind: app\n volumes: # mounts\n {lists}/old/a\n {lists}/old/b:ro\n"
f" env:\n HOME: /home/old\n command:\n {lists}python\n {lists}main.py\n image: image:latest\n"
"- name: other\n kind: app\n volumes:\n - /keep\n")
for fields in [{"volumes":["/new/a"]},{"command":["/new/python","main.py"]},{"env":{"HOME":"/home/new"}},
{"volumes":["/new/b"],"command":["/new/b/python"],"workdir":"/new/b"}]:
node.set_fields(p,"apps","app",fields)
apps=yaml.safe_load(p.read_text())["apps"]
assert {k:apps[0][k] for k in fields}==fields and apps[0]["image"]=="image:latest"
assert apps[1]=={"name":"other","kind":"app","volumes":["/keep"]}
assert "# mounts" in p.read_text()


def test_settings_update_never_writes_a_file_that_does_not_read_back(tmp_path,monkeypatch):
p=tmp_path/"apps.yaml";p.write_text("apps:\n- name: app\n kind: app\n volumes:\n - /old\n")
before=p.read_bytes()
monkeypatch.setattr(node.json,"dumps",lambda value:"[unclosed")
with pytest.raises(ValueError,match="would break"):
node.set_fields(p,"apps","app",{"volumes":["/new"]})
assert p.read_bytes()==before and not list(tmp_path.glob("*.node-tmp"))




APPS = """# machine config
Expand Down Expand Up @@ -568,6 +592,24 @@ def test_create_adds_a_disabled_app_and_refuses_a_taken_name(executor):
assert error and "already configured" in str(error)


def test_a_report_carries_a_create_done_only_with_its_entry(executor,monkeypatch):
"""A create that finishes while a report is read is reported with its entry, in the next one."""
monkeypatch.setattr(node.ray,"cluster_resources",lambda:{"GPU":8,"CPU":48})
monkeypatch.setattr(node.ray,"available_resources",lambda:{"GPU":8,"CPU":48})
entry={"name":"web","kind":"app","namespace":"a"*32,"disabled":True,"image":"web:1","port_base":50100}
def running():
# The create runs after this report read the configuration.
if not (executor.journal/"create-1.json").exists():
app_message(executor,"create-1",{"create":entry},app="web",ns="a"*32)
return {}
monkeypatch.setattr(reconcile,"running",running)
for report in [executor.report(),executor.report()]:
done=[r["id"] for r in report["results"]]
if "create-1" in done:
assert "web" in [a["name"] for a in report["apps"]], report
assert "create-1" in done


def test_create_adds_a_prover_and_refuses_an_invalid_entry(executor):
prover={"name":"proofs","kind":"ziren-gpu","namespace":"a"*32,"disabled":True,"image":"ziren-gpu:1","elf_id":"b"*64}
assert app_message(executor,"create-1",{"create":prover},app="proofs",ns="a"*32)==({"created":["proofs"]},None)
Expand Down
4 changes: 3 additions & 1 deletion docs/reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,9 @@ anywhere, the start preempts lower-priority work on its own node and keeps the r
that frees. A start whose entry is anchored with `machine` never moves; every
integrated worker's entry is. It waits for room on its node, preempting
lower-priority batch jobs of its namespace when it can. Fleet copies nothing an app
keeps on a node's disks: the app looks after its own storage.
keeps on a node's disks: the app looks after its own storage. Anchor an app that keeps
data there, or runs from it, by setting `machine` in its entry in that node's
`apps.yaml` to the node's machine, as `fleetctl node` lists it.

**States.** An app is `configured`, `starting`, `running`, `draining`, `disabled`,
`offline` or `failed`; a proof app without provers is `registered`. The table shows
Expand Down
Loading