From 2a6df55fba9b46374fe44ba0918408a4b7325334 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Tue, 1 Sep 2026 23:02:33 +0200 Subject: [PATCH 01/33] xGFabric twin as a service: twin.py's graph on a DTaaS broker Same graph and the same selection story as twin.py -- Davis wind sensor, a field agent whose three surrogate architectures compete on profiler-predicted Pi runtime, a heatmap sink -- servicified: the twin lives in the ORBIT `dt` plugin, tasks run on a rhapsody endpoint, the sensor is an external channel publisher bound with `add_input`, the data plane is ORBIT. The service/* components ship by value and fake the physics at the seams twin.py already fakes (sensor records, tk_do_simulation); the real FNO/PINN/PCR training stacks are not imported. What stays real: the shared memoised simulation subtask across the three investigators, the profiler chain (an inline timed run of the shipped inference function), and pi-runtime-driven model selection. Verified end to end against a local broker + endpoint: twin running, per-window retraining visible in the probes, pcr (the cheapest architecture) winning selection, 28 heatmaps, 24 shared-sim cache hits, clean teardown. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- service/README.md | 43 ++++++++++++ service/__init__.py | 9 +++ service/investigators.py | 86 +++++++++++++++++++++++ service/makefile | 6 ++ service/profiler.py | 87 +++++++++++++++++++++++ service/sensor_publisher.py | 45 ++++++++++++ service/sink.py | 57 +++++++++++++++ service/wind_agent.py | 97 ++++++++++++++++++++++++++ twin_service.py | 134 ++++++++++++++++++++++++++++++++++++ 9 files changed, 564 insertions(+) create mode 100644 service/README.md create mode 100644 service/__init__.py create mode 100644 service/investigators.py create mode 100644 service/makefile create mode 100644 service/profiler.py create mode 100644 service/sensor_publisher.py create mode 100644 service/sink.py create mode 100644 service/wind_agent.py create mode 100644 twin_service.py diff --git a/service/README.md b/service/README.md new file mode 100644 index 0000000..9842bef --- /dev/null +++ b/service/README.md @@ -0,0 +1,43 @@ +# xGFabric twin, service mode + +`../twin_service.py` runs twin.py's graph on a DTaaS broker: the twin +lives in the ORBIT `dt` plugin, its tasks run on a rhapsody endpoint, +the sensor is an external channel publisher. Physics is faked at the +same seams twin.py already fakes (sensor records, simulation); the +selection story is real — three surrogate architectures with different +costs, ranked by profiler-predicted Pi runtime. + +## Local run (three terminals + stack) + +Stack (from a digital.twins checkout with `./deploy/install.sh` done, +venv `ve3`/`ve.demo`): + + ./deploy/run-broker.sh $PWD/ + ./deploy/run-endpoint.sh dt_inference_ep localhost $PWD/ + +Client terminals (each): + + export RADICAL_ORBIT_BROKER_URL=wss://localhost:8000 + export DT_STREAM_BACKEND=orbit + + /bin/python service/sensor_publisher.py # terminal 1 + /bin/python twin_service.py --runtime 240 # terminal 2 + +Dashboard: `https://localhost:8000/broker/dt/ui?live=1` (broker token). +Heatmaps land in `$XGF_WORKSPACE` (default `~/xgf_twin/`) on the host +running the endpoint tasks. + +Placement: `DT_INFERENCE_ENDPOINT` / `DT_INFERENCE_BACKEND` override +the defaults (`dt_inference_ep` / `concurrent`). + +## What maps to what + +| twin.py (standalone) | twin_service.py (DTaaS) | +|----------------------------------|--------------------------------------| +| local WorkflowEngine + backend | session engine on the broker, tasks on the endpoint (ENGINES config) | +| DavisWind persistent component | external `ChannelPublisher` + `add_input` binding | +| WindFieldAgent + FNO/PINN/PCR | `ServiceWindFieldAgent` + fake `SurrogateInvestigator`s (same selection logic) | +| profiler subprocess + Pi learner | `ServiceProfiler` (inline timed run) + `ServicePiPredictor` | +| CUPS_Sink | `ServiceSink` (runtime-resolved workspace) | +| ZMQ stream | ORBIT data plane | +| asyncflow telemetry + reports | not yet — see the telemetry branch | diff --git a/service/__init__.py b/service/__init__.py new file mode 100644 index 0000000..32e0bd1 --- /dev/null +++ b/service/__init__.py @@ -0,0 +1,9 @@ +# Service-mode (DTaaS / ORBIT) variant of the xGFabric twin. +# +# Same graph and the same model-selection story as ../twin.py -- a wind +# sensor feeds a field agent whose three competing surrogates are ranked +# by profiler-predicted Pi runtime -- but the components here are +# service-safe: they ship to the broker by value, run their tasks on a +# rhapsody endpoint, and fake the physics (as twin.py already does for +# the sensor and the simulation). The real FNO/PINN/PCR training stacks +# stay in ../tasks and are NOT imported here. diff --git a/service/investigators.py b/service/investigators.py new file mode 100644 index 0000000..22f88d3 --- /dev/null +++ b/service/investigators.py @@ -0,0 +1,86 @@ +"""Service-safe surrogate investigators: FNO / PINN / PCR stand-ins. + +Each keeps the real investigators' shape -- batch sensor windows, run a +"training" flow task on the endpoint, publish the model, serve field +inference -- with fake physics: training sleeps an architecture-typical +time and inference synthesizes a wind field. What stays real is the +part the demo is about: three architectures with different costs +competing for selection, and the shared simulation subtask feeding all +of them (memoised once per sensor window, see SIM_MASTER). +""" + +import asyncio +import random + +import numpy as np + +from digitaltwin.components import ModelInvestigator, TypedData +from digitaltwin.runtime import RuntimeAPI + +from tasks.common.dtypes import SIM_MASTER, WIND_FIELD + +# architecture-typical (train_seconds, inference_seconds) -- the spread is +# what makes profiler-driven selection visible +ARCH_COST = { + "fno": (6.0, 0.20), + "pinn": (9.0, 0.60), + "pcr": (3.0, 0.05), +} + +WINDOW = 4 # sensor points per training window + + +class SurrogateInvestigator(ModelInvestigator): + """One fake surrogate; ``arch`` picks its cost profile.""" + + def __init__(self, flow, arch: str): + super().__init__(flow) + self.flow = flow + self.arch = arch + self.batch: list = [] + train_s, infer_s = ARCH_COST[arch] + + @flow.function_task + async def train(arch, points, sim): + # endpoint-side "training": cost is the architecture's + await asyncio.sleep(train_s) + speeds = [p["wind_speed"] for p in points] + return {"model": arch, "w": float(np.mean(speeds)) / 20.0, + "sim": sim[1]} + + @flow.function_task + async def infer(in_data, model="na", w=0.0, **_): + await asyncio.sleep(infer_s) + if model == "na": + return TypedData(WIND_FIELD, + {"arch": "na", "w": w, "result": None}) + # synthesized field: smooth bump scaled by the model weight + x = np.linspace(-2, 2, 32) + xx, yy = np.meshgrid(x, x) + field = (2.5 * w) * np.exp(-(xx ** 2 + yy ** 2)) + field += np.random.default_rng().normal(0, 0.05, field.shape) + return TypedData(WIND_FIELD, + {"arch": model, "w": w, "result": (3, field)}) + + self._train = train + self._infer = infer + + async def _on_input(self, in_data: TypedData) -> None: + self.batch.append(in_data.data) + + async def main_loop(self, runtime: RuntimeAPI): + runtime.subscribe_to_topic(runtime.ON_INPUT, self._on_input) + runtime.set_inference_task(self._infer) + runtime.publish_new_model({"model": "na", "w": 0.0}) + + while True: + if len(self.batch) < WINDOW: + await asyncio.sleep(1.0) + continue + + points, self.batch = self.batch[:WINDOW], self.batch[WINDOW:] + # one simulation per window, shared and memoised across the + # three investigators -- the SIM_MASTER contract from twin.py + sim = await runtime.call_shared_subtask(SIM_MASTER, points[0]) + model = await self._train(self.arch, points, sim) + runtime.publish_new_model(model, {"quality": random.random()}) diff --git a/service/makefile b/service/makefile new file mode 100644 index 0000000..da3cac6 --- /dev/null +++ b/service/makefile @@ -0,0 +1,6 @@ + +PROJECT_TYPE = python +PROJECT_NAME = xgfabric.service + +include $(HOME)/.makefile + diff --git a/service/profiler.py b/service/profiler.py new file mode 100644 index 0000000..bb37a77 --- /dev/null +++ b/service/profiler.py @@ -0,0 +1,87 @@ +"""Profiling chain, service-safe: measure, then predict for the Pi. + +``ServiceProfiler`` really profiles: it unpickles the surrogate's +inference function and times a run with the example input -- inline as a +flow task on the endpoint, instead of twin.py's exported-pickle + +subprocess round trip (whose script path does not survive shipping by +value). Results are memoised per (function, model) exactly like the +original. + +``ServicePiPredictor`` stands in for the Pi endpoint's learned runtime +model: predicted Pi runtime = measured runtime x a Pi/HPC slowdown +factor with mild noise. The real EndpointInvestigator's training on +recorded Pi profiles is the phase-2 story. +""" + +import time + +import cloudpickle + +from digitaltwin.components import DataType, ModelInvestigator, TypedData +from digitaltwin.lru import LRUCache, freeze +from digitaltwin.runtime import RuntimeAPI + +TASK_DESCRIPTION_DTYPE = DataType("TASK_INFO") +PROFILE_RESULTS = DataType("PROFILE_RESULT") +PI_PREDICT_RUNTIME = DataType("pi_PREDICT_RUNTIME") + +PI_SLOWDOWN = 7.5 # a Raspberry Pi against one HPC core, rough + + +class ServiceProfiler(ModelInvestigator): + """Times one run of a shipped inference function; memoises per model.""" + + def __init__(self, flow): + super().__init__(flow) + self.flow = flow + + @flow.function_task + async def run_profiled(blob, example, model_kwargs): + fn = cloudpickle.loads(blob) + t0 = time.monotonic() + await fn(example, **(model_kwargs or {})) + return {"runtime": time.monotonic() - t0} + + cache = LRUCache(128) + + async def do_inference(in_data: TypedData): + blob, example, model_kwargs = in_data.data + key = freeze((blob, tuple(sorted((model_kwargs or {}).items())))) + + if await cache.exists(key): + profile = await cache.fetch_item(key) + else: + profile = await run_profiled(blob, example, model_kwargs) + await cache.put_item(key, profile) + + return TypedData(PROFILE_RESULTS, + {"profile": profile, "task": in_data.data}) + + self.inference_task = do_inference + + async def main_loop(self, runtime: RuntimeAPI): + runtime.set_inference_task(self.inference_task) + runtime.publish_new_model() + + +class ServicePiPredictor(ModelInvestigator): + """PROFILE_RESULT -> predicted runtime on the Pi endpoint.""" + + def __init__(self, flow): + super().__init__(flow) + self.flow = flow + + @flow.function_task + async def predict(in_data): + import random + measured = in_data.data["profile"]["runtime"] + return measured * PI_SLOWDOWN * random.uniform(0.9, 1.1) + + async def do_inference(in_data: TypedData): + return TypedData(PI_PREDICT_RUNTIME, await predict(in_data)) + + self.inference_task = do_inference + + async def main_loop(self, runtime: RuntimeAPI): + runtime.set_inference_task(self.inference_task) + runtime.publish_new_model() diff --git a/service/sensor_publisher.py b/service/sensor_publisher.py new file mode 100644 index 0000000..77cd69a --- /dev/null +++ b/service/sensor_publisher.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Fake Davis wind sensor as an EXTERNAL channel publisher. + +In service mode the sensor is not a twin component: it publishes to a +shared channel and the twin binds it with ``add_input`` -- any number of +twins can listen. The records mirror what ``tasks.davis`` emits +(``strip_cols`` output), at the same 5 s cadence, fake for the same +reason twin.py fakes them. + +Environment: DT_STREAM_BACKEND / RADICAL_ORBIT_BROKER_URL(+_CERT) select +the data plane, exactly like every other client-side process. +""" + +import asyncio +import datetime +import random + +from digitaltwin.streaming import ChannelPublisher + +DAVIS_CHANNEL = "xgf/davis" + + +async def main() -> None: + publisher = await ChannelPublisher.open(DAVIS_CHANNEL) + n = 0 + try: + while True: + r = random.random() + record = { + "dt": datetime.datetime.now().isoformat(), + "wind_speed": round(r * 20, 1), + "wind_avg": round(r * 15, 1), + "wind_dir": round(r * 360), + } + await publisher.publish(record) + n += 1 + if n % 12 == 0: + print(f"davis: {n} records", flush=True) + await asyncio.sleep(5) + finally: + await publisher.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/service/sink.py b/service/sink.py new file mode 100644 index 0000000..21c7e94 --- /dev/null +++ b/service/sink.py @@ -0,0 +1,57 @@ +"""Terminal component: heatmap of the selected surrogate's wind field. + +Same output as ``tasks.sink.CUPS_Sink`` -- the workspace resolves at +runtime on the executing host (XGF_WORKSPACE, default the host's home), +because this module ships by value and a client-side path does not exist +where the component runs. +""" + +import os +from pathlib import Path + +from digitaltwin.components import TypedData, UtilityTask + + +def _workspace() -> Path: + base = Path(os.environ.get("XGF_WORKSPACE", "") + or Path.home() / "xgf_twin") + base.mkdir(parents=True, exist_ok=True) + return base + + +class ServiceSink(UtilityTask): + def __init__(self, flow): + super().__init__(flow) + self.flow = flow + self.count = 0 + + @flow.function_task + async def save_heatmap(data, arch, w, fname): + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + plt.figure(figsize=(6, 5)) + im = plt.imshow(data, cmap="viridis", origin="lower", + vmin=0, vmax=2.5) + plt.colorbar(im) + plt.title(f"Heatmap of {arch} at Z=3, W={round(w, 3)}") + plt.xlabel("X") + plt.ylabel("Y") + plt.savefig(fname, dpi=150, bbox_inches="tight") + plt.close() + return fname + + self._save = save_heatmap + + async def main_loop(self, runtime, in_data: TypedData): + arch = in_data.data["arch"] + print(f"[sink] field from {arch}", flush=True) + if arch == "na": + return + + self.count += 1 + fname = str(_workspace() / f"field_{self.count:04d}_{arch}.png") + await self._save(in_data.data["result"][1], arch, + in_data.data["w"], fname) + print("\n" + "=" * 30 + f"\n{fname}\n" + "=" * 30, flush=True) diff --git a/service/wind_agent.py b/service/wind_agent.py new file mode 100644 index 0000000..50ce09d --- /dev/null +++ b/service/wind_agent.py @@ -0,0 +1,97 @@ +"""The WindField agent, service-safe. + +Structure and selection logic mirror ``tasks.wind_agent.WindFieldAgent``: +three surrogate investigators under one agent, a shared (memoised) +simulation subtask, and a model selector that ranks published models by +their profiler-predicted Pi runtime -- fetched through the twin's own +``get_inference`` chain (task description -> profile -> predicted +runtime). Only the physics behind those steps is faked. +""" + +import asyncio +import random + +import cloudpickle + +from digitaltwin.components import SciAgent, TypedData, DataType +from digitaltwin.runtime import RuntimeAPI + +from tasks.common.dtypes import SIM_MASTER, DAVIS_WIND_SENSOR +from service.investigators import SurrogateInvestigator +from service.profiler import PI_PREDICT_RUNTIME, PROFILE_RESULTS, \ + TASK_DESCRIPTION_DTYPE + + +class ServiceWindFieldAgent(SciAgent): + def __init__(self, flow): + super().__init__(flow) + self.flow = flow + + self.fno = SurrogateInvestigator(flow, "fno") + self.pinn = SurrogateInvestigator(flow, "pinn") + self.pcr = SurrogateInvestigator(flow, "pcr") + + @flow.function_task + async def simulate(sensor_pt): + # twin.py's tk_do_simulation is already faked to a random + # quality plus a precomputed-sim path; same here, local path + await asyncio.sleep(1.0) + return (random.random(), f"precalc_sims/{random.randint(0, 71)}.csv") + + self.sim_master = simulate + + @flow.function_task + async def model_select(in_data: TypedData, models): + # the demo's crown jewel: pick the model with the shortest + # profiler-PREDICTED runtime on the Pi + if not models: + return 0 + best = min(models, key=lambda m: m["pi_runtime"]) + return best["investigator"] + + self.model_selector = model_select + + self.model_to_process: asyncio.Queue[dict] = asyncio.Queue() + self.models: list[dict] = [] + + async def model_publish_cb(self, inv, model_args: dict, acc: dict): + await self.model_to_process.put({ + "investigator": inv.get_id(), + "model_args": model_args, + "metrics": acc, + }) + + async def main_loop(self, runtime: RuntimeAPI): + runtime.start_investigator(self.fno) + runtime.start_investigator(self.pinn) + runtime.start_investigator(self.pcr) + + runtime.register_shared_subtask(SIM_MASTER, self.sim_master, 64) + runtime.subscribe_to_topic(runtime.ON_MODEL_PUBLISH, + self.model_publish_cb) + + runtime.set_model_selection_task(self.model_selector) + runtime.update_model_selector(models=[]) + + while True: + item = await self.model_to_process.get() + if item["model_args"]["model"] == "na": + continue + + raw = runtime.get_inference_tasks()[item["investigator"]] + raw = getattr(raw, "__wrapped__", raw) + + probe_pt = {"dt": "probe", "wind_speed": 10.0, + "wind_avg": 8.0, "wind_dir": 180.0} + description = (cloudpickle.dumps(raw), + TypedData(DAVIS_WIND_SENSOR, probe_pt), + item["model_args"]) + + profile = await runtime.get_inference( + TypedData(TASK_DESCRIPTION_DTYPE, description), + PROFILE_RESULTS) + pi = await runtime.get_inference(profile, PI_PREDICT_RUNTIME) + + item["pi_runtime"] = pi.data + self.models.append(item) + runtime.update_model_selector(models=self.models[-10:]) diff --git a/twin_service.py b/twin_service.py new file mode 100644 index 0000000..ff28e08 --- /dev/null +++ b/twin_service.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""The xGFabric twin as a service: twin.py's graph on a DTaaS broker. + +Same story as twin.py -- Davis wind sensor, a field agent whose three +surrogate architectures compete on profiler-predicted Pi runtime, a +heatmap sink -- but the twin lives on an ORBIT broker and its tasks run +on a rhapsody endpoint. The sensor is an external channel publisher +(service/sensor_publisher.py); components ship by value (service/*, +fake physics, see service/__init__.py). + +Environment (client side): + RADICAL_ORBIT_BROKER_URL(+_CERT) the broker + DT_STREAM_BACKEND=orbit the data plane + DT_SERVICE_HOST participant hosting `dt` (broker) + DT_INFERENCE_ENDPOINT / _BACKEND task placement (local endpoint / + concurrent by default) +""" + +import argparse +import os +import sys +import time + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from radical.orbit import EndpointRuntime + +from digitaltwin.components import NULL_DTYPE, TypedData +from digitaltwin.service import register_user_modules + +import service +import service.investigators +import service.profiler +import service.sink +import service.wind_agent +import tasks +import tasks.common +import tasks.common.dtypes + +from tasks.common.dtypes import DAVIS_WIND_SENSOR, WIND_FIELD +from service.profiler import ( + PI_PREDICT_RUNTIME, + PROFILE_RESULTS, + TASK_DESCRIPTION_DTYPE, + ServicePiPredictor, + ServiceProfiler, +) +from service.sensor_publisher import DAVIS_CHANNEL +from service.sink import ServiceSink +from service.wind_agent import ServiceWindFieldAgent + +register_user_modules([ + tasks, tasks.common, tasks.common.dtypes, + service, service.investigators, service.profiler, + service.sink, service.wind_agent, +]) + +DT_HOST = os.environ.get("DT_SERVICE_HOST", "broker") + +INFERENCE_EP = os.environ.get("DT_INFERENCE_ENDPOINT", "dt_inference_ep") +INFERENCE_BE = os.environ.get("DT_INFERENCE_BACKEND", "concurrent") + +ENGINES = { + "engines": { + "inference": {"endpoint_name": INFERENCE_EP, + "backends": [INFERENCE_BE]}, + } +} + + +def main(args) -> int: + runtime = EndpointRuntime() + runtime.start(wait=True) + + try: + dt = runtime.get_plugin(DT_HOST, "dt", config=ENGINES) + print(f"[client] session: {dt.sid} (reattach with this sid)") + + twin = dt.create_twin() + print(f"[client] twin: {twin}") + + agent = dt.package(ServiceWindFieldAgent) + profiler = dt.package(ServiceProfiler) + pi = dt.package(ServicePiPredictor) + sink = dt.package(ServiceSink) + + # sensor is external: bind its channel to the input dtype + dt.add_input(twin, DAVIS_WIND_SENSOR, DAVIS_CHANNEL) + + dt.add_agent(twin, agent, DAVIS_WIND_SENSOR, WIND_FIELD) + dt.add_task(twin, sink, WIND_FIELD, NULL_DTYPE) + + # the profiling chain the agent's selector queries + dt.add_investigator(twin, profiler, TASK_DESCRIPTION_DTYPE, + PROFILE_RESULTS) + dt.add_investigator(twin, pi, PROFILE_RESULTS, PI_PREDICT_RUNTIME) + + dt.start(twin) + + # client-side feedback: the pipeline is stream driven, so poll the + # twin and probe the field inference -- a stuck twin shows in 10s + probe_pt = {"dt": "probe", "wind_speed": 12.0, "wind_avg": 9.0, + "wind_dir": 90.0} + deadline = time.time() + args.runtime + while time.time() < deadline: + time.sleep(10) + + info = dt.twin(twin) + print(f"[client] state={info['state']}" + f" calls={info.get('calls') or {}}", flush=True) + if info["state"] == "failed": + print(f"[client] twin failed: {info.get('last_error')}") + return 1 + + answer = dt.get_inference( + twin, TypedData(DAVIS_WIND_SENSOR, probe_pt), WIND_FIELD, + timeout=60) + arch = answer.data["arch"] + print(f"[client] field probe -> arch={arch}" + f" w={answer.data.get('w')}", flush=True) + + print("SHUTDOWN") + dt.twin_close(twin) + return 0 + + finally: + runtime.stop() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="xGFabric twin, service mode") + parser.add_argument("--runtime", type=int, default=240, + help="seconds to keep the twin running") + sys.exit(main(parser.parse_args())) From 108b5199cfbd2b213312d2771664cdf0167eac29 Mon Sep 17 00:00:00 2001 From: Benjamin C Carter <59350660+BenCarter44@users.noreply.github.com> Date: Wed, 2 Sep 2026 05:21:04 -0700 Subject: [PATCH 02/33] Comment out picture save, as requires endpoint filesystem --- service/sink.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/service/sink.py b/service/sink.py index 21c7e94..9dea9ab 100644 --- a/service/sink.py +++ b/service/sink.py @@ -47,11 +47,11 @@ async def save_heatmap(data, arch, w, fname): async def main_loop(self, runtime, in_data: TypedData): arch = in_data.data["arch"] print(f"[sink] field from {arch}", flush=True) - if arch == "na": - return - - self.count += 1 - fname = str(_workspace() / f"field_{self.count:04d}_{arch}.png") - await self._save(in_data.data["result"][1], arch, - in_data.data["w"], fname) - print("\n" + "=" * 30 + f"\n{fname}\n" + "=" * 30, flush=True) + # if arch == "na": + # return + + # self.count += 1 + # fname = str(_workspace() / f"field_{self.count:04d}_{arch}.png") + # await self._save(in_data.data["result"][1], arch, + # in_data.data["w"], fname) + # print("\n" + "=" * 30 + f"\n{fname}\n" + "=" * 30, flush=True) From bccf99ed73abcf8e2e7ac7786550eae53fac9c81 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Thu, 3 Sep 2026 14:28:43 +0200 Subject: [PATCH 03/33] service twin: r3/Perlmutter deploy kit, adapted from dt-complete Same debugged constraints as the AmSC kit (dragon launcher, python >= 3.12.1, SLURM_EXPORT_ENV, PATH-by-name helpers, cert staging), with the xGFabric deltas: numpy/matplotlib for the by-value components, the endpoint registers as 'hpc' with the client's `remote` profile selecting it on dragon_v3, XGF_WORKSPACE on scratch -- and rhapsody pinned to fix/dragon-cancel-and-traceback on every tier, the branch carrying the idempotent-cancel fixes this demo surfaced (the orbit one runs broker-side). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- service/README.md | 26 ++++++++++++++++++++++++ service/deploy/client-env.sh | 23 +++++++++++++++++++++ service/deploy/run-hpc-endpoint.sh | 28 ++++++++++++++++++++++++++ service/deploy/setup-broker.sh | 23 +++++++++++++++++++++ service/deploy/setup-hpc-endpoint.sh | 30 ++++++++++++++++++++++++++++ 5 files changed, 130 insertions(+) create mode 100755 service/deploy/client-env.sh create mode 100755 service/deploy/run-hpc-endpoint.sh create mode 100755 service/deploy/setup-broker.sh create mode 100755 service/deploy/setup-hpc-endpoint.sh diff --git a/service/README.md b/service/README.md index 9842bef..2b7d3a5 100644 --- a/service/README.md +++ b/service/README.md @@ -30,6 +30,32 @@ running the endpoint tasks. Placement: `DT_INFERENCE_ENDPOINT` / `DT_INFERENCE_BACKEND` override the defaults (`dt_inference_ep` / `concurrent`). +## Remote run (broker on radical.3, endpoint on Perlmutter) + +`service/deploy/` adapts the AmSC dt-complete deploy kit (same debugged +constraints: dragon launcher requirement, python >= 3.12.1, +SLURM_EXPORT_ENV, cert staging), pinning rhapsody's +`fix/dragon-cancel-and-traceback` branch on every tier for the +idempotent-cancel fixes this demo surfaced. + + # broker host, once + service/deploy/setup-broker.sh + cd ~/digital_twins && ./deploy/run-broker.sh $PWD/ve.demo + + # Perlmutter login node, once + service/deploy/setup-hpc-endpoint.sh + # then inside salloc -N1 -C cpu -q interactive -t 2:00:00 -A : + service/deploy/run-hpc-endpoint.sh # registers as 'hpc' + + # client terminals (driver + sensor), each: + source service/deploy/client-env.sh remote + /bin/python service/sensor_publisher.py # terminal 1 + /bin/python twin_service.py --runtime 240 # terminal 2 + +The client venv must be the same Python minor (digital.twins +`./deploy/install.sh client` + `pip install numpy`); a 3.13 venv is +rejected at the first verb. + ## What maps to what | twin.py (standalone) | twin_service.py (DTaaS) | diff --git a/service/deploy/client-env.sh b/service/deploy/client-env.sh new file mode 100755 index 0000000..9b3a812 --- /dev/null +++ b/service/deploy/client-env.sh @@ -0,0 +1,23 @@ +# Client-side environment for the xGFabric service twin -- source me in +# EVERY client terminal (driver and sensor): +# +# source service/deploy/client-env.sh [remote] +# +# With `remote`, task placement targets the HPC endpoint ('hpc' on +# dragon_v3, as run-hpc-endpoint.sh launches it); without it the local +# defaults apply (dt_inference_ep / concurrent). +# +# The client venv is digital.twins' `./deploy/install.sh client` plus +# `pip install numpy` -- same Python minor as broker and endpoint. +# DT_BROKER_CERT overrides the pinned-cert path when this machine runs +# a broker of its own. +BROKER="${1:?usage: source client-env.sh [remote]}" + +export RADICAL_ORBIT_BROKER_URL="wss://$BROKER:8000" +export RADICAL_ORBIT_BROKER_CERT="${DT_BROKER_CERT:-$HOME/.radical/orbit/broker_cert.pem}" +export DT_STREAM_BACKEND=orbit + +if [ "${2:-}" = "remote" ]; then + export DT_INFERENCE_ENDPOINT=hpc + export DT_INFERENCE_BACKEND=dragon_v3 +fi diff --git a/service/deploy/run-hpc-endpoint.sh b/service/deploy/run-hpc-endpoint.sh new file mode 100755 index 0000000..e2ef1bb --- /dev/null +++ b/service/deploy/run-hpc-endpoint.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Run the xGFabric twin's HPC endpoint -- INSIDE a compute allocation. +# Adapted from the AmSC dt-complete deploy kit; the launch constraints +# (dragon launcher, PATH-by-name helpers, SLURM_EXPORT_ENV) are the +# debugged ones from there. +# +# ./run-hpc-endpoint.sh +# +# The endpoint registers as 'hpc' -- what the client's remote profile +# (client-env.sh remote) asks for. DT_DIR as in setup-hpc-endpoint.sh. +set -euo pipefail +BROKER="${1:?usage: $0 }" +DT_DIR="${DT_DIR:-$HOME/digital_twins}" +VENV="$DT_DIR/ve.demo" + +# dragon resolves its helpers BY NAME through srun on the task side +export PATH="$VENV/bin:$PATH" +export RADICAL_ORBIT_BROKER_URL="wss://$BROKER:8000" +export RADICAL_ORBIT_BROKER_CERT="$HOME/.radical/orbit/broker_cert.pem" +export RADICAL_ORBIT_RHAPSODY_BACKEND=dragon_v3 +export RADICAL_ORBIT_RHAPSODY_NOTIFY_WINDOW=0 +export SLURM_EXPORT_ENV=ALL # inner sruns must not scrub the env +export DT_STREAM_BACKEND=orbit +# heatmaps land on scratch where available +export XGF_WORKSPACE="${XGF_WORKSPACE:-${SCRATCH:-$HOME}/xgf_twin}" + +"$VENV/bin/dragon" "$VENV/bin/radical-orbit-endpoint.py" -n hpc \ + 2>&1 | tee endpoint.log diff --git a/service/deploy/setup-broker.sh b/service/deploy/setup-broker.sh new file mode 100755 index 0000000..e2daabe --- /dev/null +++ b/service/deploy/setup-broker.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# DTaaS broker host for the xGFabric service twin (e.g. radical.3). +# Run there. Adapted from the AmSC dt-complete deploy kit, which +# carries the debugged install/launch lore; deltas here: numpy for the +# by-value components, and rhapsody from the cancel-fix branch (the +# broker-side engine is what showed the teardown KeyError noise). +# +# Once per host: broker_cert.pem / broker_key.pem / broker.token in +# ~/.radical/orbit/. DT_DIR overrides the checkout+venv location. +set -euo pipefail +DT_DIR="${DT_DIR:-$HOME/digital_twins}" + +[ -d "$DT_DIR" ] || git clone https://github.com/radical-cybertools/digital.twins.git "$DT_DIR" +cd "$DT_DIR" && git checkout devel && git pull + +./deploy/install.sh broker # pinned stack -> ./ve.demo +./ve.demo/bin/pip install -q numpy +# idempotent cancels (dragon+orbit backends); main-based, includes pools +./ve.demo/bin/pip install -q --force-reinstall --no-deps \ + "rhapsody-py[telemetry] @ git+https://github.com/radical-cybertools/rhapsody@fix/dragon-cancel-and-traceback" + +echo "done. start the broker with:" +echo " cd $DT_DIR && ./deploy/run-broker.sh \$PWD/ve.demo" diff --git a/service/deploy/setup-hpc-endpoint.sh b/service/deploy/setup-hpc-endpoint.sh new file mode 100755 index 0000000..4547236 --- /dev/null +++ b/service/deploy/setup-hpc-endpoint.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# HPC-endpoint venv for the xGFabric service twin (e.g. Perlmutter). +# Run on a login node. Adapted from the AmSC dt-complete deploy kit. +# +# ./setup-hpc-endpoint.sh +# +# DT_DIR overrides the checkout+venv location (default ~/digital_twins). +set -euo pipefail +BROKER="${1:?usage: $0 }" +DT_DIR="${DT_DIR:-$HOME/digital_twins}" + +# same Python minor as every other host -- the service rejects skew, and +# exactly 3.12.0 breaks dragon's transport import (needs >= 3.12.1) +module load python/3.12 2>/dev/null || true + +[ -d "$DT_DIR" ] || git clone https://github.com/radical-cybertools/digital.twins.git "$DT_DIR" +cd "$DT_DIR" && git checkout devel && git pull + +./deploy/install.sh endpoint # pinned stack -> ./ve.demo +# the surrogate/sink task bodies unpickle and run here +./ve.demo/bin/pip install -q numpy matplotlib +# dragon backend + idempotent cancels + failure-traceback logging +./ve.demo/bin/pip install -q --force-reinstall --no-deps \ + "rhapsody-py[telemetry,dragon] @ git+https://github.com/radical-cybertools/rhapsody@fix/dragon-cancel-and-traceback" + +mkdir -p ~/.radical/orbit +scp "$BROKER:.radical/orbit/broker_cert.pem" "$BROKER:.radical/orbit/broker.token" ~/.radical/orbit/ + +echo "done. get an allocation (salloc -N1 -C cpu -q interactive -t 2:00:00 -A )," +echo "then run: service/deploy/run-hpc-endpoint.sh $BROKER" From 09c8615d4e3686dbf395a5101c8e5b49ba355fd8 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Thu, 3 Sep 2026 14:44:45 +0200 Subject: [PATCH 04/33] deploy: the Perlmutter checkout and venv default to $SCRATCH A home-quota casualty on the first remote attempt; the venv does not fit there. XGF_WORKSPACE already landed on scratch. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- service/deploy/run-hpc-endpoint.sh | 2 +- service/deploy/setup-hpc-endpoint.sh | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/service/deploy/run-hpc-endpoint.sh b/service/deploy/run-hpc-endpoint.sh index e2ef1bb..a826572 100755 --- a/service/deploy/run-hpc-endpoint.sh +++ b/service/deploy/run-hpc-endpoint.sh @@ -10,7 +10,7 @@ # (client-env.sh remote) asks for. DT_DIR as in setup-hpc-endpoint.sh. set -euo pipefail BROKER="${1:?usage: $0 }" -DT_DIR="${DT_DIR:-$HOME/digital_twins}" +DT_DIR="${DT_DIR:-${SCRATCH:-$HOME}/digital_twins}" VENV="$DT_DIR/ve.demo" # dragon resolves its helpers BY NAME through srun on the task side diff --git a/service/deploy/setup-hpc-endpoint.sh b/service/deploy/setup-hpc-endpoint.sh index 4547236..35b5340 100755 --- a/service/deploy/setup-hpc-endpoint.sh +++ b/service/deploy/setup-hpc-endpoint.sh @@ -4,10 +4,11 @@ # # ./setup-hpc-endpoint.sh # -# DT_DIR overrides the checkout+venv location (default ~/digital_twins). +# DT_DIR overrides the checkout+venv location; the default lands on +# $SCRATCH -- the venv is far too big for a Perlmutter home quota. set -euo pipefail BROKER="${1:?usage: $0 }" -DT_DIR="${DT_DIR:-$HOME/digital_twins}" +DT_DIR="${DT_DIR:-${SCRATCH:-$HOME}/digital_twins}" # same Python minor as every other host -- the service rejects skew, and # exactly 3.12.0 breaks dragon's transport import (needs >= 3.12.1) From 42f96dbecd4aa82bc74aecc1ff71a859dd59ce6d Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Thu, 3 Sep 2026 15:10:33 +0200 Subject: [PATCH 05/33] deploy: pin the dragon-compatible rhapsody branch on both tiers The main-based fix/dragon-cancel-and-traceback pin broke the endpoint: main's dragon backend passes task_logs= to Batch(), which the pinned dragonhpc 0.14.1 does not accept (Session init failed on endpoint: Batch.__init__() got an unexpected keyword argument 'task_logs'). Both tiers now pin fix/dragon-cancel-idempotent (e491cd2-based) -- the branch the AmSC demo proved on this same dragon, carrying the dragon cancel + traceback fixes. Trade-off: the orbit-backend cancel fix lives only on the main-based #91 branch, so a harmless KeyError-cancel line may appear on broker teardown. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- service/README.md | 6 ++++-- service/deploy/setup-broker.sh | 13 ++++++++----- service/deploy/setup-hpc-endpoint.sh | 7 +++++-- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/service/README.md b/service/README.md index 2b7d3a5..43fdc0e 100644 --- a/service/README.md +++ b/service/README.md @@ -35,8 +35,10 @@ the defaults (`dt_inference_ep` / `concurrent`). `service/deploy/` adapts the AmSC dt-complete deploy kit (same debugged constraints: dragon launcher requirement, python >= 3.12.1, SLURM_EXPORT_ENV, cert staging), pinning rhapsody's -`fix/dragon-cancel-and-traceback` branch on every tier for the -idempotent-cancel fixes this demo surfaced. +`fix/dragon-cancel-idempotent` branch (e491cd2-based) on every tier: it +carries the dragon cancel + traceback fixes and stays compatible with +the pinned dragonhpc 0.14.1 (rhapsody main's dragon backend passes +`task_logs=` to `Batch()`, which that dragon rejects). # broker host, once service/deploy/setup-broker.sh diff --git a/service/deploy/setup-broker.sh b/service/deploy/setup-broker.sh index e2daabe..a91d375 100755 --- a/service/deploy/setup-broker.sh +++ b/service/deploy/setup-broker.sh @@ -1,9 +1,8 @@ #!/usr/bin/env bash # DTaaS broker host for the xGFabric service twin (e.g. radical.3). # Run there. Adapted from the AmSC dt-complete deploy kit, which -# carries the debugged install/launch lore; deltas here: numpy for the -# by-value components, and rhapsody from the cancel-fix branch (the -# broker-side engine is what showed the teardown KeyError noise). +# carries the debugged install/launch lore; delta here: numpy for the +# by-value components. # # Once per host: broker_cert.pem / broker_key.pem / broker.token in # ~/.radical/orbit/. DT_DIR overrides the checkout+venv location. @@ -15,9 +14,13 @@ cd "$DT_DIR" && git checkout devel && git pull ./deploy/install.sh broker # pinned stack -> ./ve.demo ./ve.demo/bin/pip install -q numpy -# idempotent cancels (dragon+orbit backends); main-based, includes pools +# match the endpoint's rhapsody exactly (e491cd2-based): main breaks the +# endpoint's dragon backend (task_logs= to Batch), so both tiers pin the +# proven branch. The dragon cancel fix on it is a no-op broker-side; the +# orbit-backend cancel fix lives only on the main-based #91 branch, so a +# harmless KeyError-cancel line may appear on teardown here. ./ve.demo/bin/pip install -q --force-reinstall --no-deps \ - "rhapsody-py[telemetry] @ git+https://github.com/radical-cybertools/rhapsody@fix/dragon-cancel-and-traceback" + "rhapsody-py[telemetry] @ git+https://github.com/radical-cybertools/rhapsody@fix/dragon-cancel-idempotent" echo "done. start the broker with:" echo " cd $DT_DIR && ./deploy/run-broker.sh \$PWD/ve.demo" diff --git a/service/deploy/setup-hpc-endpoint.sh b/service/deploy/setup-hpc-endpoint.sh index 35b5340..73694bd 100755 --- a/service/deploy/setup-hpc-endpoint.sh +++ b/service/deploy/setup-hpc-endpoint.sh @@ -20,9 +20,12 @@ cd "$DT_DIR" && git checkout devel && git pull ./deploy/install.sh endpoint # pinned stack -> ./ve.demo # the surrogate/sink task bodies unpickle and run here ./ve.demo/bin/pip install -q numpy matplotlib -# dragon backend + idempotent cancels + failure-traceback logging +# dragon backend + idempotent cancel + failure-traceback logging. +# e491cd2-based (NOT main): main's dragon backend passes task_logs= to +# Batch(), which the pinned dragonhpc 0.14.1 does not accept. This is +# the branch the AmSC demo proved on the same dragon. ./ve.demo/bin/pip install -q --force-reinstall --no-deps \ - "rhapsody-py[telemetry,dragon] @ git+https://github.com/radical-cybertools/rhapsody@fix/dragon-cancel-and-traceback" + "rhapsody-py[telemetry,dragon] @ git+https://github.com/radical-cybertools/rhapsody@fix/dragon-cancel-idempotent" mkdir -p ~/.radical/orbit scp "$BROKER:.radical/orbit/broker_cert.pem" "$BROKER:.radical/orbit/broker.token" ~/.radical/orbit/ From 709e54a01b0a679cff13e7a5e52e49968170a287 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Thu, 3 Sep 2026 15:55:50 +0200 Subject: [PATCH 06/33] service twin: surrogate retraining runs on its own learning engine The twin declared only an inference engine, so every task -- retraining included -- shared one dashboard lane. twin_service now declares a learning engine (same endpoint, concurrent executor by default; env knobs DT_LEARNING_ENDPOINT/_BACKEND), and each SurrogateInvestigator's train task carries backend="learning" so it routes there and shows as its own lane; inference, profiling and selection stay on the inference engine. learn_backend threads agent -> investigator and defaults to None (no label) so the components still work on a single-engine session. Verified locally: rhapsody..inference and rhapsody..learning both register, training routes to learning, selection still converges on pcr, no errors. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- service/investigators.py | 9 +++++++-- service/wind_agent.py | 8 ++++---- twin_service.py | 9 ++++++++- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/service/investigators.py b/service/investigators.py index 22f88d3..1c56476 100644 --- a/service/investigators.py +++ b/service/investigators.py @@ -33,14 +33,19 @@ class SurrogateInvestigator(ModelInvestigator): """One fake surrogate; ``arch`` picks its cost profile.""" - def __init__(self, flow, arch: str): + def __init__(self, flow, arch: str, learn_backend: str | None = None): super().__init__(flow) self.flow = flow self.arch = arch self.batch: list = [] train_s, infer_s = ARCH_COST[arch] - @flow.function_task + # Route retraining to the 'learning' engine when the session has + # one, so it shows in its own dashboard lane; inference stays on + # the default (inference) engine. backend=None is the plain + # default -- asyncflow has no aliasing, so a label is set only + # when the caller declares the matching engine. + @flow.function_task(backend=learn_backend) async def train(arch, points, sim): # endpoint-side "training": cost is the architecture's await asyncio.sleep(train_s) diff --git a/service/wind_agent.py b/service/wind_agent.py index 50ce09d..ad9894b 100644 --- a/service/wind_agent.py +++ b/service/wind_agent.py @@ -23,13 +23,13 @@ class ServiceWindFieldAgent(SciAgent): - def __init__(self, flow): + def __init__(self, flow, learn_backend: str | None = None): super().__init__(flow) self.flow = flow - self.fno = SurrogateInvestigator(flow, "fno") - self.pinn = SurrogateInvestigator(flow, "pinn") - self.pcr = SurrogateInvestigator(flow, "pcr") + self.fno = SurrogateInvestigator(flow, "fno", learn_backend) + self.pinn = SurrogateInvestigator(flow, "pinn", learn_backend) + self.pcr = SurrogateInvestigator(flow, "pcr", learn_backend) @flow.function_task async def simulate(sensor_pt): diff --git a/twin_service.py b/twin_service.py index ff28e08..a8f9bae 100644 --- a/twin_service.py +++ b/twin_service.py @@ -59,11 +59,18 @@ INFERENCE_EP = os.environ.get("DT_INFERENCE_ENDPOINT", "dt_inference_ep") INFERENCE_BE = os.environ.get("DT_INFERENCE_BACKEND", "concurrent") +# learning defaults to the same endpoint on the concurrent executor: its +# own dashboard lane for the surrogate retraining, no second dragon +# runtime, and the training tasks stay clear of dragon's mp bridge +LEARNING_EP = os.environ.get("DT_LEARNING_ENDPOINT", INFERENCE_EP) +LEARNING_BE = os.environ.get("DT_LEARNING_BACKEND", "concurrent") ENGINES = { "engines": { "inference": {"endpoint_name": INFERENCE_EP, "backends": [INFERENCE_BE]}, + "learning": {"endpoint_name": LEARNING_EP, + "backends": [LEARNING_BE]}, } } @@ -79,7 +86,7 @@ def main(args) -> int: twin = dt.create_twin() print(f"[client] twin: {twin}") - agent = dt.package(ServiceWindFieldAgent) + agent = dt.package(ServiceWindFieldAgent, learn_backend="learning") profiler = dt.package(ServiceProfiler) pi = dt.package(ServicePiPredictor) sink = dt.package(ServiceSink) From 651e1c54c092d7e110f89e9edc274c801c97560a Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Thu, 3 Sep 2026 17:05:25 +0200 Subject: [PATCH 07/33] deploy: real-workload endpoint kit (conda env, one run script, two modes) Level A provisioning for the actual FNO/PINN/PCR trainings, which need TensorFlow. Rather than build it, setup-hpc-endpoint-real.sh clones Ben's cfdaai conda env (which carries the stack) and installs our runtime into the CLONE -- never his shared env -- checks out the tasks tree the profiler shells into, stages the Pi-predictor dataset, and fails fast if the env is not Python 3.12 (wire contract). run-hpc-endpoint.sh now serves both modes: DT_VENV selects the venv (default ve.demo for the faked demo, the conda clone for real), and XGF_DIR puts the tasks tree on PYTHONPATH. Broker and client are unchanged -- they stay TF-free. README documents the path and the two gating items still open: lazy TensorFlow imports in the investigator wrappers (so packaging on client/broker does not need TF) and cloudpickle parity between the conda clone and ve.demo. Those are code changes on the investigators, tracked separately. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- service/README.md | 35 ++++++++++ service/deploy/run-hpc-endpoint.sh | 16 ++++- service/deploy/setup-hpc-endpoint-real.sh | 78 +++++++++++++++++++++++ 3 files changed, 128 insertions(+), 1 deletion(-) create mode 100755 service/deploy/setup-hpc-endpoint-real.sh diff --git a/service/README.md b/service/README.md index 43fdc0e..c1d17c7 100644 --- a/service/README.md +++ b/service/README.md @@ -58,6 +58,41 @@ The client venv must be the same Python minor (digital.twins `./deploy/install.sh client` + `pip install numpy`); a 3.13 venv is rejected at the first verb. +## Real workload (Level A: real training, faked simulation) + +The fake `service/*` components run the DTaaS mechanics end to end. The +real path swaps in the actual FNO/PINN/PCR investigators (TensorFlow / +scikit-learn) and the real profiler, still on precalc simulation data +(`/global/cfs/cdirs/m5290/precalc_sims`) -- real OpenFOAM (Level B, +`cups_structure.zip`) is a later step. + +Only the **endpoint** changes: the trainings need TensorFlow, so it runs +in a clone of Ben's `cfdaai` conda env (which carries the stack) with our +runtime installed into the clone -- built by `setup-hpc-endpoint-real.sh`, +then launched via `run-hpc-endpoint.sh` with `DT_VENV`/`XGF_DIR` set (the +setup script prints the exact line). Broker and client stay on `ve.demo` +and need no TensorFlow. + + # Perlmutter login node, once (clones cfdaai, installs our runtime, + # checks out the tasks tree, stages the profiler dataset) + service/deploy/setup-hpc-endpoint-real.sh + # then in an allocation, per the line it prints: + DT_VENV= XGF_DIR= \ + service/deploy/run-hpc-endpoint.sh + +Two things are NOT done yet and gate an actual real run: + +- **Lazy TensorFlow imports.** The real investigator wrappers + `import tensorflow` at module load, which would drag TF onto the + client and broker just to package/instantiate the class. The wrappers + need their heavy imports deferred into the task bodies (which run on + the endpoint, where TF lives). Until that lands, only the endpoint + tier is provisioned; the driver still uses the fake components. +- **cloudpickle parity.** The cloned conda env and `ve.demo` may carry + different cloudpickle versions; align them (the endpoint install pulls + our pinned stack, but verify against the client) or unpickling the + shipped classes can fail. + ## What maps to what | twin.py (standalone) | twin_service.py (DTaaS) | diff --git a/service/deploy/run-hpc-endpoint.sh b/service/deploy/run-hpc-endpoint.sh index a826572..cf8fd0e 100755 --- a/service/deploy/run-hpc-endpoint.sh +++ b/service/deploy/run-hpc-endpoint.sh @@ -8,10 +8,17 @@ # # The endpoint registers as 'hpc' -- what the client's remote profile # (client-env.sh remote) asks for. DT_DIR as in setup-hpc-endpoint.sh. +# +# Two modes, same script: +# faked demo -- DT_VENV unset: uses $DT_DIR/ve.demo (setup-hpc-endpoint.sh) +# real workload-- DT_VENV=, XGF_DIR=: the +# real trainings need TensorFlow (Ben's cloned cfdaai env) +# and the profiler/task bodies import from the tasks tree +# (setup-hpc-endpoint-real.sh prints the exact invocation). set -euo pipefail BROKER="${1:?usage: $0 }" DT_DIR="${DT_DIR:-${SCRATCH:-$HOME}/digital_twins}" -VENV="$DT_DIR/ve.demo" +VENV="${DT_VENV:-$DT_DIR/ve.demo}" # dragon resolves its helpers BY NAME through srun on the task side export PATH="$VENV/bin:$PATH" @@ -24,5 +31,12 @@ export DT_STREAM_BACKEND=orbit # heatmaps land on scratch where available export XGF_WORKSPACE="${XGF_WORKSPACE:-${SCRATCH:-$HOME}/xgf_twin}" +# real workload: the profiler shells out to a script in the xGFabric tasks +# tree and the (lazy-imported) task bodies import tasks.* -- put the +# checkout on PYTHONPATH when XGF_DIR names one +if [ -n "${XGF_DIR:-}" ]; then + export PYTHONPATH="$XGF_DIR:${PYTHONPATH:-}" +fi + "$VENV/bin/dragon" "$VENV/bin/radical-orbit-endpoint.py" -n hpc \ 2>&1 | tee endpoint.log diff --git a/service/deploy/setup-hpc-endpoint-real.sh b/service/deploy/setup-hpc-endpoint-real.sh new file mode 100755 index 0000000..7ec27cc --- /dev/null +++ b/service/deploy/setup-hpc-endpoint-real.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Real-workload HPC endpoint for the xGFabric service twin (Perlmutter). +# Run on a login node. +# +# ./setup-hpc-endpoint-real.sh +# +# Unlike setup-hpc-endpoint.sh (faked components, plain ve.demo), the real +# FNO/PINN/PCR trainings need TensorFlow + the CFD/ML stack. Rather than +# build that, we CLONE Ben's cfdaai conda env (which has it) and install +# our runtime into the clone -- never into his shared env. +# +# Prerequisites (read access): /global/common/software/m5290 (Ben's envs +# + cups_structure.zip) and /global/cfs/cdirs/m5290/precalc_sims. +# `conda` must be on PATH (module load, or source Ben's mconda). +# +# Overrides: DT_DIR (our checkout), XGF_DIR (xGFabric checkout), +# DT_ENV (clone location), DT_BASE_ENV (base env name, default cfdaai). +set -euo pipefail +BROKER="${1:?usage: $0 }" +DT_DIR="${DT_DIR:-${SCRATCH:-$HOME}/digital_twins}" +XGF_DIR="${XGF_DIR:-${SCRATCH:-$HOME}/xGFabric}" +ENV_PREFIX="${DT_ENV:-${SCRATCH:-$HOME}/dt-endpoint-env}" +BEN_ENVS="/global/common/software/m5290/bcarter/mconda/envs" +BASE_ENV="${DT_BASE_ENV:-cfdaai}" + +command -v conda >/dev/null || { + echo "ERROR: conda not on PATH -- module load python, or source Ben's" >&2 + echo " mconda, then re-run." >&2; exit 1; } + +# clone Ben's env (never install into his shared copy) +conda config --add envs_dirs "$BEN_ENVS" 2>/dev/null || true +if [ ! -d "$ENV_PREFIX" ]; then + echo "==> cloning $BASE_ENV -> $ENV_PREFIX (this is large; once)" + conda create -y -p "$ENV_PREFIX" --clone "$BEN_ENVS/$BASE_ENV" +fi +PY="$ENV_PREFIX/bin/python" + +# wire contract: the same Python minor on every tier +ver="$("$PY" -c 'import sys; print("%d.%d" % sys.version_info[:2])')" +if [ "$ver" != "3.12" ]; then + echo "ERROR: $BASE_ENV is Python $ver; the DT wire contract pins 3.12" >&2 + echo " on every host. Set DT_BASE_ENV to a 3.12 env (try" >&2 + echo " xgfabric), or rebuild from its environment.yml on 3.12." >&2 + exit 1 +fi + +# our runtime into the clone -- git-pinned deps first (as deploy/install.sh +# does), then digitaltwin resolves the rest (asyncflow, orbit) from PyPI +[ -d "$DT_DIR" ] || git clone https://github.com/radical-cybertools/digital.twins.git "$DT_DIR" +( cd "$DT_DIR" && git checkout devel && git pull ) +"$PY" -m pip install -q \ + "rose @ git+https://github.com/radical-cybertools/ROSE@64330d9cb43c3e13ca67daf0d8ae84a2ae6c3f17" +"$PY" -m pip install -q --force-reinstall --no-deps \ + "rhapsody-py[telemetry,dragon] @ git+https://github.com/radical-cybertools/rhapsody@fix/dragon-cancel-idempotent" +"$PY" -m pip install -q "$DT_DIR" + +# xGFabric tasks tree: the profiler shells out to a script here, and the +# real task bodies import tasks.* at run time (run-hpc-endpoint.sh puts +# XGF_DIR on PYTHONPATH) +[ -d "$XGF_DIR" ] || git clone https://github.com/radical-collaboration/xGFabric.git "$XGF_DIR" +( cd "$XGF_DIR" && git checkout feature/dtaas-twin && git pull ) +# the Pi predictor's clean-slate dataset (Ben: the sample IS the real one) +cp -n "$XGF_DIR/tasks/profiler/pi_profiler/data.csv.sample" \ + "$XGF_DIR/tasks/profiler/pi_profiler/data.csv" 2>/dev/null || true + +# broker cert + token +mkdir -p ~/.radical/orbit +scp "$BROKER:.radical/orbit/broker_cert.pem" "$BROKER:.radical/orbit/broker.token" ~/.radical/orbit/ + +cat < Date: Fri, 4 Sep 2026 11:00:42 +0200 Subject: [PATCH 08/33] sink: render the heatmap to bytes and surface it to the dashboard The sink renders on the endpoint and returns the PNG bytes inline (a downscaled ~16 KB image, well under the return-value cap -- the DT service has no file staging), then records it via runtime.record_output so it shows in the dashboard's Outputs panel as fields arrive. Still written to XGF_WORKSPACE on the endpoint too. Needs digitaltwin's record_output (digital.twins#38). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- service/sink.py | 52 +++++++++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/service/sink.py b/service/sink.py index 9dea9ab..e26e5dc 100644 --- a/service/sink.py +++ b/service/sink.py @@ -1,11 +1,14 @@ """Terminal component: heatmap of the selected surrogate's wind field. -Same output as ``tasks.sink.CUPS_Sink`` -- the workspace resolves at -runtime on the executing host (XGF_WORKSPACE, default the host's home), -because this module ships by value and a client-side path does not exist -where the component runs. +The heatmap is rendered on the endpoint and returned inline (small PNG +bytes) so the runtime can surface it to the dashboard via +`record_output` -- the DT service has no file staging, and a downscaled +heatmap is well under the return-value cap. It is also written to +XGF_WORKSPACE on the endpoint for the record (resolved at runtime, since +this module ships by value). """ +import base64 import os from pathlib import Path @@ -26,32 +29,43 @@ def __init__(self, flow): self.count = 0 @flow.function_task - async def save_heatmap(data, arch, w, fname): + async def render_heatmap(data, arch, w, fname): + import io + import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt - plt.figure(figsize=(6, 5)) + fig = plt.figure(figsize=(4, 3.2)) im = plt.imshow(data, cmap="viridis", origin="lower", vmin=0, vmax=2.5) plt.colorbar(im) - plt.title(f"Heatmap of {arch} at Z=3, W={round(w, 3)}") + plt.title(f"{arch} W={round(w, 3)}") plt.xlabel("X") plt.ylabel("Y") - plt.savefig(fname, dpi=150, bbox_inches="tight") - plt.close() - return fname - self._save = save_heatmap + # to the endpoint filesystem, for the record + fig.savefig(fname, dpi=90, bbox_inches="tight") + # and to memory, small, for the inline return + buf = io.BytesIO() + fig.savefig(buf, format="png", dpi=72, bbox_inches="tight") + plt.close(fig) + return buf.getvalue() + + self._render = render_heatmap async def main_loop(self, runtime, in_data: TypedData): arch = in_data.data["arch"] print(f"[sink] field from {arch}", flush=True) - # if arch == "na": - # return - - # self.count += 1 - # fname = str(_workspace() / f"field_{self.count:04d}_{arch}.png") - # await self._save(in_data.data["result"][1], arch, - # in_data.data["w"], fname) - # print("\n" + "=" * 30 + f"\n{fname}\n" + "=" * 30, flush=True) + if arch == "na" or not in_data.data.get("result"): + return + + self.count += 1 + fname = str(_workspace() / f"field_{self.count:04d}_{arch}.png") + png = await self._render(in_data.data["result"][1], arch, + in_data.data["w"], fname) + + # surface it to the dashboard -- small enough to ride inline + b64 = base64.b64encode(png).decode("ascii") + runtime.record_output(f"field {self.count} ({arch})", + f"data:image/png;base64,{b64}") From bbddf04f49880ee2a5cd310687ca7452f359ed7b Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Thu, 3 Sep 2026 17:20:54 +0200 Subject: [PATCH 09/33] real workload: lazy heavy imports + a real-component service driver Lazy imports so the real investigators package/instantiate on the client and broker without TensorFlow (it loads only in the task bodies, which run on the endpoint): - the three surrogate wrappers (fno/pinn/pcr) defer `from .main import ...` into their task bodies; - each do_*/__init__.py replaces eager `from .main import *` with a PEP 562 __getattr__, so importing a submodule no longer drags TF / scikit-learn in, while `tasks..tk_*` still resolves on access (wrapper.py's executable path, on the endpoint). Verified: the three investigators + wind_agent + sink + profiler all import TF-free (pyspot submodule + dotenv are the only remaining client/broker deps, both light). twin_service_real.py mirrors twin.py's graph with the real FNO/PINN/PCR investigators, real profiler + Pi predictor, real sink, and the external fake sensor -- experimental, the on-Perlmutter starting point. setup-hpc-endpoint-real.sh inits the pyspot submodule; README documents the client deps and the two open items (shared-filesystem topology, cloudpickle parity). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- service/README.md | 31 +++-- service/deploy/setup-hpc-endpoint-real.sh | 3 +- tasks/do_fno/__init__.py | 10 +- tasks/do_fno/fno_investigator.py | 9 +- tasks/do_pcr/__init__.py | 10 +- tasks/do_pcr/pcr_investigator.py | 11 +- tasks/do_pinn/__init__.py | 10 +- tasks/do_pinn/pinn_investigator.py | 7 +- twin_service_real.py | 154 ++++++++++++++++++++++ 9 files changed, 225 insertions(+), 20 deletions(-) create mode 100644 twin_service_real.py diff --git a/service/README.md b/service/README.md index c1d17c7..732f2e6 100644 --- a/service/README.md +++ b/service/README.md @@ -80,18 +80,27 @@ and need no TensorFlow. DT_VENV= XGF_DIR= \ service/deploy/run-hpc-endpoint.sh -Two things are NOT done yet and gate an actual real run: - -- **Lazy TensorFlow imports.** The real investigator wrappers - `import tensorflow` at module load, which would drag TF onto the - client and broker just to package/instantiate the class. The wrappers - need their heavy imports deferred into the task bodies (which run on - the endpoint, where TF lives). Until that lands, only the endpoint - tier is provisioned; the driver still uses the fake components. +Driver: `twin_service_real.py` wires the real components (done, but +experimental -- not yet validated end to end; it is the on-Perlmutter +starting point). The lazy-import refactor is in place, so the real +investigators import TF-free on the client/broker; TF loads only in the +task bodies on the endpoint. + +Client/broker still need the light deps the real components carry: + + git submodule update --init --recursive # pyspot (sensor) + /bin/pip install python-dotenv numpy pandas + +Two items remain open before a real run is trustworthy: + +- **Shared filesystem.** The real components write to + `config['PLAYGROUND_DIR']` from both main_loops (broker) and tasks + (endpoint); those line up only on a shared filesystem, so run the + **broker on Perlmutter too** for the real workload (PLAYGROUND_DIR on + `$SCRATCH`). A broker on radical.3 splits the playground across hosts. - **cloudpickle parity.** The cloned conda env and `ve.demo` may carry - different cloudpickle versions; align them (the endpoint install pulls - our pinned stack, but verify against the client) or unpickling the - shipped classes can fail. + different cloudpickle versions; align them or unpickling the shipped + classes can fail. ## What maps to what diff --git a/service/deploy/setup-hpc-endpoint-real.sh b/service/deploy/setup-hpc-endpoint-real.sh index 7ec27cc..e053887 100755 --- a/service/deploy/setup-hpc-endpoint-real.sh +++ b/service/deploy/setup-hpc-endpoint-real.sh @@ -58,7 +58,8 @@ fi # real task bodies import tasks.* at run time (run-hpc-endpoint.sh puts # XGF_DIR on PYTHONPATH) [ -d "$XGF_DIR" ] || git clone https://github.com/radical-collaboration/xGFabric.git "$XGF_DIR" -( cd "$XGF_DIR" && git checkout feature/dtaas-twin && git pull ) +( cd "$XGF_DIR" && git checkout feature/dtaas-twin && git pull \ + && git submodule update --init --recursive ) # pyspot (davis/sensor) # the Pi predictor's clean-slate dataset (Ben: the sample IS the real one) cp -n "$XGF_DIR/tasks/profiler/pi_profiler/data.csv.sample" \ "$XGF_DIR/tasks/profiler/pi_profiler/data.csv" 2>/dev/null || true diff --git a/tasks/do_fno/__init__.py b/tasks/do_fno/__init__.py index 15b6a64..0791656 100644 --- a/tasks/do_fno/__init__.py +++ b/tasks/do_fno/__init__.py @@ -1 +1,9 @@ -from .main import * +# Lazy re-export: `from .main import *` pulled TensorFlow/scikit-learn at +# package import, which forced those onto the client and broker just to +# import a submodule (e.g. the investigator wrapper). PEP 562 defers it +# to attribute access -- `tasks..tk_*` still resolves (on the +# endpoint, where the stack lives; used by wrapper.py), but importing +# `tasks..` no longer drags the heavy deps in. +def __getattr__(name): + from . import main + return getattr(main, name) diff --git a/tasks/do_fno/fno_investigator.py b/tasks/do_fno/fno_investigator.py index 80e7e1c..f82fbfa 100644 --- a/tasks/do_fno/fno_investigator.py +++ b/tasks/do_fno/fno_investigator.py @@ -4,7 +4,11 @@ from radical.asyncflow import WorkflowEngine from digitaltwin.components import Any, ModelInvestigator, TypedData from digitaltwin.runtime import DTRuntime, RuntimeAPI -from .main import tk_do_fno, tk_fno_eval + +# NOTE: `.main` pulls TensorFlow at import. It is imported lazily inside +# the task bodies below (which run on the endpoint, where TF lives), so +# packaging/instantiating this class on the client and broker stays +# TF-free. See service/README.md "Real workload". from rose.al.active_learner import Learner @@ -52,6 +56,8 @@ async def do_inference(in_data: TypedData, config=None, model="na"): # pass through, no model yet return TypedData(WIND_FIELD, {"arch": "na"}) + from .main import tk_fno_eval + # model available! # logger.info(f"Do FNO inference: {model}") wind = in_data.data["wind_speed"] @@ -76,6 +82,7 @@ async def do_train(self, runtime: RuntimeAPI, batch): @self.learner.training_task(as_executable=False) async def do_fno(config, sims): + from .main import tk_do_fno return tk_do_fno(config, sims) self.config["TASK_COUNTER"] = self.task_counter diff --git a/tasks/do_pcr/__init__.py b/tasks/do_pcr/__init__.py index 15b6a64..0791656 100644 --- a/tasks/do_pcr/__init__.py +++ b/tasks/do_pcr/__init__.py @@ -1 +1,9 @@ -from .main import * +# Lazy re-export: `from .main import *` pulled TensorFlow/scikit-learn at +# package import, which forced those onto the client and broker just to +# import a submodule (e.g. the investigator wrapper). PEP 562 defers it +# to attribute access -- `tasks..tk_*` still resolves (on the +# endpoint, where the stack lives; used by wrapper.py), but importing +# `tasks..` no longer drags the heavy deps in. +def __getattr__(name): + from . import main + return getattr(main, name) diff --git a/tasks/do_pcr/pcr_investigator.py b/tasks/do_pcr/pcr_investigator.py index d8adf20..6c3595d 100644 --- a/tasks/do_pcr/pcr_investigator.py +++ b/tasks/do_pcr/pcr_investigator.py @@ -6,9 +6,9 @@ from digitaltwin.components import Any, ModelInvestigator, TypedData from digitaltwin.runtime import RuntimeAPI -from .main import tk_pcr_eval, tk_pcr_partition -from .main import tk_do_pcr -from .main import tk_do_pcr_pack +# NOTE: `.main` pulls scikit-learn (and the PCR train stack) at import -- +# the tk_* helpers are imported lazily in the task bodies below (which run +# on the endpoint) so packaging/instantiating on client/broker stays light. from rose.al.active_learner import Learner @@ -32,6 +32,8 @@ async def do_inference(in_data: TypedData, config=None, model="na"): # PCR works on a range rather than a single data point. # for now, assume the wind is stable. + from .main import tk_pcr_eval + wind_single = in_data.data["wind_speed"] wind = np.ones(13) * wind_single result = tk_pcr_eval(config, model, wind) @@ -87,14 +89,17 @@ async def do_train(self, runtime: RuntimeAPI, batch): @self.flow.function_task async def partition(config, sims, sensor_vals): + from .main import tk_pcr_partition return tk_pcr_partition(config, sims, sensor_vals) @self.flow.function_task async def do_pcr(config, machine_data_output): + from .main import tk_do_pcr return tk_do_pcr(config, machine_data_output) @self.flow.function_task async def do_pack(config, *pcr_output_dirs): + from .main import tk_do_pcr_pack return tk_do_pcr_pack(config, *pcr_output_dirs) self.config["TASK_NAME"] = "partition" diff --git a/tasks/do_pinn/__init__.py b/tasks/do_pinn/__init__.py index 15b6a64..0791656 100644 --- a/tasks/do_pinn/__init__.py +++ b/tasks/do_pinn/__init__.py @@ -1 +1,9 @@ -from .main import * +# Lazy re-export: `from .main import *` pulled TensorFlow/scikit-learn at +# package import, which forced those onto the client and broker just to +# import a submodule (e.g. the investigator wrapper). PEP 562 defers it +# to attribute access -- `tasks..tk_*` still resolves (on the +# endpoint, where the stack lives; used by wrapper.py), but importing +# `tasks..` no longer drags the heavy deps in. +def __getattr__(name): + from . import main + return getattr(main, name) diff --git a/tasks/do_pinn/pinn_investigator.py b/tasks/do_pinn/pinn_investigator.py index fe82984..59aad7f 100644 --- a/tasks/do_pinn/pinn_investigator.py +++ b/tasks/do_pinn/pinn_investigator.py @@ -4,7 +4,9 @@ from radical.asyncflow import WorkflowEngine from digitaltwin.components import Any, ModelInvestigator, TypedData from digitaltwin.runtime import RuntimeAPI -from .main import tk_do_pinn, tk_pinn_eval + +# NOTE: `.main` pulls TensorFlow at import -- imported lazily in the task +# bodies below (which run on the endpoint) so client/broker stay TF-free. from rose.al.active_learner import Learner @@ -54,6 +56,8 @@ async def do_inference(in_data: TypedData, config=None, model="na"): # pass through, no model yet return TypedData(WIND_FIELD, {"arch": "na"}) + from .main import tk_pinn_eval + # model available! wind = in_data.data["wind_speed"] result = tk_pinn_eval(config, model, wind) @@ -76,6 +80,7 @@ async def do_train(self, runtime: RuntimeAPI, batch): @self.learner.training_task(as_executable=False) async def do_pinn(config, sims): + from .main import tk_do_pinn return tk_do_pinn(config, sims) self.config["TASK_COUNTER"] = self.task_counter diff --git a/twin_service_real.py b/twin_service_real.py new file mode 100644 index 0000000..1c6a2f8 --- /dev/null +++ b/twin_service_real.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""The xGFabric twin as a service, with the REAL workload. + +twin.py's graph, run on a DTaaS broker like twin_service.py -- but the +field agent, its FNO/PINN/PCR surrogates, the profiler and the Pi +predictor are the real components from tasks/, not the service/* fakes. +Physics stays where twin.py already fakes it (the sensor, and +tk_do_simulation's precalc-CSV shortcut); the trainings, the profiler +timing, and the selection are real. + +STATUS: experimental scaffold, NOT yet validated end to end. It is the +starting point for the on-Perlmutter test cycle, not a proven driver. +Known open items (see service/README.md "Real workload"): + + * Endpoint must run in a TF-carrying env (Ben's cfdaai clone) -- + setup-hpc-endpoint-real.sh. The lazy-import refactor keeps the + client/broker TF-free, but they still need dotenv + numpy + pandas + and the pyspot submodule (`git submodule update --init`). + * Shared-filesystem assumption: the real components write to + config['PLAYGROUND_DIR'] both from main_loops (broker side) and + from tasks (endpoint side). Those line up only if broker and + endpoint share a filesystem -- so run the BROKER on Perlmutter too + for the real workload, with PLAYGROUND_DIR on $SCRATCH. A broker on + radical.3 will split the playground across two hosts. + * Backends default to 'concurrent' here: real TF/sklearn under dragon + is untested (joblib's mp bridge, TF process model). Flip to + dragon_v3 via the env knobs once it is proven. + +Environment: config.sh (tasks/common/config.sh) supplies PLAYGROUND_DIR, +CSPOT_LIMIT, endpoint/model paths etc., loaded via dotenv as in twin.py. +Client wire env as usual: RADICAL_ORBIT_BROKER_URL(+_CERT), +DT_STREAM_BACKEND=orbit, DT_SERVICE_HOST. +""" + +import argparse +import os +import sys +import time + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import dotenv + +from radical.orbit import EndpointRuntime + +from digitaltwin.components import NULL_DTYPE, DataType, TypedData +from digitaltwin.service import register_user_modules + +# real components (TF is deferred to the task bodies -- import is light) +import tasks +import tasks.common +import tasks.common.dtypes +import tasks.wind_agent +import tasks.sink +import tasks.profiler.components +import tasks.do_fno.fno_investigator +import tasks.do_pinn.pinn_investigator +import tasks.do_pcr.pcr_investigator + +from tasks.common.dtypes import DAVIS_WIND_SENSOR, WIND_FIELD +from tasks.wind_agent import WindFieldAgent +from tasks.sink import CUPS_Sink +from tasks.profiler.components import ( + ProfilerInvestigator, + EndpointInvestigator, + TASK_DESCRIPTION_DTYPE, + PROFILE_RESULTS, +) +from service.sensor_publisher import DAVIS_CHANNEL + +PI_PREDICT_RUNTIME = DataType("pi_PREDICT_RUNTIME") + +dotenv.load_dotenv("tasks/common/config.sh") + +register_user_modules([ + tasks, tasks.common, tasks.common.dtypes, + tasks.wind_agent, tasks.sink, tasks.profiler.components, + tasks.do_fno.fno_investigator, + tasks.do_pinn.pinn_investigator, + tasks.do_pcr.pcr_investigator, +]) + +DT_HOST = os.environ.get("DT_SERVICE_HOST", "broker") + +INFERENCE_EP = os.environ.get("DT_INFERENCE_ENDPOINT", "hpc") +INFERENCE_BE = os.environ.get("DT_INFERENCE_BACKEND", "concurrent") +LEARNING_EP = os.environ.get("DT_LEARNING_ENDPOINT", INFERENCE_EP) +LEARNING_BE = os.environ.get("DT_LEARNING_BACKEND", "concurrent") + +ENGINES = { + "engines": { + "inference": {"endpoint_name": INFERENCE_EP, "backends": [INFERENCE_BE]}, + "learning": {"endpoint_name": LEARNING_EP, "backends": [LEARNING_BE]}, + } +} + + +def main(args) -> int: + config = dict(os.environ) + playground = config.get("PLAYGROUND_DIR", os.path.expanduser("~/xgf_twin")) + + runtime = EndpointRuntime() + runtime.start(wait=True) + try: + dt = runtime.get_plugin(DT_HOST, "dt", config=ENGINES) + print(f"[client] session: {dt.sid} (reattach with this sid)") + + twin = dt.create_twin() + print(f"[client] twin: {twin}") + + field = dt.package(WindFieldAgent, config) + sink = dt.package(CUPS_Sink, config) + base_profiler = dt.package( + ProfilerInvestigator, playground + "/profiler/nersc_profiler") + pi_profiler = dt.package( + EndpointInvestigator, "pi", playground + "/profiler/pi_profiler") + + # sensor is external (fake Davis, as twin.py fakes it); bind its + # channel to the input dtype + dt.add_input(twin, DAVIS_WIND_SENSOR, DAVIS_CHANNEL) + + dt.add_agent(twin, field, DAVIS_WIND_SENSOR, WIND_FIELD) + dt.add_task(twin, sink, WIND_FIELD, NULL_DTYPE) + + dt.add_investigator(twin, base_profiler, + TASK_DESCRIPTION_DTYPE, PROFILE_RESULTS) + dt.add_investigator(twin, pi_profiler, + PROFILE_RESULTS, PI_PREDICT_RUNTIME) + + dt.start(twin) + + deadline = time.time() + args.runtime + while time.time() < deadline: + time.sleep(10) + info = dt.twin(twin) + print(f"[client] state={info['state']}" + f" calls={info.get('calls') or {}}", flush=True) + if info["state"] == "failed": + print(f"[client] twin failed: {info.get('last_error')}") + return 1 + + print("SHUTDOWN") + dt.twin_close(twin) + return 0 + finally: + runtime.stop() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="xGFabric twin, service mode, REAL workload") + parser.add_argument("--runtime", type=int, default=600, + help="seconds to keep the twin running") + sys.exit(main(parser.parse_args())) From 73eaab5cefed7e68e0cf812a411f90a5e671eb3d Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 4 Sep 2026 10:48:09 +0200 Subject: [PATCH 10/33] profiler: append the Pi data.csv row on the endpoint, not the broker EndpointInvestigator.main_loop runs on the broker, but it appended new rows to data.csv that train_model (an executable task) reads on the endpoint -- different filesystems once broker and endpoint are split. The append moves into a function_task (whose body runs on the endpoint), so data.csv is produced and consumed on the same host. This is the one cross-host file hand-off that is fixable without staging. The rest of the profiler chain uses executable_tasks, whose bodies run on the engine (broker) to build the command while the command runs on the endpoint -- so the .pkl / .json inputs they write are broker-side and the commands read them endpoint-side. Bridging that is the ORBIT staging work (separate), not a filesystem assumption to fix here. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- tasks/profiler/components.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tasks/profiler/components.py b/tasks/profiler/components.py index 542d90f..491738a 100644 --- a/tasks/profiler/components.py +++ b/tasks/profiler/components.py @@ -153,6 +153,21 @@ async def train_model(): self.train_task = train_model + @self.flow.function_task + async def append_row(datastore, row): + # runs on the endpoint (a function_task body IS the task), so the + # row lands in the same data.csv that train_model (executable, + # also endpoint) reads -- see main_loop. Doing this append in + # main_loop instead would write it on the broker, on a different + # filesystem from the training task. + import os + + os.makedirs(datastore, exist_ok=True) + with open(f"{datastore}/data.csv", "a") as fh: + fh.write(row + "\n") + + self._append_row = append_row + @self.flow.executable_task async def call_inference(in_data: TypedData, model=None, name=""): # for inference, just run the simulation. @@ -224,8 +239,9 @@ async def main_loop(self, runtime: RuntimeAPI): out = ",".join([str(f) for f in nersc_profile.values()]) + "," out += pi_time - with open(f"{self.datastore}/data.csv", "a") as f: - f.write(out + "\n") + # append on the endpoint (where train_model reads data.csv), not + # here on the broker -- see append_row + await self._append_row(self.datastore, out) out = json.loads(await self.train_task()) model = out["model"] From 1f85c06c69689c03a405f16bccfcd4f4128170e9 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 4 Sep 2026 11:29:43 +0200 Subject: [PATCH 11/33] profiler: measure the inference in-process (approach 1), not via subprocess The profiler cloudpickled the inference to a file and ran the standalone profiler.py under a fresh process group. Because an executable task's command-builder runs on the broker while its command runs on the endpoint, that .pkl was written broker-side and read endpoint-side -- a cross-host hand-off needing a shared filesystem. inproc.py runs the call in-process on the endpoint and reads the process's own counters (perf_counter, getrusage, psutil.io_counters), returning the profiler's exact six-key schema so data.csv and endpoint_trainer.py are unchanged. ProfilerInvestigator and EndpointInvestigator now profile via a function-task (endpoint-side, no file, no subprocess). Verified locally: real wall/cpu/mem on a dummy inference; components import clean. Trade-off (no process-group isolation, peak RSS not child PSS) is fine for a comparative fingerprint. PLAN-telemetry-profiling.md captures approach 2 -- read the endpoint's own per-task telemetry instead of re-running -- for later, plus the remaining executable-task input hand-offs in the Pi-predictor (train/eval) that want the same treatment. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- tasks/profiler/PLAN-telemetry-profiling.md | 80 +++++++++++++++++++++ tasks/profiler/components.py | 76 +++++++------------- tasks/profiler/inproc.py | 82 ++++++++++++++++++++++ 3 files changed, 189 insertions(+), 49 deletions(-) create mode 100644 tasks/profiler/PLAN-telemetry-profiling.md create mode 100644 tasks/profiler/inproc.py diff --git a/tasks/profiler/PLAN-telemetry-profiling.md b/tasks/profiler/PLAN-telemetry-profiling.md new file mode 100644 index 0000000..567d428 --- /dev/null +++ b/tasks/profiler/PLAN-telemetry-profiling.md @@ -0,0 +1,80 @@ +# Plan: profile from endpoint telemetry (approach 2), for later + +Status: **not started.** Approach (1), in-process measurement, is live +(`inproc.py` + `ProfilerInvestigator`/`EndpointInvestigator`). This is +the follow-on that turns "profile" into "run once, read telemetry." + +## Why + +The surrogate selector needs a resource fingerprint of each candidate's +inference to predict its edge (Pi) runtime. Approach (1) measures the +call in-process with `resource.getrusage` + `psutil.io_counters`. That +works and is staging-free, but: + +- it re-runs the inference *only to measure it* (a second execution on + top of the one the twin already does for real), and +- in-process counters include a little of the task-runner's own + activity and give peak RSS, not the isolated child's PSS. + +The rhapsody endpoint **already records** per-task telemetry -- task +lifecycle plus `ResourceUpdate` (cpu %, memory %, disk read/write bytes) +polled while the task runs -- into +`telemetry-output/session.*.telemetry.jsonl` (the `[telemetry]` extra; +this is what `service/collect_reports.py` renders). So the cost of a +candidate's inference is captured for free when it runs as a normal +task. Approach (2): read that instead of re-running under a profiler. + +## Shape + +1. **Correlate a task to its telemetry.** The DT already ties a task uid + to its twin/component (`DTRuntime.note_task`, carried in `twin_list` + as `tasks`/`task_components`). The telemetry events carry the same + `task_id`. A helper reads the session jsonl and, for a given uid, + returns the `{total_seconds, cpu_seconds, disk_read_bytes, + disk_write_bytes, memory_bytes}` aggregated from that task's + `TaskStarted`/`TaskCompleted` span and its `ResourceUpdate` samples. + +2. **Profiler investigator becomes a reader, not a runner.** When the + agent asks to profile a candidate, the profiler no longer executes + the inference: it (a) ensures the candidate has run at least once + (the twin's normal inference already does this), (b) looks up that + task's uid, (c) reads the telemetry record for it, (d) returns the + same six-key profile dict. No second execution, no subprocess. + +3. **Keep the schema.** Emit the exact keys `inproc.py` and the + subprocess profiler use, so `data.csv`, `endpoint_trainer.py`, and + the Pi predictor are unchanged; approach (2) is a drop-in source + swap. + +## Open questions to resolve when building it + +- **Telemetry access from the profiler.** The jsonl lives on the + endpoint; the profiler component's `main_loop` runs on the broker. + Reading it needs either (i) a small function-task on the endpoint that + greps the jsonl for the uid and returns the record (endpoint-side, + staging-free -- preferred), or (ii) an engine-side telemetry API on + the DT service (see digital.twins#36, engine-side telemetry) that + surfaces per-task metrics to the runtime. +- **Timing / flush.** `ResourceUpdate` is polled (default ~0.5 s); a + very fast inference may produce few samples. Fall back to the task + span's wall/cpu when samples are sparse, or lower the poll interval + for the profiling window. +- **Attribution granularity.** Confirm the telemetry `task_id` matches + the uid the DT records for the *inference* task specifically (not a + wrapping flow task), mirroring the exact-attribution work the + dashboard relies on. +- **Isolation.** Telemetry measures the task as it actually ran + (shared with whatever else the endpoint was doing). For a + comparative fingerprint that is acceptable; note it if absolute + numbers ever matter. + +## Relationship to the remaining executable-task hand-offs + +The Pi-predictor's `train_model` / `call_inference` (`endpoint_trainer.py` +/ `endpoint_eval.py`) are still executable tasks whose *command-builder* +bodies prepare input files (`model.json`, `inf.json`). `data.csv` is now +endpoint-local (the append moved to a function-task); `inf.json` in +`call_inference` is still written broker-side. Those are the prediction +*model*, not profiling, and are out of scope here -- but the same fix +applies: write their inputs in a preceding function-task (endpoint-side) +or pass them as command arguments. Track that alongside this. diff --git a/tasks/profiler/components.py b/tasks/profiler/components.py index 491738a..eec7bdc 100644 --- a/tasks/profiler/components.py +++ b/tasks/profiler/components.py @@ -33,50 +33,36 @@ def __init__(self, flow: WorkflowEngine, workdir: str = "."): super().__init__(flow) self.flow = flow self.workdir = workdir - os.makedirs(self.workdir, exist_ok=True) + # no makedirs here: the profiling runs in-process on the endpoint + # (below), so there is no file to prepare on this (broker) side. - @self.flow.executable_task - async def exec_profiler(task, example_data: TypedData, model_kwargs: dict): - if model_kwargs is None: - model_kwargs = {} - print("Exec profiler request") - # fix to use a unique file name. - export_inference_function( - f"{self.workdir}/meta-profiler.pkl", task, example_data, **model_kwargs - ) + @self.flow.function_task + async def run_profiled(blob, example_data, model_kwargs): + # runs on the endpoint: reconstruct the candidate's inference + # callable and measure one call in-process -- no cloudpickle + # file, no subprocess, nothing written on the broker. + from .inproc import load_callable, profile_call - # call profiler - return shlex.join( - [ - "python3", - f"{script_path}/profiler.py", - f"{self.workdir}/meta-profiler.pkl", - ] - ) + func = load_callable(blob) + return await profile_call(func, example_data, model_kwargs or {}) sim_lock = asyncio.Lock() sim_lru = LRUCache(128) # store 128 different sims async def do_inference(in_data: TypedData): - # # for inference, just run the simulation. async with sim_lock: - # ignore example_data task, example_data, model_kwargs = in_data.data key = freeze((task, model_kwargs)) if await sim_lru.exists(key): profile = await sim_lru.fetch_item(key) - task_data = in_data.data - out = {"profile": profile, "task": task_data} - return TypedData(PROFILE_RESULTS, out) + return TypedData(PROFILE_RESULTS, + {"profile": profile, "task": in_data.data}) - result = await exec_profiler(task, example_data, model_kwargs) - r = json.loads(result) + r = await run_profiled(task, example_data, model_kwargs) await sim_lru.put_item(key, r) - - task_data = in_data.data - out = {"profile": r, "task": task_data} - return TypedData(PROFILE_RESULTS, out) + return TypedData(PROFILE_RESULTS, + {"profile": r, "task": in_data.data}) self.inference_task = do_inference @@ -118,25 +104,16 @@ def __init__( self.learner = Learner(flow) - @self.flow.executable_task + @self.flow.function_task async def exec_profiler(task_data): - task, example_data, model_kwargs = task_data + # the endpoint ("Pi") measurement, in-process on this endpoint: + # run the candidate's inference and read the counters, no + # cloudpickle file and no subprocess (approach 1). + from .inproc import load_callable, profile_call - # fix to use a unique file name. - export_inference_function( - f"{self.datastore}/profile.pkl", task, example_data, **model_kwargs - ) - - # call profiler - print("endpoint profiler request") - return shlex.join( - [ - "python3", - f"{script_path}/profiler.py", - "--csv", - f"{self.datastore}/profile.pkl", - ] - ) + task, example_data, model_kwargs = task_data + func = load_callable(task) + return await profile_call(func, example_data, model_kwargs or {}) self.exec_profiler = exec_profiler @@ -230,9 +207,10 @@ async def main_loop(self, runtime: RuntimeAPI): while True: item = await self.callback_jobs.get() - pi_out = await self.exec_profiler(item["task"]) - # I only want the first column - pi_time = pi_out.split(",", 1)[0] + # exec_profiler now returns the profile dict (in-process); the + # Pi runtime is its wall time + pi_profile = await self.exec_profiler(item["task"]) + pi_time = str(pi_profile["total_seconds"]) # label the endpoint_time as "pi_seconds" nersc_profile = item["profile"] diff --git a/tasks/profiler/inproc.py b/tasks/profiler/inproc.py new file mode 100644 index 0000000..7264c80 --- /dev/null +++ b/tasks/profiler/inproc.py @@ -0,0 +1,82 @@ +"""In-process resource profiling of an inference call. + +Approach (1) from the profiler discussion: instead of cloudpickling the +inference function to a file and running the standalone ``profiler.py`` +under a fresh process group, run the call in-process and read the +process's own resource counters around it. This runs entirely where the +task runs (the endpoint) -- no file hand-off between the executable-task +command-builder (broker) and the command (endpoint), which is what the +subprocess profiler needed a shared filesystem for. + +Trade-off vs the subprocess profiler: no process-group isolation, so the +numbers include a little of the task runner's own activity during the +call, and ``memory_bytes`` is the process peak RSS (a high-water mark) +rather than the child's PSS. For a *comparative* fingerprint across the +candidate surrogates -- which is all the selector needs -- that is fine. + +The returned dict keeps the exact keys and order the subprocess profiler +emitted, so ``data.csv`` and ``endpoint_trainer.py`` are unchanged. +""" + +import os +import resource +import time + +import cloudpickle +import psutil + + +def load_callable(blob): + """Reconstruct the inference callable the agent cloudpickled.""" + obj = cloudpickle.loads(blob) + # the agent ships either the raw callable or the {"func": ...} payload + if isinstance(obj, dict) and "func" in obj: + obj = obj["func"] + return obj + + +async def profile_call(func, example_data, kwargs=None) -> dict: + """Run ``func(example_data, **kwargs)`` once and measure its cost. + + Returns the profiler's schema: + total_seconds, cpu_seconds, disk_read_bytes, disk_write_bytes, + sys_read_bytes, memory_bytes + """ + + kwargs = kwargs or {} + proc = psutil.Process(os.getpid()) + + try: + io0 = proc.io_counters() + except Exception: + io0 = None + ru0 = resource.getrusage(resource.RUSAGE_SELF) + t0 = time.perf_counter() + + result = func(example_data, **kwargs) + if hasattr(result, "__await__"): + await result + + wall = time.perf_counter() - t0 + ru1 = resource.getrusage(resource.RUSAGE_SELF) + try: + io1 = proc.io_counters() + except Exception: + io1 = None + + def io_delta(attr): + if io0 is None or io1 is None: + return 0 + return max(0, getattr(io1, attr, 0) - getattr(io0, attr, 0)) + + return { + "total_seconds": wall, + "cpu_seconds": (ru1.ru_utime - ru0.ru_utime) + + (ru1.ru_stime - ru0.ru_stime), + "disk_read_bytes": io_delta("read_bytes"), + "disk_write_bytes": io_delta("write_bytes"), + "sys_read_bytes": io_delta("read_chars"), + # ru_maxrss is peak RSS in KiB on Linux (bytes on macOS); the peak + # high-water mark, matching the subprocess profiler's peak metric. + "memory_bytes": int(ru1.ru_maxrss) * 1024, + } From 3936706f628adeb9f836615b660cbceeb9ac350d Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 4 Sep 2026 11:38:34 +0200 Subject: [PATCH 12/33] real workload: stage inf.json endpoint-side; sink surfaces its heatmap - EndpointInvestigator.call_inference wrote inf.json in its executable-task command-builder (broker) while endpoint_eval.py reads it on the endpoint. A stage_inf function-task now writes it on the endpoint before the command runs -- the last cross-host file hand-off in the profiler chain. The datastore makedirs in __init__ (broker) is guarded; the endpoint tasks create it. - CUPS_Sink renders the heatmap to bytes and calls record_output so the real driver shows heatmaps in the dashboard Outputs panel (needs digital.twins#38), and imports matplotlib lazily so packaging on the client/broker stays light. Verified locally: all real-driver components import clean, TF-free and matplotlib-free at import. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- tasks/profiler/components.py | 28 +++++++++-- tasks/sink.py | 93 +++++++++++++++++++++--------------- 2 files changed, 78 insertions(+), 43 deletions(-) diff --git a/tasks/profiler/components.py b/tasks/profiler/components.py index eec7bdc..7b20324 100644 --- a/tasks/profiler/components.py +++ b/tasks/profiler/components.py @@ -97,7 +97,13 @@ def __init__( else: self.datastore = datastore_path - os.makedirs(self.datastore, exist_ok=True) + # NOT created here: __init__ runs on the broker, but the datastore + # is an endpoint path (where the tasks read/write it). The + # endpoint-side tasks makedirs it (stage_inf / append_row). + try: + os.makedirs(self.datastore, exist_ok=True) + except OSError: + pass self.callback_jobs: asyncio.Queue = asyncio.Queue() self.done_jobs: set = set() @@ -145,12 +151,26 @@ async def append_row(datastore, row): self._append_row = append_row + @self.flow.function_task + async def stage_inf(datastore, pf): + # write endpoint_eval's input on the endpoint (a function_task + # body runs there), so the executable command below reads it on + # the same host -- not the broker, where the executable-task + # command-builder runs. + import json as _json + import os + + os.makedirs(datastore, exist_ok=True) + with open(f"{datastore}/inf.json", "w") as fh: + _json.dump(pf, fh) + + self._stage_inf = stage_inf + @self.flow.executable_task async def call_inference(in_data: TypedData, model=None, name=""): - # for inference, just run the simulation. pf = in_data.data["profile"] - with open(f"{self.datastore}/inf.json", "w") as f: - json.dump(pf, f) + # stage the input endpoint-side before the command runs + await self._stage_inf(self.datastore, pf) return shlex.join( [ "python3", diff --git a/tasks/sink.py b/tasks/sink.py index e6af586..87fa98c 100644 --- a/tasks/sink.py +++ b/tasks/sink.py @@ -1,62 +1,77 @@ -import asyncio +"""Terminal component: heatmap of the selected surrogate's wind field. + +The heatmap renders on the endpoint and is returned inline (small PNG +bytes) so the runtime can surface it to the dashboard via +`record_output` -- the DT service has no file staging, and a downscaled +heatmap is well under the return-value cap. It is also written to +PLAYGROUND_DIR on the endpoint for the record. + +matplotlib is imported lazily inside the task body (which runs on the +endpoint) so packaging/instantiating this class on the client and broker +does not need it. +""" + +import base64 +import os from radical.asyncflow import WorkflowEngine from digitaltwin.components import UtilityTask, TypedData from .common.dtypes import * import logging -import matplotlib.pyplot as plt logger = logging.getLogger(__name__) -def graph(data, fname, model_name, z, w): - - plt.figure(figsize=(6, 5)) - - # Plot heatmap - im = plt.imshow(data, cmap="viridis", origin="lower", vmin=0, vmax=2.5) - - # Add colorbar - plt.colorbar(im) - - # Optional labels - plt.title(f"Heatmap of {model_name} at Z={z}, W={round(w,3)}") - plt.xlabel("X") - plt.ylabel("Y") - - # Save image - plt.savefig(fname, dpi=300, bbox_inches="tight") - - plt.close() - - class CUPS_Sink(UtilityTask): def __init__(self, flow: WorkflowEngine, config): super().__init__(flow) self.flow = flow self.config = config + self.count = 0 @self.flow.function_task - async def save_photo(in_data, config): - fname = config["PLAYGROUND_DIR"] + "/out.png" - graph( - in_data.data["result"][1], - fname, - in_data.data["arch"], - 3, - w=in_data.data["w"], - ) - print("\n\n" + "=" * 30 + "\n" + str(fname) + "\n" + "=" * 30) - - self.save_photo = save_photo + async def render(field, arch, w, fname): + import io + + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + fig = plt.figure(figsize=(4, 3.2)) + im = plt.imshow(field, cmap="viridis", origin="lower", + vmin=0, vmax=2.5) + plt.colorbar(im) + plt.title(f"{arch} W={round(w, 3)}") + plt.xlabel("X") + plt.ylabel("Y") + + if fname: + os.makedirs(os.path.dirname(fname), exist_ok=True) + fig.savefig(fname, dpi=150, bbox_inches="tight") + buf = io.BytesIO() + fig.savefig(buf, format="png", dpi=72, bbox_inches="tight") + plt.close(fig) + return buf.getvalue() + + self._render = render async def main_loop(self, runtime, in_data: TypedData): - print(f"Received Inference: {in_data.data['arch']}") + arch = in_data.data["arch"] + print(f"Received Inference: {arch}") - if in_data.data["arch"] == "na": + if arch == "na" or not in_data.data.get("result"): print("No surrogate ready yet") return - # create graph - await self.save_photo(in_data, self.config) + self.count += 1 + fname = os.path.join(self.config.get("PLAYGROUND_DIR", "."), + f"out_{self.count:04d}.png") + png = await self._render(in_data.data["result"][1], arch, + in_data.data["w"], fname) + + # surface it to the dashboard -- small enough to ride inline + b64 = base64.b64encode(png).decode("ascii") + runtime.record_output(f"field {self.count} ({arch})", + f"data:image/png;base64,{b64}") + print("\n" + "=" * 30 + f"\n{fname}\n" + "=" * 30) From 36d7369fc2e345649b2c125a7f4db6ef19092c35 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 4 Sep 2026 12:01:12 +0200 Subject: [PATCH 13/33] real demo config: point the endpoint tasks tree and PLAYGROUND_DIR at us - setup-hpc-endpoint-real.sh checks out feature/dtaas-twin-real for the endpoint tasks tree -- the base branch lacks the in-process profiler, the inf.json staging fix and the output-emitting sink. - PLAYGROUND_DIR -> /pscratch/sd/m/merzky/xgf_playground (a real PM scratch path; the driver reads it from config.sh on the client and ships it to the components, which use it endpoint-side). Demo-specific values; to be generalised later. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- service/deploy/setup-hpc-endpoint-real.sh | 2 +- tasks/common/config.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/service/deploy/setup-hpc-endpoint-real.sh b/service/deploy/setup-hpc-endpoint-real.sh index e053887..89ba638 100755 --- a/service/deploy/setup-hpc-endpoint-real.sh +++ b/service/deploy/setup-hpc-endpoint-real.sh @@ -58,7 +58,7 @@ fi # real task bodies import tasks.* at run time (run-hpc-endpoint.sh puts # XGF_DIR on PYTHONPATH) [ -d "$XGF_DIR" ] || git clone https://github.com/radical-collaboration/xGFabric.git "$XGF_DIR" -( cd "$XGF_DIR" && git checkout feature/dtaas-twin && git pull \ +( cd "$XGF_DIR" && git checkout feature/dtaas-twin-real && git pull \ && git submodule update --init --recursive ) # pyspot (davis/sensor) # the Pi predictor's clean-slate dataset (Ben: the sample IS the real one) cp -n "$XGF_DIR/tasks/profiler/pi_profiler/data.csv.sample" \ diff --git a/tasks/common/config.sh b/tasks/common/config.sh index e274c09..9cd236d 100644 --- a/tasks/common/config.sh +++ b/tasks/common/config.sh @@ -12,7 +12,7 @@ NODE_COUNT=4 # ROSE / Python config -PLAYGROUND_DIR=/pscratch/sd/b/bcarter/playground_dt +PLAYGROUND_DIR=/pscratch/sd/m/merzky/xgf_playground COMMON_DIR="../common" From a0f0481c9ed6a186302a68d9c8cebbb595b89d28 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 4 Sep 2026 12:31:11 +0200 Subject: [PATCH 14/33] deploy: force-reinstall orbit on the endpoint; add client deps script - setup-hpc-endpoint-real.sh force-reinstalls radical.orbit: a cloned conda env may already carry a stale one that pip treats as satisfied, so the endpoint script (radical-orbit-endpoint.py) never installs. - setup-client.sh installs the light client-side deps the real components import at packaging time (python-dotenv, numpy, pandas) and inits the pyspot submodule; TensorFlow/matplotlib stay lazy. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- service/deploy/setup-client.sh | 21 +++++++++++++++++++++ service/deploy/setup-hpc-endpoint-real.sh | 3 +++ 2 files changed, 24 insertions(+) create mode 100755 service/deploy/setup-client.sh diff --git a/service/deploy/setup-client.sh b/service/deploy/setup-client.sh new file mode 100755 index 0000000..e28f72f --- /dev/null +++ b/service/deploy/setup-client.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Client-side deps for the real service twin, on top of digital.twins' +# `./deploy/install.sh client` (which provides the DT stack + rose). +# +# ./setup-client.sh [ve.demo path] +# +# The real components import these at packaging time -- TensorFlow and +# matplotlib stay lazy (endpoint-only), so the client needs only the +# light ones plus the pyspot submodule the sensor uses. +set -euo pipefail +VENV="${1:-$HOME/radical/digital_twins/ve.demo}" + +"$VENV/bin/pip" install -q python-dotenv numpy pandas + +# pyspot (tasks/common/pyspot) is a git submodule -- davis.py imports it +git submodule update --init --recursive + +echo "done. run from this checkout:" +echo " source service/deploy/client-env.sh remote" +echo " $VENV/bin/python service/sensor_publisher.py # terminal 1" +echo " $VENV/bin/python twin_service_real.py # terminal 2" diff --git a/service/deploy/setup-hpc-endpoint-real.sh b/service/deploy/setup-hpc-endpoint-real.sh index 89ba638..93435f4 100755 --- a/service/deploy/setup-hpc-endpoint-real.sh +++ b/service/deploy/setup-hpc-endpoint-real.sh @@ -52,6 +52,9 @@ fi "rose @ git+https://github.com/radical-cybertools/ROSE@64330d9cb43c3e13ca67daf0d8ae84a2ae6c3f17" "$PY" -m pip install -q --force-reinstall --no-deps \ "rhapsody-py[telemetry,dragon] @ git+https://github.com/radical-cybertools/rhapsody@fix/dragon-cancel-idempotent" +# force-reinstall orbit: a cloned env may already carry a stale radical.orbit +# that pip would treat as satisfied, leaving the endpoint script uninstalled +"$PY" -m pip install -q --force-reinstall --no-deps "radical.orbit>=0.7" "$PY" -m pip install -q "$DT_DIR" # xGFabric tasks tree: the profiler shells out to a script here, and the From 24ed0e4226d7dd0594ae3ca29413a3950e35cc1b Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 4 Sep 2026 14:19:20 +0200 Subject: [PATCH 15/33] deploy: orbit log to scratch (HOME quota); backfill orbit deps - run-hpc-endpoint.sh points RADICAL_ORBIT_LOG_FILE at scratch and sets the level to WARNING: the default ~/.radical/orbit/logs is on NERSC HOME, whose tiny quota the endpoint log (esp. at debug) blows out -- the registration itself succeeds but every log write then errors. - setup-hpc-endpoint-real.sh backfills orbit's own deps after the --no-deps script reinstall (fastapi/uvicorn/websockets), so a clone missing them still starts. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- service/deploy/run-hpc-endpoint.sh | 5 +++++ service/deploy/setup-hpc-endpoint-real.sh | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/service/deploy/run-hpc-endpoint.sh b/service/deploy/run-hpc-endpoint.sh index cf8fd0e..dd66be1 100755 --- a/service/deploy/run-hpc-endpoint.sh +++ b/service/deploy/run-hpc-endpoint.sh @@ -30,6 +30,11 @@ export SLURM_EXPORT_ENV=ALL # inner sruns must not scrub the env export DT_STREAM_BACKEND=orbit # heatmaps land on scratch where available export XGF_WORKSPACE="${XGF_WORKSPACE:-${SCRATCH:-$HOME}/xgf_twin}" +# the orbit log defaults to ~/.radical/orbit/logs, which is HOME -- a tiny +# quota on NERSC. Put it on scratch and keep the level modest. +export RADICAL_ORBIT_LOG_LVL="${RADICAL_ORBIT_LOG_LVL:-WARNING}" +export RADICAL_ORBIT_LOG_FILE="${RADICAL_ORBIT_LOG_FILE:-${SCRATCH:-$HOME}/orbit-logs/hpc.log}" +mkdir -p "$(dirname "$RADICAL_ORBIT_LOG_FILE")" # real workload: the profiler shells out to a script in the xGFabric tasks # tree and the (lazy-imported) task bodies import tasks.* -- put the diff --git a/service/deploy/setup-hpc-endpoint-real.sh b/service/deploy/setup-hpc-endpoint-real.sh index 93435f4..063854b 100755 --- a/service/deploy/setup-hpc-endpoint-real.sh +++ b/service/deploy/setup-hpc-endpoint-real.sh @@ -53,8 +53,11 @@ fi "$PY" -m pip install -q --force-reinstall --no-deps \ "rhapsody-py[telemetry,dragon] @ git+https://github.com/radical-cybertools/rhapsody@fix/dragon-cancel-idempotent" # force-reinstall orbit: a cloned env may already carry a stale radical.orbit -# that pip would treat as satisfied, leaving the endpoint script uninstalled +# that pip would treat as satisfied, leaving the endpoint script uninstalled. +# --no-deps places the script/version; the plain install backfills orbit's +# own deps (fastapi, uvicorn, websockets) without disturbing the pins above. "$PY" -m pip install -q --force-reinstall --no-deps "radical.orbit>=0.7" +"$PY" -m pip install -q "radical.orbit>=0.7" "$PY" -m pip install -q "$DT_DIR" # xGFabric tasks tree: the profiler shells out to a script here, and the From 212944d3cc6818d82017d554ae8a97b13b7bc1c0 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 4 Sep 2026 14:28:07 +0200 Subject: [PATCH 16/33] deploy: install rhapsody LAST, with extras -- fixes task_logs + opentelemetry Two regressions from the conda-clone endpoint: - pip install "$DT_DIR" (digitaltwin) ran after the rhapsody branch pin and pulled a different (main-based) rhapsody over it, reintroducing the Batch(task_logs=...) crash. rhapsody now installs LAST so the dragon-compatible branch wins. - the --no-deps rhapsody install skipped the telemetry extra's opentelemetry. A second, extras install (no --no-deps) backfills opentelemetry + dragonhpc without re-resolving the pinned stack. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- service/deploy/setup-hpc-endpoint-real.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/service/deploy/setup-hpc-endpoint-real.sh b/service/deploy/setup-hpc-endpoint-real.sh index 063854b..1bc8f95 100755 --- a/service/deploy/setup-hpc-endpoint-real.sh +++ b/service/deploy/setup-hpc-endpoint-real.sh @@ -48,10 +48,10 @@ fi # does), then digitaltwin resolves the rest (asyncflow, orbit) from PyPI [ -d "$DT_DIR" ] || git clone https://github.com/radical-cybertools/digital.twins.git "$DT_DIR" ( cd "$DT_DIR" && git checkout devel && git pull ) +RH="rhapsody-py[telemetry,dragon] @ git+https://github.com/radical-cybertools/rhapsody@fix/dragon-cancel-idempotent" + "$PY" -m pip install -q \ "rose @ git+https://github.com/radical-cybertools/ROSE@64330d9cb43c3e13ca67daf0d8ae84a2ae6c3f17" -"$PY" -m pip install -q --force-reinstall --no-deps \ - "rhapsody-py[telemetry,dragon] @ git+https://github.com/radical-cybertools/rhapsody@fix/dragon-cancel-idempotent" # force-reinstall orbit: a cloned env may already carry a stale radical.orbit # that pip would treat as satisfied, leaving the endpoint script uninstalled. # --no-deps places the script/version; the plain install backfills orbit's @@ -59,6 +59,14 @@ fi "$PY" -m pip install -q --force-reinstall --no-deps "radical.orbit>=0.7" "$PY" -m pip install -q "radical.orbit>=0.7" "$PY" -m pip install -q "$DT_DIR" +# rhapsody LAST, so the dragon-compatible branch wins over whatever +# digitaltwin's install pulled (main's dragon backend passes task_logs= to +# Batch(), which the pinned dragonhpc rejects). Two steps: --no-deps pins +# the branch build, then the extras install backfills opentelemetry +# (telemetry) and dragonhpc (dragon) without re-resolving the rest. +"$PY" -m pip install -q --force-reinstall --no-deps \ + "rhapsody-py @ git+https://github.com/radical-cybertools/rhapsody@fix/dragon-cancel-idempotent" +"$PY" -m pip install -q "$RH" # xGFabric tasks tree: the profiler shells out to a script here, and the # real task bodies import tasks.* at run time (run-hpc-endpoint.sh puts From e110f4484d7f2c25567a14b06cf4b9c043e91812 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 4 Sep 2026 14:41:15 +0200 Subject: [PATCH 17/33] deploy: fold every real-demo fix into the scripts (idempotent, site-specific) So a plain re-run of setup + run does everything, no one-off commands: - setup-hpc-endpoint-real.sh seeds the Pi dataset at the RUNTIME datastore (PLAYGROUND_DIR/profiler/pi_profiler, where endpoint_trainer reads it), not just the tasks tree; PLAYGROUND_DIR is a script var matching config.sh. - run-hpc-endpoint.sh clears any old HOME orbit log (NERSC quota) on top of redirecting the log to scratch at WARNING. - setup-broker.sh backfills the telemetry extra's deps after the --no-deps rhapsody pin, mirroring the endpoint. Paths are NERSC/demo-specific for now, to be generalised later. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- service/deploy/run-hpc-endpoint.sh | 4 +++- service/deploy/setup-broker.sh | 4 ++++ service/deploy/setup-hpc-endpoint-real.sh | 11 +++++++++-- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/service/deploy/run-hpc-endpoint.sh b/service/deploy/run-hpc-endpoint.sh index dd66be1..bd8e5bd 100755 --- a/service/deploy/run-hpc-endpoint.sh +++ b/service/deploy/run-hpc-endpoint.sh @@ -31,10 +31,12 @@ export DT_STREAM_BACKEND=orbit # heatmaps land on scratch where available export XGF_WORKSPACE="${XGF_WORKSPACE:-${SCRATCH:-$HOME}/xgf_twin}" # the orbit log defaults to ~/.radical/orbit/logs, which is HOME -- a tiny -# quota on NERSC. Put it on scratch and keep the level modest. +# quota on NERSC. Put it on scratch and keep the level modest; also clear +# any old HOME log so a full quota there cannot block the run. export RADICAL_ORBIT_LOG_LVL="${RADICAL_ORBIT_LOG_LVL:-WARNING}" export RADICAL_ORBIT_LOG_FILE="${RADICAL_ORBIT_LOG_FILE:-${SCRATCH:-$HOME}/orbit-logs/hpc.log}" mkdir -p "$(dirname "$RADICAL_ORBIT_LOG_FILE")" +rm -f "$HOME/.radical/orbit/logs/"*.log 2>/dev/null || true # real workload: the profiler shells out to a script in the xGFabric tasks # tree and the (lazy-imported) task bodies import tasks.* -- put the diff --git a/service/deploy/setup-broker.sh b/service/deploy/setup-broker.sh index a91d375..c10a5c7 100755 --- a/service/deploy/setup-broker.sh +++ b/service/deploy/setup-broker.sh @@ -21,6 +21,10 @@ cd "$DT_DIR" && git checkout devel && git pull # harmless KeyError-cancel line may appear on teardown here. ./ve.demo/bin/pip install -q --force-reinstall --no-deps \ "rhapsody-py[telemetry] @ git+https://github.com/radical-cybertools/rhapsody@fix/dragon-cancel-idempotent" +# backfill the telemetry extra's deps (opentelemetry) in case a fresh +# install did not carry them +./ve.demo/bin/pip install -q \ + "rhapsody-py[telemetry] @ git+https://github.com/radical-cybertools/rhapsody@fix/dragon-cancel-idempotent" echo "done. start the broker with:" echo " cd $DT_DIR && ./deploy/run-broker.sh \$PWD/ve.demo" diff --git a/service/deploy/setup-hpc-endpoint-real.sh b/service/deploy/setup-hpc-endpoint-real.sh index 1bc8f95..3af7ee4 100755 --- a/service/deploy/setup-hpc-endpoint-real.sh +++ b/service/deploy/setup-hpc-endpoint-real.sh @@ -22,6 +22,10 @@ XGF_DIR="${XGF_DIR:-${SCRATCH:-$HOME}/xGFabric}" ENV_PREFIX="${DT_ENV:-${SCRATCH:-$HOME}/dt-endpoint-env}" BEN_ENVS="/global/common/software/m5290/bcarter/mconda/envs" BASE_ENV="${DT_BASE_ENV:-cfdaai}" +# demo-specific: must match PLAYGROUND_DIR in tasks/common/config.sh (the +# client ships that value to the components, which use it here on the +# endpoint). The Pi-predictor's datastore is seeded under it below. +PLAYGROUND_DIR="${PLAYGROUND_DIR:-${SCRATCH:-$HOME}/xgf_playground}" command -v conda >/dev/null || { echo "ERROR: conda not on PATH -- module load python, or source Ben's" >&2 @@ -74,9 +78,12 @@ RH="rhapsody-py[telemetry,dragon] @ git+https://github.com/radical-cybertools/rh [ -d "$XGF_DIR" ] || git clone https://github.com/radical-collaboration/xGFabric.git "$XGF_DIR" ( cd "$XGF_DIR" && git checkout feature/dtaas-twin-real && git pull \ && git submodule update --init --recursive ) # pyspot (davis/sensor) -# the Pi predictor's clean-slate dataset (Ben: the sample IS the real one) +# the Pi predictor's clean-slate dataset (Ben: the sample IS the real one), +# seeded at the RUNTIME datastore under PLAYGROUND_DIR -- that is where +# endpoint_trainer.py reads it (the tasks-tree copy is not that path) +mkdir -p "$PLAYGROUND_DIR/profiler/pi_profiler" cp -n "$XGF_DIR/tasks/profiler/pi_profiler/data.csv.sample" \ - "$XGF_DIR/tasks/profiler/pi_profiler/data.csv" 2>/dev/null || true + "$PLAYGROUND_DIR/profiler/pi_profiler/data.csv" 2>/dev/null || true # broker cert + token mkdir -p ~/.radical/orbit From 826a8cf65c3413c9ecad7a4c5ef79c2a5b12ef53 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 4 Sep 2026 15:02:33 +0200 Subject: [PATCH 18/33] demo: consolidated one-command-per-role Sep_04 scripts Five self-contained scripts, each printing the demo banner and running all-that's-needed for its role (deploy fix + env + data dirs + run): Sep_04_broker.sh radical.3: install stack, pin dragon rhapsody, run broker Sep_04_endpoint_setup.sh PM login node: checkouts, cfdaai clone, runtime, seed data Sep_04_endpoint.sh PM allocation: env + launch endpoint under dragon Sep_04_sensor.sh client: publish davis-wind Sep_04_client.sh client: light deps + drive twin_service_real.py Args are only the broker IP (and 'hpc' endpoint name where relevant); everything else (SCRATCH paths, branch, PLAYGROUND_DIR, conda base env) is baked in for the demo. The endpoint is split because the allocation cannot pip-install; setup runs on a login node first. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- demo/Sep_04_broker.sh | 34 ++++++++++++++++ demo/Sep_04_client.sh | 42 +++++++++++++++++++ demo/Sep_04_endpoint.sh | 41 +++++++++++++++++++ demo/Sep_04_endpoint_setup.sh | 77 +++++++++++++++++++++++++++++++++++ demo/Sep_04_sensor.sh | 26 ++++++++++++ 5 files changed, 220 insertions(+) create mode 100755 demo/Sep_04_broker.sh create mode 100755 demo/Sep_04_client.sh create mode 100755 demo/Sep_04_endpoint.sh create mode 100755 demo/Sep_04_endpoint_setup.sh create mode 100755 demo/Sep_04_sensor.sh diff --git a/demo/Sep_04_broker.sh b/demo/Sep_04_broker.sh new file mode 100755 index 0000000..2580b2f --- /dev/null +++ b/demo/Sep_04_broker.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# xGFabric demo -- DTaaS broker. Run on the broker host (radical.3). +# Sets up (install + fixes) then starts the broker. Idempotent. +# +# demo/Sep_04_broker.sh [broker-ip] +# +# Assumes broker_cert.pem / broker_key.pem / broker.token in ~/.radical/orbit/. +set -euo pipefail + +BROKER_IP="${1:-95.217.193.116}" + +# --- demo/site specific ---------------------------------------------------- +DT_DIR="${DT_DIR:-$HOME/digital_twins}" +RH="rhapsody-py[telemetry] @ git+https://github.com/radical-cybertools/rhapsody@fix/dragon-cancel-idempotent" +# --------------------------------------------------------------------------- + +echo "--------------------------" +echo "xGFabric Demo September 04" +echo "Broker on $BROKER_IP ($(hostname -f)) | Endpoint HPC 'hpc' | Sensor davis-wind | Client twin_service_real.py" +echo "--------------------------" + +# checkout + install the pinned stack, then pin the dragon-compatible +# rhapsody branch (extras backfilled) +[ -d "$DT_DIR" ] || git clone https://github.com/radical-cybertools/digital.twins.git "$DT_DIR" +( cd "$DT_DIR" && git checkout devel && git pull ) +( cd "$DT_DIR" + ./deploy/install.sh broker + ./ve.demo/bin/pip install -q numpy + ./ve.demo/bin/pip install -q --force-reinstall --no-deps \ + "rhapsody-py @ git+https://github.com/radical-cybertools/rhapsody@fix/dragon-cancel-idempotent" + ./ve.demo/bin/pip install -q "$RH" ) + +echo "starting broker (wss://0.0.0.0:8000) ..." +exec "$DT_DIR/deploy/run-broker.sh" "$DT_DIR/ve.demo" diff --git a/demo/Sep_04_client.sh b/demo/Sep_04_client.sh new file mode 100755 index 0000000..b3bd95b --- /dev/null +++ b/demo/Sep_04_client.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# xGFabric demo -- client driver. Run on the client. Ensures the light +# client-side deps, then drives the real twin against the HPC endpoint. +# +# demo/Sep_04_client.sh [broker-ip] +set -euo pipefail + +BROKER_IP="${1:-95.217.193.116}" + +# --- demo/site specific ---------------------------------------------------- +VENV="${DT_VENV:-$HOME/radical/digital_twins/ve.demo}" +DT_DIR="${DT_DIR:-$HOME/radical/digital_twins}" +HERE="$(cd "$(dirname "$0")/.." && pwd)" # xGFabric checkout root +RUNTIME="${RUNTIME:-600}" +# --------------------------------------------------------------------------- + +echo "--------------------------" +echo "xGFabric Demo September 04" +echo "Broker on $BROKER_IP | Endpoint HPC 'hpc' | Sensor davis-wind | Client twin_service_real.py ($(hostname -f))" +echo "--------------------------" + +cd "$HERE" + +# client-side deps the real components import at packaging time (TF and +# matplotlib stay lazy / endpoint-only); pyspot submodule for the sensor. +"$VENV/bin/pip" install -q python-dotenv numpy pandas +git submodule update --init --recursive +# keep the client digitaltwin current (record_output etc.) +( cd "$DT_DIR" && git checkout devel && git pull ) && "$VENV/bin/pip" install -q "$DT_DIR" + +# placement: run the twin's tasks on the HPC endpoint (dragon) +export RADICAL_ORBIT_BROKER_URL="wss://$BROKER_IP:8000" +export RADICAL_ORBIT_BROKER_CERT="$HOME/.radical/orbit/broker_cert.pem" +export DT_STREAM_BACKEND=orbit +export DT_INFERENCE_ENDPOINT=hpc +export DT_INFERENCE_BACKEND=dragon_v3 +export DT_LEARNING_ENDPOINT=hpc +export DT_LEARNING_BACKEND=concurrent + +echo "driving the real twin (runtime ${RUNTIME}s) ..." +echo "dashboard: https://$BROKER_IP:8000/broker/dt/ui?live=1" +exec "$VENV/bin/python" twin_service_real.py --runtime "$RUNTIME" diff --git a/demo/Sep_04_endpoint.sh b/demo/Sep_04_endpoint.sh new file mode 100755 index 0000000..c3e429d --- /dev/null +++ b/demo/Sep_04_endpoint.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# xGFabric demo -- Perlmutter endpoint RUN. Run INSIDE the compute +# allocation (no pip here; Sep_04_endpoint_setup.sh did that on the login +# node). Launches the rhapsody endpoint under the dragon launcher. +# +# demo/Sep_04_endpoint.sh [broker-ip] [endpoint-name] +set -euo pipefail + +BROKER_IP="${1:-95.217.193.116}" +EP="${2:-hpc}" + +# --- demo/site specific ---------------------------------------------------- +ENV_PREFIX="${DT_ENV:-$SCRATCH/dt-endpoint-env}" +XGF_DIR="${XGF_DIR:-$SCRATCH/xgfabric}" +PLAYGROUND_DIR="${PLAYGROUND_DIR:-$SCRATCH/xgf_playground}" +# --------------------------------------------------------------------------- + +echo "--------------------------" +echo "xGFabric Demo September 04" +echo "Broker on $BROKER_IP | Endpoint HPC '$EP' on $(hostname -f) | Sensor davis-wind | Client twin_service_real.py" +echo "--------------------------" + +# dragon resolves its helpers BY NAME via srun on the task side +export PATH="$ENV_PREFIX/bin:$PATH" +export PYTHONPATH="$XGF_DIR:${PYTHONPATH:-}" +export RADICAL_ORBIT_BROKER_URL="wss://$BROKER_IP:8000" +export RADICAL_ORBIT_BROKER_CERT="$HOME/.radical/orbit/broker_cert.pem" +export RADICAL_ORBIT_RHAPSODY_BACKEND=dragon_v3 +export RADICAL_ORBIT_RHAPSODY_NOTIFY_WINDOW=0 +export SLURM_EXPORT_ENV=ALL # inner sruns must not scrub the env +export DT_STREAM_BACKEND=orbit +export XGF_WORKSPACE="$PLAYGROUND_DIR" +# keep the orbit log off HOME (tiny NERSC quota) and modest +export RADICAL_ORBIT_LOG_LVL="${RADICAL_ORBIT_LOG_LVL:-WARNING}" +export RADICAL_ORBIT_LOG_FILE="$SCRATCH/orbit-logs/$EP.log" +mkdir -p "$SCRATCH/orbit-logs" "$PLAYGROUND_DIR" +rm -f "$HOME/.radical/orbit/logs/"*.log 2>/dev/null || true + +echo "launching endpoint '$EP' under dragon ..." +exec "$ENV_PREFIX/bin/dragon" "$ENV_PREFIX/bin/radical-orbit-endpoint.py" -n "$EP" \ + 2>&1 | tee "$SCRATCH/orbit-logs/$EP.console.log" diff --git a/demo/Sep_04_endpoint_setup.sh b/demo/Sep_04_endpoint_setup.sh new file mode 100755 index 0000000..134b8f0 --- /dev/null +++ b/demo/Sep_04_endpoint_setup.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# xGFabric demo -- Perlmutter endpoint SETUP. Run on a LOGIN NODE (it +# pip-installs and clones; the allocation cannot). Idempotent. +# +# demo/Sep_04_endpoint_setup.sh [broker-ip] (broker-ip only for the banner) +# +# Assumes keys/token already staged in ~/.radical/orbit/. +set -euo pipefail + +BROKER_IP="${1:-95.217.193.116}" +EP=hpc + +# --- demo/site specific ---------------------------------------------------- +BRANCH=feature/dtaas-twin-real +DT_DIR="${DT_DIR:-$SCRATCH/digital_twins}" +XGF_DIR="${XGF_DIR:-$SCRATCH/xgfabric}" +ENV_PREFIX="${DT_ENV:-$SCRATCH/dt-endpoint-env}" +PLAYGROUND_DIR="${PLAYGROUND_DIR:-$SCRATCH/xgf_playground}" +BEN_ENVS="/global/common/software/m5290/bcarter/mconda/envs" +BASE_ENV=cfdaai +RH="rhapsody-py[telemetry,dragon] @ git+https://github.com/radical-cybertools/rhapsody@fix/dragon-cancel-idempotent" +ROSE="rose @ git+https://github.com/radical-cybertools/ROSE@64330d9cb43c3e13ca67daf0d8ae84a2ae6c3f17" +# --------------------------------------------------------------------------- + +echo "--------------------------" +echo "xGFabric Demo September 04 -- ENDPOINT SETUP (login node)" +echo "Broker $BROKER_IP | Endpoint '$EP' | env $ENV_PREFIX | playground $PLAYGROUND_DIR" +echo "--------------------------" + +module load python/3.12 2>/dev/null || true +command -v conda >/dev/null || { echo "ERROR: conda not on PATH" >&2; exit 1; } + +# checkouts +[ -d "$DT_DIR" ] || git clone https://github.com/radical-cybertools/digital.twins.git "$DT_DIR" +( cd "$DT_DIR" && git checkout devel && git pull ) +[ -d "$XGF_DIR" ] || git clone https://github.com/radical-collaboration/xGFabric.git "$XGF_DIR" +( cd "$XGF_DIR" && git checkout "$BRANCH" && git pull && git submodule update --init --recursive ) + +# conda env: a clone of Ben's cfdaai (TensorFlow + CFD/ML stack), never his +conda config --add envs_dirs "$BEN_ENVS" 2>/dev/null || true +[ -d "$ENV_PREFIX" ] || conda create -y -p "$ENV_PREFIX" --clone "$BEN_ENVS/$BASE_ENV" +PY="$ENV_PREFIX/bin/python" + +ver="$("$PY" -c 'import sys;print("%d.%d"%sys.version_info[:2])')" +[ "$ver" = "3.12" ] || { echo "ERROR: $BASE_ENV is Python $ver, need 3.12" >&2; exit 1; } + +# our runtime into the clone. orbit force-reinstalled (a clone may carry a +# stale one that hides the endpoint script), then deps backfilled. +echo "==> installing runtime into $ENV_PREFIX" +"$PY" -m pip install -q "$ROSE" +"$PY" -m pip install -q --force-reinstall --no-deps "radical.orbit>=0.7" +"$PY" -m pip install -q "radical.orbit>=0.7" +"$PY" -m pip install -q "$DT_DIR" +# rhapsody LAST so the dragon-compatible branch wins over what digitaltwin +# pulled (main passes task_logs= to Batch(), which the pinned dragonhpc +# rejects); extras step backfills opentelemetry + dragonhpc. +"$PY" -m pip install -q --force-reinstall --no-deps \ + "rhapsody-py @ git+https://github.com/radical-cybertools/rhapsody@fix/dragon-cancel-idempotent" +"$PY" -m pip install -q "$RH" + +# data dirs: seed the Pi-predictor dataset at the RUNTIME datastore (where +# endpoint_trainer.py reads it) +mkdir -p "$PLAYGROUND_DIR/profiler/pi_profiler" +cp -n "$XGF_DIR/tasks/profiler/pi_profiler/data.csv.sample" \ + "$PLAYGROUND_DIR/profiler/pi_profiler/data.csv" 2>/dev/null || true + +# sanity +echo "==> verify" +"$PY" -c "import radical.orbit, digitaltwin, rhapsody, opentelemetry; print(' imports ok')" +"$PY" -c "import rhapsody.backends.execution.dragon as d; n=open(d.__file__).read().count('task_logs'); print(' dragon task_logs (want 0):', n)" +[ -x "$ENV_PREFIX/bin/radical-orbit-endpoint.py" ] && echo " endpoint script ok" || echo " WARN: endpoint script missing" +[ -x "$ENV_PREFIX/bin/dragon" ] && echo " dragon ok" || echo " WARN: dragon missing" + +echo "--------------------------" +echo "done. get an allocation, then run:" +echo " $XGF_DIR/demo/Sep_04_endpoint.sh $BROKER_IP $EP" +echo "--------------------------" diff --git a/demo/Sep_04_sensor.sh b/demo/Sep_04_sensor.sh new file mode 100755 index 0000000..ee6d53b --- /dev/null +++ b/demo/Sep_04_sensor.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# xGFabric demo -- external wind sensor (fake Davis). Run on the client. +# Publishes to the twin's input channel over the ORBIT data plane. +# +# demo/Sep_04_sensor.sh [broker-ip] +set -euo pipefail + +BROKER_IP="${1:-95.217.193.116}" + +# --- demo/site specific ---------------------------------------------------- +VENV="${DT_VENV:-$HOME/radical/digital_twins/ve.demo}" +HERE="$(cd "$(dirname "$0")/.." && pwd)" # xGFabric checkout root +# --------------------------------------------------------------------------- + +echo "--------------------------" +echo "xGFabric Demo September 04" +echo "Broker on $BROKER_IP | Endpoint HPC 'hpc' | Sensor davis-wind ($(hostname -f)) | Client twin_service_real.py" +echo "--------------------------" + +export RADICAL_ORBIT_BROKER_URL="wss://$BROKER_IP:8000" +export RADICAL_ORBIT_BROKER_CERT="$HOME/.radical/orbit/broker_cert.pem" +export DT_STREAM_BACKEND=orbit + +cd "$HERE" +echo "publishing davis-wind to channel xgf/davis ..." +exec "$VENV/bin/python" service/sensor_publisher.py From 8b6c196c25fb9baec718b753ec00304fc933e990 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 4 Sep 2026 15:09:20 +0200 Subject: [PATCH 19/33] demo: banner shows only each script's own role Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- demo/Sep_04_broker.sh | 2 +- demo/Sep_04_client.sh | 2 +- demo/Sep_04_endpoint.sh | 2 +- demo/Sep_04_sensor.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/demo/Sep_04_broker.sh b/demo/Sep_04_broker.sh index 2580b2f..d0ca6da 100755 --- a/demo/Sep_04_broker.sh +++ b/demo/Sep_04_broker.sh @@ -16,7 +16,7 @@ RH="rhapsody-py[telemetry] @ git+https://github.com/radical-cybertools/rhapsody@ echo "--------------------------" echo "xGFabric Demo September 04" -echo "Broker on $BROKER_IP ($(hostname -f)) | Endpoint HPC 'hpc' | Sensor davis-wind | Client twin_service_real.py" +echo "Broker on $BROKER_IP ($(hostname -f))" echo "--------------------------" # checkout + install the pinned stack, then pin the dragon-compatible diff --git a/demo/Sep_04_client.sh b/demo/Sep_04_client.sh index b3bd95b..7ac01f0 100755 --- a/demo/Sep_04_client.sh +++ b/demo/Sep_04_client.sh @@ -16,7 +16,7 @@ RUNTIME="${RUNTIME:-600}" echo "--------------------------" echo "xGFabric Demo September 04" -echo "Broker on $BROKER_IP | Endpoint HPC 'hpc' | Sensor davis-wind | Client twin_service_real.py ($(hostname -f))" +echo "Client twin_service_real.py ($(hostname -f)) -> broker $BROKER_IP, endpoint HPC 'hpc'" echo "--------------------------" cd "$HERE" diff --git a/demo/Sep_04_endpoint.sh b/demo/Sep_04_endpoint.sh index c3e429d..3def3f8 100755 --- a/demo/Sep_04_endpoint.sh +++ b/demo/Sep_04_endpoint.sh @@ -17,7 +17,7 @@ PLAYGROUND_DIR="${PLAYGROUND_DIR:-$SCRATCH/xgf_playground}" echo "--------------------------" echo "xGFabric Demo September 04" -echo "Broker on $BROKER_IP | Endpoint HPC '$EP' on $(hostname -f) | Sensor davis-wind | Client twin_service_real.py" +echo "Endpoint HPC '$EP' on $(hostname -f) -> broker $BROKER_IP" echo "--------------------------" # dragon resolves its helpers BY NAME via srun on the task side diff --git a/demo/Sep_04_sensor.sh b/demo/Sep_04_sensor.sh index ee6d53b..f144d96 100755 --- a/demo/Sep_04_sensor.sh +++ b/demo/Sep_04_sensor.sh @@ -14,7 +14,7 @@ HERE="$(cd "$(dirname "$0")/.." && pwd)" # xGFabric checkout root echo "--------------------------" echo "xGFabric Demo September 04" -echo "Broker on $BROKER_IP | Endpoint HPC 'hpc' | Sensor davis-wind ($(hostname -f)) | Client twin_service_real.py" +echo "Sensor davis-wind ($(hostname -f)) -> broker $BROKER_IP" echo "--------------------------" export RADICAL_ORBIT_BROKER_URL="wss://$BROKER_IP:8000" From 021134152a0b93e1f8ea82b8c0d7762948b46079 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 4 Sep 2026 15:23:02 +0200 Subject: [PATCH 20/33] twin_service_real: ship whole tasks.* subtree by value add_agent failed on the broker with ModuleNotFoundError: no module named 'utils_architecture'. WindFieldAgent references tasks.davis (and tasks.do_simulation, tasks.common.log_formatter), which import the repo-root utils_architecture; the curated register_user_modules list missed them, so they pickled by reference and the broker -- which has no xGFabric checkout -- could not import them. Sweep every already imported tasks.* module plus utils_architecture instead. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- twin_service_real.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/twin_service_real.py b/twin_service_real.py index 1c6a2f8..1bd42de 100644 --- a/twin_service_real.py +++ b/twin_service_real.py @@ -72,12 +72,15 @@ dotenv.load_dotenv("tasks/common/config.sh") +# Ship every user module in the agent's reference graph by value: the +# broker has no xGFabric checkout, so anything pickled by reference +# (tasks.davis -> utils_architecture, tasks.do_simulation, the common +# helpers, ...) fails to import there. Sweep the whole tasks.* subtree +# plus the repo-root helpers rather than curate a list that drifts. register_user_modules([ - tasks, tasks.common, tasks.common.dtypes, - tasks.wind_agent, tasks.sink, tasks.profiler.components, - tasks.do_fno.fno_investigator, - tasks.do_pinn.pinn_investigator, - tasks.do_pcr.pcr_investigator, + m for name, m in list(sys.modules.items()) + if m is not None and (name == "tasks" or name.startswith("tasks.") + or name == "utils_architecture") ]) DT_HOST = os.environ.get("DT_SERVICE_HOST", "broker") From a6dbe2cfeae9bfd5490d82e58d2b6cec595465fe Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 4 Sep 2026 15:25:53 +0200 Subject: [PATCH 21/33] demo: broker venv needs pandas (agent runs broker-side) Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- demo/Sep_04_broker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demo/Sep_04_broker.sh b/demo/Sep_04_broker.sh index d0ca6da..f15e2f8 100755 --- a/demo/Sep_04_broker.sh +++ b/demo/Sep_04_broker.sh @@ -25,7 +25,7 @@ echo "--------------------------" ( cd "$DT_DIR" && git checkout devel && git pull ) ( cd "$DT_DIR" ./deploy/install.sh broker - ./ve.demo/bin/pip install -q numpy + ./ve.demo/bin/pip install -q numpy pandas # the agent runs broker-side ./ve.demo/bin/pip install -q --force-reinstall --no-deps \ "rhapsody-py @ git+https://github.com/radical-cybertools/rhapsody@fix/dragon-cancel-idempotent" ./ve.demo/bin/pip install -q "$RH" ) From 2f9005383d945d49c2f0b9deb07e6757001b339c Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 4 Sep 2026 15:27:51 +0200 Subject: [PATCH 22/33] wind_agent: logger optional (service instantiates flow+config only) The DTaaS service builds agents as cls(flow, *args, **engines); the required 'logger' positional made add_agent raise TypeError. Default it to the module logger. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- tasks/wind_agent.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tasks/wind_agent.py b/tasks/wind_agent.py index ae32fa8..fb5cb54 100644 --- a/tasks/wind_agent.py +++ b/tasks/wind_agent.py @@ -25,8 +25,11 @@ class WindFieldAgent(SciAgent): - def __init__(self, flow: WorkflowEngine, config: dict, logger): + def __init__(self, flow: WorkflowEngine, config: dict, logger=None): super().__init__(flow) + # the DTaaS service instantiates with flow + config only; fall + # back to the module logger when no logger is injected + logger = logger or logging.getLogger(__name__) self.flow = flow self.config = config.copy() self.config["AGENT_NAME"] = "WindField" From 531b44cfe50a709c82ac57062780d218e0dcbd26 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 4 Sep 2026 16:32:21 +0200 Subject: [PATCH 23/33] profiler: invoke endpoint scripts as modules, not client __file__ path script_path was os.path.realpath(__file__) captured at import on the CLIENT; shipped by value, the executable/training commands then pointed at the laptop checkout (/home/merzky/projects/xgfabric/...) while running on Perlmutter. Invoke via 'python3 -m tasks.profiler.endpoint_*' instead -- the endpoint puts XGF_DIR on PYTHONPATH, so the path is resolved on whatever host runs the command. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- tasks/profiler/components.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tasks/profiler/components.py b/tasks/profiler/components.py index 7b20324..f6c50a0 100644 --- a/tasks/profiler/components.py +++ b/tasks/profiler/components.py @@ -25,7 +25,6 @@ TASK_DESCRIPTION_DTYPE = DataType("TASK_INFO") PROFILE_RESULTS = DataType("PROFILE_RESULT") -script_path = os.path.dirname(os.path.realpath(__file__)) class ProfilerInvestigator(ModelInvestigator): @@ -127,8 +126,7 @@ async def exec_profiler(task_data): async def train_model(): return shlex.join( [ - "python3", - f"{script_path}/endpoint_trainer.py", + "python3", "-m", "tasks.profiler.endpoint_trainer", f"{self.datastore}/data.csv", f"{self.datastore}/model.json", ] @@ -173,8 +171,7 @@ async def call_inference(in_data: TypedData, model=None, name=""): await self._stage_inf(self.datastore, pf) return shlex.join( [ - "python3", - f"{script_path}/endpoint_eval.py", + "python3", "-m", "tasks.profiler.endpoint_eval", model, f"{self.datastore}/inf.json", ] From 3d5fc361249ffb1327f2c21916e4afe537b98a83 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 4 Sep 2026 16:36:34 +0200 Subject: [PATCH 24/33] demo: endpoint env needs xgboost (+scikit-learn) for the Pi trainer endpoint_trainer.py/endpoint_eval.py import xgboost; the cfdaai clone does not carry it. Install into the endpoint env at setup. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- demo/Sep_04_endpoint_setup.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/demo/Sep_04_endpoint_setup.sh b/demo/Sep_04_endpoint_setup.sh index 134b8f0..9a33d1d 100755 --- a/demo/Sep_04_endpoint_setup.sh +++ b/demo/Sep_04_endpoint_setup.sh @@ -57,6 +57,8 @@ echo "==> installing runtime into $ENV_PREFIX" "$PY" -m pip install -q --force-reinstall --no-deps \ "rhapsody-py @ git+https://github.com/radical-cybertools/rhapsody@fix/dragon-cancel-idempotent" "$PY" -m pip install -q "$RH" +# endpoint_trainer.py / endpoint_eval.py need these; cfdaai lacks xgboost +"$PY" -m pip install -q xgboost scikit-learn # data dirs: seed the Pi-predictor dataset at the RUNTIME datastore (where # endpoint_trainer.py reads it) From 3f3aeb33949a9a8ebfb9d335b0306473f1187573 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 4 Sep 2026 17:01:29 +0200 Subject: [PATCH 25/33] real twin: add a visible learning lane on the wind stream The real learners (FNO/PINN/PCR, Pi predictor) run on the default 'inference' engine, so the dashboard showed no learning lane. Add a compact WindTrendLearner that consumes the same sensor stream and routes its training to the 'learning' backend (the @function_task(backend=...) label the service demo's SurrogateInvestigator uses). It publishes to its own WIND_TREND dtype, so it never competes with the WIND_FIELD path feeding the sink. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- tasks/wind_learner.py | 66 +++++++++++++++++++++++++++++++++++++++++++ twin_service_real.py | 8 ++++++ 2 files changed, 74 insertions(+) create mode 100644 tasks/wind_learner.py diff --git a/tasks/wind_learner.py b/tasks/wind_learner.py new file mode 100644 index 0000000..c9ec639 --- /dev/null +++ b/tasks/wind_learner.py @@ -0,0 +1,66 @@ +"""A visible learning lane for the real twin. + +The real workload's own learners (FNO/PINN/PCR and the Pi predictor) run +on the twin's default ('inference') engine, so the dashboard shows no +distinct learning lane. This component adds one: it consumes the same +wind-sensor stream, and every window of points runs a training task +routed to the 'learning' backend -- exactly the labeling trick from the +service demo's SurrogateInvestigator (@function_task(backend=...)). It +publishes to its own WIND_TREND dtype, so it never competes with the +real WIND_FIELD path feeding the sink. +""" + +import asyncio +import random + +from digitaltwin.components import DataType, ModelInvestigator, TypedData +from digitaltwin.runtime import RuntimeAPI + +WIND_TREND = DataType("WindTrend_dt") + +WINDOW = 4 # sensor points per training window +TRAIN_SECONDS = 4.0 # endpoint-side "training" cost, visible in the lane + + +class WindTrendLearner(ModelInvestigator): + """Online mean-wind learner; its training runs in the learning lane.""" + + def __init__(self, flow, learn_backend: str | None = None): + super().__init__(flow) + self.flow = flow + self.batch: list = [] + + # the label rides here: with learn_backend='learning' the training + # task routes to the session's learning engine and shows in its own + # lane; inference stays on the default engine. backend=None is the + # plain default (asyncflow has no aliasing -- a label is set only + # when the caller declares the matching engine). + @flow.function_task(backend=learn_backend) + async def train(points): + await asyncio.sleep(TRAIN_SECONDS) + speeds = [p["wind_speed"] for p in points] + return {"model": "wind_trend", "mean": sum(speeds) / len(speeds)} + + @flow.function_task + async def infer(in_data, model="na", mean=0.0, **_): + return TypedData(WIND_TREND, {"model": model, "mean": mean}) + + self._train = train + self._infer = infer + + async def _on_input(self, in_data: TypedData) -> None: + self.batch.append(in_data.data) + + async def main_loop(self, runtime: RuntimeAPI): + runtime.subscribe_to_topic(runtime.ON_INPUT, self._on_input) + runtime.set_inference_task(self._infer) + runtime.publish_new_model({"model": "na", "mean": 0.0}) + + while True: + if len(self.batch) < WINDOW: + await asyncio.sleep(1.0) + continue + + points, self.batch = self.batch[:WINDOW], self.batch[WINDOW:] + model = await self._train(points) + runtime.publish_new_model(model, {"quality": random.random()}) diff --git a/twin_service_real.py b/twin_service_real.py index 1bd42de..b539bf0 100644 --- a/twin_service_real.py +++ b/twin_service_real.py @@ -51,6 +51,7 @@ import tasks.common import tasks.common.dtypes import tasks.wind_agent +import tasks.wind_learner import tasks.sink import tasks.profiler.components import tasks.do_fno.fno_investigator @@ -59,6 +60,7 @@ from tasks.common.dtypes import DAVIS_WIND_SENSOR, WIND_FIELD from tasks.wind_agent import WindFieldAgent +from tasks.wind_learner import WindTrendLearner, WIND_TREND from tasks.sink import CUPS_Sink from tasks.profiler.components import ( ProfilerInvestigator, @@ -113,6 +115,9 @@ def main(args) -> int: field = dt.package(WindFieldAgent, config) sink = dt.package(CUPS_Sink, config) + # a visible learning lane on the wind stream (see wind_learner.py); + # its training routes to the 'learning' engine declared in ENGINES + learner = dt.package(WindTrendLearner, learn_backend="learning") base_profiler = dt.package( ProfilerInvestigator, playground + "/profiler/nersc_profiler") pi_profiler = dt.package( @@ -130,6 +135,9 @@ def main(args) -> int: dt.add_investigator(twin, pi_profiler, PROFILE_RESULTS, PI_PREDICT_RUNTIME) + # visible learning lane, fed by the same sensor stream + dt.add_investigator(twin, learner, DAVIS_WIND_SENSOR, WIND_TREND) + dt.start(twin) deadline = time.time() + args.runtime From 4c5e034ba98dced05dfcae285d6be6dfc6a0abde Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 4 Sep 2026 17:02:58 +0200 Subject: [PATCH 26/33] demo/endpoint: expose CUDA driver so TF uses the GPU TF fell back to CPU because libcuda.so.1 was not on the task LD_LIBRARY_PATH. module load cudatoolkit + prepend /usr/lib64 (compute node driver) and the env's CUDA libs; log nvidia-smi -L so we can see whether the allocation actually has a GPU. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- demo/Sep_04_endpoint.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/demo/Sep_04_endpoint.sh b/demo/Sep_04_endpoint.sh index 3def3f8..c4d8278 100755 --- a/demo/Sep_04_endpoint.sh +++ b/demo/Sep_04_endpoint.sh @@ -36,6 +36,17 @@ export RADICAL_ORBIT_LOG_FILE="$SCRATCH/orbit-logs/$EP.log" mkdir -p "$SCRATCH/orbit-logs" "$PLAYGROUND_DIR" rm -f "$HOME/.radical/orbit/logs/"*.log 2>/dev/null || true +# --- GPU: expose the NVIDIA driver + CUDA to the TF tasks ------------------ +# Ben's cfdaai TF fell back to CPU ("Could not find cuda drivers") because +# libcuda.so.1 was not on the task LD_LIBRARY_PATH. Load PM's CUDA module +# and prepend the compute-node driver path + the env's own CUDA libs. All +# best-effort (|| true) so a CPU allocation still launches -- the nvidia-smi +# line below records whether a GPU is actually present. +module load cudatoolkit 2>/dev/null || true +export LD_LIBRARY_PATH="/usr/lib64:$ENV_PREFIX/lib:${LD_LIBRARY_PATH:-}" +echo "GPU check (nvidia-smi -L):" +nvidia-smi -L 2>&1 | sed 's/^/ /' || echo " no GPU visible -- CPU allocation? (need salloc -C gpu)" + echo "launching endpoint '$EP' under dragon ..." exec "$ENV_PREFIX/bin/dragon" "$ENV_PREFIX/bin/radical-orbit-endpoint.py" -n "$EP" \ 2>&1 | tee "$SCRATCH/orbit-logs/$EP.console.log" From f6a4f3d9493b2d8965183628a985db4b45bb69d1 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 4 Sep 2026 17:09:56 +0200 Subject: [PATCH 27/33] demo/client: default to the fast fake-surrogate driver on CPU No GPU allocation, so the real TF surrogates (~54s/inference on CPU) are too slow for a live demo. Default the demo client to twin_service.py -- fake SurrogateInvestigators (sleeps + numpy field synthesis, no TF or xgboost) that produce heatmaps in seconds and carry the learning lane (learn_backend='learning'). RUN_REAL=1 still selects twin_service_real.py for a GPU-backed run. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- demo/Sep_04_client.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/demo/Sep_04_client.sh b/demo/Sep_04_client.sh index 7ac01f0..56953f7 100755 --- a/demo/Sep_04_client.sh +++ b/demo/Sep_04_client.sh @@ -12,11 +12,16 @@ VENV="${DT_VENV:-$HOME/radical/digital_twins/ve.demo}" DT_DIR="${DT_DIR:-$HOME/radical/digital_twins}" HERE="$(cd "$(dirname "$0")/.." && pwd)" # xGFabric checkout root RUNTIME="${RUNTIME:-600}" +# default to the fast fake-surrogate driver (CPU-friendly: sleeps + numpy, +# no TF/xgboost, heatmaps in seconds, learning lane built in). RUN_REAL=1 +# switches to the real TF workload (needs a GPU endpoint to be timely). +DRIVER="twin_service.py" +[ "${RUN_REAL:-}" = 1 ] && DRIVER="twin_service_real.py" # --------------------------------------------------------------------------- echo "--------------------------" echo "xGFabric Demo September 04" -echo "Client twin_service_real.py ($(hostname -f)) -> broker $BROKER_IP, endpoint HPC 'hpc'" +echo "Client $DRIVER ($(hostname -f)) -> broker $BROKER_IP, endpoint HPC 'hpc'" echo "--------------------------" cd "$HERE" @@ -37,6 +42,6 @@ export DT_INFERENCE_BACKEND=dragon_v3 export DT_LEARNING_ENDPOINT=hpc export DT_LEARNING_BACKEND=concurrent -echo "driving the real twin (runtime ${RUNTIME}s) ..." +echo "driving twin via $DRIVER (runtime ${RUNTIME}s) ..." echo "dashboard: https://$BROKER_IP:8000/broker/dt/ui?live=1" -exec "$VENV/bin/python" twin_service_real.py --runtime "$RUNTIME" +exec "$VENV/bin/python" "$DRIVER" --runtime "$RUNTIME" From c9c4e789cadd67a21d33109206e517749c50cf49 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 4 Sep 2026 17:16:33 +0200 Subject: [PATCH 28/33] service/sink: resolve heatmap file path endpoint-side, best-effort main_loop runs on the broker; it built the PNG path from the broker's home (XGF_WORKSPACE unset there -> ~/xgf_twin) and mkdir'd it there, then handed that broker path to render_heatmap, which runs on the endpoint and failed to write it (no such dir on Perlmutter). Resolve XGF_WORKSPACE and mkdir inside the task instead, and wrap the disk copy in try/except so it can never fail the field -- the dashboard copy rides inline regardless. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- service/sink.py | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/service/sink.py b/service/sink.py index e26e5dc..4567695 100644 --- a/service/sink.py +++ b/service/sink.py @@ -3,25 +3,17 @@ The heatmap is rendered on the endpoint and returned inline (small PNG bytes) so the runtime can surface it to the dashboard via `record_output` -- the DT service has no file staging, and a downscaled -heatmap is well under the return-value cap. It is also written to -XGF_WORKSPACE on the endpoint for the record (resolved at runtime, since -this module ships by value). +heatmap is well under the return-value cap. A copy is also written to +XGF_WORKSPACE, but that path is resolved and created *inside* the task +(endpoint-side); the sink's main_loop runs on the broker, a different +host, so it must not build or create that path itself. """ import base64 -import os -from pathlib import Path from digitaltwin.components import TypedData, UtilityTask -def _workspace() -> Path: - base = Path(os.environ.get("XGF_WORKSPACE", "") - or Path.home() / "xgf_twin") - base.mkdir(parents=True, exist_ok=True) - return base - - class ServiceSink(UtilityTask): def __init__(self, flow): super().__init__(flow) @@ -29,8 +21,10 @@ def __init__(self, flow): self.count = 0 @flow.function_task - async def render_heatmap(data, arch, w, fname): + async def render_heatmap(data, arch, w, count): import io + import os + from pathlib import Path import matplotlib matplotlib.use("Agg") @@ -44,8 +38,18 @@ async def render_heatmap(data, arch, w, fname): plt.xlabel("X") plt.ylabel("Y") - # to the endpoint filesystem, for the record - fig.savefig(fname, dpi=90, bbox_inches="tight") + # a copy to the endpoint filesystem, for the record -- resolved + # here (endpoint XGF_WORKSPACE) and best-effort, so a missing or + # read-only path never fails the field + try: + base = Path(os.environ.get("XGF_WORKSPACE", "") + or Path.home() / "xgf_twin") + base.mkdir(parents=True, exist_ok=True) + fig.savefig(str(base / f"field_{count:04d}_{arch}.png"), + dpi=90, bbox_inches="tight") + except OSError: + pass + # and to memory, small, for the inline return buf = io.BytesIO() fig.savefig(buf, format="png", dpi=72, bbox_inches="tight") @@ -61,9 +65,8 @@ async def main_loop(self, runtime, in_data: TypedData): return self.count += 1 - fname = str(_workspace() / f"field_{self.count:04d}_{arch}.png") png = await self._render(in_data.data["result"][1], arch, - in_data.data["w"], fname) + in_data.data["w"], self.count) # surface it to the dashboard -- small enough to ride inline b64 = base64.b64encode(png).decode("ascii") From 1a338eb7068fb76ddf249ccafd0c7102a93d6ca2 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 4 Sep 2026 17:20:16 +0200 Subject: [PATCH 29/33] demo/broker: force digitaltwin refresh (install.sh skips on version match) install.sh installs digitaltwin by exact version string and skips when it is unchanged, so the broker kept pre-record_output code and the sink crashed with AttributeError: RuntimeAPI has no attribute record_output. Force-reinstall the checked-out devel tip after install.sh. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- demo/Sep_04_broker.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/demo/Sep_04_broker.sh b/demo/Sep_04_broker.sh index f15e2f8..479926e 100755 --- a/demo/Sep_04_broker.sh +++ b/demo/Sep_04_broker.sh @@ -25,6 +25,10 @@ echo "--------------------------" ( cd "$DT_DIR" && git checkout devel && git pull ) ( cd "$DT_DIR" ./deploy/install.sh broker + # install.sh compares by version string and skips when it is unchanged, + # so it can leave stale digitaltwin code (e.g. missing record_output). + # Force the checked-out devel tip over whatever it left. + ./ve.demo/bin/pip install -q --force-reinstall --no-deps "$DT_DIR" ./ve.demo/bin/pip install -q numpy pandas # the agent runs broker-side ./ve.demo/bin/pip install -q --force-reinstall --no-deps \ "rhapsody-py @ git+https://github.com/radical-cybertools/rhapsody@fix/dragon-cancel-idempotent" From 4ce640e0ace426feb083a7aa7e3e6de243a9019b Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 4 Sep 2026 17:47:23 +0200 Subject: [PATCH 30/33] demo/endpoint: gate GPU env behind RUN_GPU=1 (off by default) Prepending /usr/lib64 to LD_LIBRARY_PATH can shadow the conda env's libs and break the dragon task launch -- the endpoint registers but its engine never comes up, so session init times out. Useless on the CPU/fake path anyway; make it opt-in. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- demo/Sep_04_endpoint.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/demo/Sep_04_endpoint.sh b/demo/Sep_04_endpoint.sh index c4d8278..e7ed34b 100755 --- a/demo/Sep_04_endpoint.sh +++ b/demo/Sep_04_endpoint.sh @@ -36,16 +36,16 @@ export RADICAL_ORBIT_LOG_FILE="$SCRATCH/orbit-logs/$EP.log" mkdir -p "$SCRATCH/orbit-logs" "$PLAYGROUND_DIR" rm -f "$HOME/.radical/orbit/logs/"*.log 2>/dev/null || true -# --- GPU: expose the NVIDIA driver + CUDA to the TF tasks ------------------ -# Ben's cfdaai TF fell back to CPU ("Could not find cuda drivers") because -# libcuda.so.1 was not on the task LD_LIBRARY_PATH. Load PM's CUDA module -# and prepend the compute-node driver path + the env's own CUDA libs. All -# best-effort (|| true) so a CPU allocation still launches -- the nvidia-smi -# line below records whether a GPU is actually present. -module load cudatoolkit 2>/dev/null || true -export LD_LIBRARY_PATH="/usr/lib64:$ENV_PREFIX/lib:${LD_LIBRARY_PATH:-}" -echo "GPU check (nvidia-smi -L):" -nvidia-smi -L 2>&1 | sed 's/^/ /' || echo " no GPU visible -- CPU allocation? (need salloc -C gpu)" +# --- GPU (opt-in: RUN_GPU=1) ----------------------------------------------- +# Only for a real GPU allocation. Prepending /usr/lib64 to LD_LIBRARY_PATH +# can shadow the conda env's own libs and break the dragon task launch, so +# it stays OFF by default -- the CPU/fake demo path must not touch the env. +if [ "${RUN_GPU:-}" = 1 ]; then + module load cudatoolkit 2>/dev/null || true + export LD_LIBRARY_PATH="/usr/lib64:$ENV_PREFIX/lib:${LD_LIBRARY_PATH:-}" + echo "GPU check (nvidia-smi -L):" + nvidia-smi -L 2>&1 | sed 's/^/ /' || echo " no GPU visible (need salloc -C gpu)" +fi echo "launching endpoint '$EP' under dragon ..." exec "$ENV_PREFIX/bin/dragon" "$ENV_PREFIX/bin/radical-orbit-endpoint.py" -n "$EP" \ From 7048fa8dce3d874385e9d9c1d17939ce3eb6607f Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Fri, 4 Sep 2026 17:54:23 +0200 Subject: [PATCH 31/33] twin_service: drop the noisy per-poll state/calls line Keep the field-probe line as the heartbeat; the state dict is only used for the failed-state check now. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- twin_service.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/twin_service.py b/twin_service.py index a8f9bae..ba5c323 100644 --- a/twin_service.py +++ b/twin_service.py @@ -113,8 +113,6 @@ def main(args) -> int: time.sleep(10) info = dt.twin(twin) - print(f"[client] state={info['state']}" - f" calls={info.get('calls') or {}}", flush=True) if info["state"] == "failed": print(f"[client] twin failed: {info.get('last_error')}") return 1 From 790d4d2da9e5d928fbdda038b49bbf912d874691 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Sat, 5 Sep 2026 23:02:08 +0200 Subject: [PATCH 32/33] twin_service_real: refresh stale STATUS docstring Reflects reality after the demo cycle: runs end to end on the DTaaS stack; the shared-FS caveat is resolved (compute + hand-offs are endpoint-side); the TF path is GPU-bound so the demo defaults to the fake driver (RUN_REAL=1 selects this one). Points at the Sep_04 scripts. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- twin_service_real.py | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/twin_service_real.py b/twin_service_real.py index b539bf0..a6ebdf0 100644 --- a/twin_service_real.py +++ b/twin_service_real.py @@ -8,23 +8,26 @@ tk_do_simulation's precalc-CSV shortcut); the trainings, the profiler timing, and the selection are real. -STATUS: experimental scaffold, NOT yet validated end to end. It is the -starting point for the on-Perlmutter test cycle, not a proven driver. -Known open items (see service/README.md "Real workload"): - - * Endpoint must run in a TF-carrying env (Ben's cfdaai clone) -- - setup-hpc-endpoint-real.sh. The lazy-import refactor keeps the - client/broker TF-free, but they still need dotenv + numpy + pandas - and the pyspot submodule (`git submodule update --init`). - * Shared-filesystem assumption: the real components write to - config['PLAYGROUND_DIR'] both from main_loops (broker side) and - from tasks (endpoint side). Those line up only if broker and - endpoint share a filesystem -- so run the BROKER on Perlmutter too - for the real workload, with PLAYGROUND_DIR on $SCRATCH. A broker on - radical.3 will split the playground across two hosts. - * Backends default to 'concurrent' here: real TF/sklearn under dragon - is untested (joblib's mp bridge, TF process model). Flip to - dragon_v3 via the env knobs once it is proven. +STATUS: runs end to end on the DTaaS stack -- broker on radical.3, the +ORBIT data plane, and a rhapsody/dragon endpoint on Perlmutter. The real +TF surrogates are GPU-bound, though: on a CPU allocation each inference is +tens of seconds, too slow for a live demo, so the September demo defaults +to the fake driver (twin_service.py) and selects this one only with +RUN_REAL=1 (see demo/Sep_04_client.sh) on a GPU endpoint. + +Notes: + + * Endpoint env: demo/Sep_04_endpoint_setup.sh clones Ben's cfdaai + (TF + CFD/ML) and adds xgboost. The lazy-import refactor keeps the + client/broker TF-free -- they need only dotenv + numpy + pandas and + the pyspot submodule (`git submodule update --init`). + * No shared-filesystem requirement: the compute chain runs entirely on + the endpoint, and the cross-host hand-offs were moved endpoint-side + (data.csv append and inf.json staging are function_tasks, the profiler + measures in-process), so a broker on radical.3 with the endpoint on + Perlmutter works. + * Backends: inference on dragon_v3, learning on concurrent, set via the + DT_* env knobs (see demo/Sep_04_client.sh). Environment: config.sh (tasks/common/config.sh) supplies PLAYGROUND_DIR, CSPOT_LIMIT, endpoint/model paths etc., loaded via dotenv as in twin.py. From 1837a3fa551e12c4f71196386370b82025e9965c Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Tue, 1 Sep 2026 23:07:18 +0200 Subject: [PATCH 33/33] service twin: telemetry reports from the endpoint's own recording Nothing to enable: the rhapsody plugin on the endpoint already records every task plus a resource poll (the [telemetry] extra) into telemetry-output/session.*.telemetry.jsonl. collect_reports.py points twin.py's existing report generators at those files -- waterfall, dependency wait, stage timers, swimlane, concurrency/resources, and the gantt render unmodified (verified against the service run's 8292 events). Known gap, documented: per-workflow gantt grouping needs asyncflow.workflow_id, which only workflow_scope() stamps -- an engine-side telemetry feature for the DT service, tracked in digital.twins. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016Npyz3Hbnwos12ESsdJ2YU --- service/README.md | 21 ++++++++++++- service/collect_reports.py | 62 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 service/collect_reports.py diff --git a/service/README.md b/service/README.md index 732f2e6..042ba68 100644 --- a/service/README.md +++ b/service/README.md @@ -112,4 +112,23 @@ Two items remain open before a real run is trustworthy: | profiler subprocess + Pi learner | `ServiceProfiler` (inline timed run) + `ServicePiPredictor` | | CUPS_Sink | `ServiceSink` (runtime-resolved workspace) | | ZMQ stream | ORBIT data plane | -| asyncflow telemetry + reports | not yet — see the telemetry branch | +| asyncflow telemetry + reports | endpoint-side rhapsody telemetry + `collect_reports.py` | + +## Telemetry and reports + +Nothing to enable: the rhapsody plugin on the endpoint records every +task plus a resource poll on its own (the `[telemetry]` extra), into +`telemetry-output/session.*.telemetry.jsonl` under the endpoint's +working directory. After (or during) a run: + + /bin/python service/collect_reports.py [telemetry-dir] + +renders twin.py's report set — task waterfall, dependency wait, stage +timers, swimlane, concurrency/resources, and the gantt — next to the +jsonl. For a remote endpoint, scp the jsonl files first. + +Known gap: per-workflow grouping in the gantt needs +`asyncflow.workflow_id`, which only `workflow_scope()` stamps — the +service engine does not run one. That is an engine-side telemetry +feature for the DT service (digital.twins), tracked there; every other +report is complete without it. diff --git a/service/collect_reports.py b/service/collect_reports.py new file mode 100644 index 0000000..bb73f7b --- /dev/null +++ b/service/collect_reports.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Render twin.py's telemetry reports from a service-mode run. + +In service mode nobody calls ``flow.start_telemetry`` -- and nobody has +to: the rhapsody plugin on the ENDPOINT records every task and a +resource poll on its own (the ``[telemetry]`` extra), as +``telemetry-output/session.*.telemetry.jsonl`` in the endpoint's +working directory. This script points twin.py's existing report +generators at those files. + + python service/collect_reports.py [telemetry-dir] [--last N] + +``telemetry-dir`` defaults to ``telemetry-output/`` under the local +digital.twins checkout; for a remote endpoint, scp the jsonl files over +first. Reports land next to the jsonl. + +Known gap: the workflow-gantt renders but cannot group per workflow -- +service twins do not run inside ``workflow_scope()``, so no +``asyncflow.workflow_id`` is stamped. Grouping needs engine-side +telemetry in the DT service (a digital.twins feature, not a demo-side +one); the task waterfall / swimlane / resource dashboards are complete +without it. +""" + +import argparse +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from reports.plot_workflow_dashboard import plot_split +from reports.plot_workflow_gantt import plot as plot_gantt + +DEFAULT_DIR = os.path.expanduser("~/radical/digital_twins/telemetry-output") + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Render reports from endpoint telemetry") + parser.add_argument("telemetry_dir", nargs="?", default=DEFAULT_DIR) + parser.add_argument("--last", type=int, default=1, + help="how many of the newest sessions to render") + args = parser.parse_args() + + files = sorted(Path(args.telemetry_dir).glob("*.telemetry.jsonl"), + key=lambda p: p.stat().st_mtime, reverse=True) + if not files: + print(f"no *.telemetry.jsonl under {args.telemetry_dir}", + file=sys.stderr) + return 1 + + for f in files[:args.last]: + print(f"== {f.name}") + plot_split(str(f)) + plot_gantt(f) + + return 0 + + +if __name__ == "__main__": + sys.exit(main())