From c019623fdbe1fc01659e16eaaaf9b11ff889aeb0 Mon Sep 17 00:00:00 2001 From: Ken O'Brien Date: Tue, 9 Jun 2026 11:33:29 +0100 Subject: [PATCH 01/20] flower init commit --- README.md | 1 + packages/federated/flower-base/Dockerfile | 21 ++++ packages/federated/flower-base/README.md | 34 ++++++ packages/federated/flower-base/config.yaml | 12 ++ packages/federated/flower-base/test.sh | 29 +++++ .../federated/flower-superexec/Dockerfile | 24 ++++ packages/federated/flower-superexec/README.md | 94 +++++++++++++++ .../federated/flower-superexec/app/README.md | 75 ++++++++++++ .../flower-superexec/app/pyproject.toml | 33 ++++++ .../app/pytorchexample/__init__.py | 1 + .../app/pytorchexample/client_app.py | 82 +++++++++++++ .../app/pytorchexample/server_app.py | 61 ++++++++++ .../app/pytorchexample/task.py | 109 ++++++++++++++++++ .../federated/flower-superexec/config.yaml | 13 +++ packages/federated/flower-superexec/test.sh | 31 +++++ .../federated/flower-superlink/Dockerfile | 26 +++++ packages/federated/flower-superlink/README.md | 65 +++++++++++ .../federated/flower-superlink/config.yaml | 16 +++ .../federated/flower-superlink/gen-certs.sh | 64 ++++++++++ packages/federated/flower-superlink/test.sh | 31 +++++ .../federated/flower-supernode/Dockerfile | 16 +++ packages/federated/flower-supernode/README.md | 57 +++++++++ .../federated/flower-supernode/config.yaml | 18 +++ packages/federated/flower-supernode/test.sh | 23 ++++ 24 files changed, 936 insertions(+) create mode 100644 packages/federated/flower-base/Dockerfile create mode 100644 packages/federated/flower-base/README.md create mode 100644 packages/federated/flower-base/config.yaml create mode 100644 packages/federated/flower-base/test.sh create mode 100644 packages/federated/flower-superexec/Dockerfile create mode 100644 packages/federated/flower-superexec/README.md create mode 100644 packages/federated/flower-superexec/app/README.md create mode 100644 packages/federated/flower-superexec/app/pyproject.toml create mode 100644 packages/federated/flower-superexec/app/pytorchexample/__init__.py create mode 100644 packages/federated/flower-superexec/app/pytorchexample/client_app.py create mode 100644 packages/federated/flower-superexec/app/pytorchexample/server_app.py create mode 100644 packages/federated/flower-superexec/app/pytorchexample/task.py create mode 100644 packages/federated/flower-superexec/config.yaml create mode 100644 packages/federated/flower-superexec/test.sh create mode 100644 packages/federated/flower-superlink/Dockerfile create mode 100644 packages/federated/flower-superlink/README.md create mode 100644 packages/federated/flower-superlink/config.yaml create mode 100644 packages/federated/flower-superlink/gen-certs.sh create mode 100644 packages/federated/flower-superlink/test.sh create mode 100644 packages/federated/flower-supernode/Dockerfile create mode 100644 packages/federated/flower-supernode/README.md create mode 100644 packages/federated/flower-supernode/config.yaml create mode 100644 packages/federated/flower-supernode/test.sh diff --git a/README.md b/README.md index 0a162185..f839e15f 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ These dockerfiles will also be pushed and actively maintained in their original | Vision | [`OpenCV`](packages/vision/opencv), [`SAM`](packages/vision/sam), [`MobileSAM`](packages/vision/mobilesam), [`ncnn`](packages/vision/ncnn), [`DINOv3`](packages/vision/dinov3), [`SAM3`](packages/vision/sam3), [`Ultralytics`](packages/vision/ultralytics) | | Ryzen AI NPU | [`XDNA`](packages/npu/xdna), [`IRON`](packages/npu/iron), [`NPUEval`](packages/npu/npueval), [`Ryzen AI CVML`](packages/npu/ryzenai_cvml) | | Adaptive SoCs | [`PYNQ.remote`](packages/adaptive-socs/pynq-remote) | +| Federated Learning | [`Flower`](packages/federated/flower-base) ([`SuperLink`](packages/federated/flower-superlink), [`SuperNode`](packages/federated/flower-supernode), [`SuperExec`](packages/federated/flower-superexec)) | | Utilities | [`JupyterLab`](packages/ide/jupyterlab), [`amdgpu_top`](packages/init/amdgpu_top) | --- diff --git a/packages/federated/flower-base/Dockerfile b/packages/federated/flower-base/Dockerfile new file mode 100644 index 00000000..83c2bcf2 --- /dev/null +++ b/packages/federated/flower-base/Dockerfile @@ -0,0 +1,21 @@ +# Copyright(C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT + +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +ENV DEBIAN_FRONTEND=noninteractive + +# Flower CLI binaries (flower-superlink, flower-supernode, flower-superexec) +# all come from this single pip package. Torch + ROCm is provided by the +# default rocm/pytorch base image, so no torch install is needed here. +ARG FLWR_VERSION=1.26.1 +RUN pip3 install --no-cache-dir --break-system-packages \ + "flwr[simulation]==${FLWR_VERSION}" \ + "flwr-datasets[vision]>=0.5.0" + +WORKDIR /ryzers +COPY test.sh /ryzers/test_flower-base.sh +RUN chmod +x /ryzers/test_flower-base.sh + +CMD /ryzers/test_flower-base.sh diff --git a/packages/federated/flower-base/README.md b/packages/federated/flower-base/README.md new file mode 100644 index 00000000..41569f4d --- /dev/null +++ b/packages/federated/flower-base/README.md @@ -0,0 +1,34 @@ +# Flower Base Docker Setup + +Shared base layer for the Flower federated-learning Ryzers. Installs the +[Flower](https://flower.ai) framework (`flwr[simulation]==1.26.1`) on top +of the default ROCm/PyTorch base image, so the three flower binaries +(`flower-superlink`, `flower-supernode`, `flower-superexec`) and a +ROCm-enabled PyTorch are available to layers that build on this one. + +You usually don't build or run this Ryzer on its own — chain it with one of +the role Ryzers: + +```sh +ryzers build flower-base flower-superlink # server box +ryzers build flower-base flower-supernode # client box +ryzers build flower-base flower-superexec # serverapp or clientapp runner +``` + +## Build & Run (standalone smoke test) + +```sh +ryzers build flower-base +ryzers run +``` + +The default `CMD` runs `test_flower-base.sh`, which verifies the CLI +binaries are installed and that `torch.cuda.is_available()` returns true +under ROCm. + +## References + +- [Flower documentation](https://flower.ai/docs/framework/) +- [Multi-machine Docker tutorial](https://flower.ai/docs/framework/docker/tutorial-deploy-on-multiple-machines.html) + +Copyright(C) 2026 Advanced Micro Devices, Inc. All rights reserved. diff --git a/packages/federated/flower-base/config.yaml b/packages/federated/flower-base/config.yaml new file mode 100644 index 00000000..c3b2b261 --- /dev/null +++ b/packages/federated/flower-base/config.yaml @@ -0,0 +1,12 @@ +# Copyright(C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT + +# flower-base is a composable layer. It provides the flwr CLI on top of the +# default ROCm/PyTorch base image. Most users will not run it directly; +# instead chain it with one of: +# ryzers build flower-base flower-superlink +# ryzers build flower-base flower-supernode +# ryzers build flower-base flower-superexec + +build_arguments: +- "FLWR_VERSION=1.26.1" diff --git a/packages/federated/flower-base/test.sh b/packages/federated/flower-base/test.sh new file mode 100644 index 00000000..0c01c022 --- /dev/null +++ b/packages/federated/flower-base/test.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT + +set -e + +echo "Running tests for flower-base..." + +# Verify all three flower CLI entrypoints are installed +for bin in flower-superlink flower-supernode flower-superexec flwr; do + if ! command -v "$bin" >/dev/null 2>&1; then + echo "FAIL: $bin not on PATH" + exit 1 + fi + echo "Found: $(command -v "$bin")" +done + +# Verify flwr Python package and torch+ROCm +python3 - <<'PY' +import flwr +import torch +print(f"flwr version: {flwr.__version__}") +print(f"torch version: {torch.__version__}") +print(f"torch.cuda.is_available(): {torch.cuda.is_available()}") +print(f"torch HIP version: {torch.version.hip}") +PY + +echo "Tests passed!" diff --git a/packages/federated/flower-superexec/Dockerfile b/packages/federated/flower-superexec/Dockerfile new file mode 100644 index 00000000..e7843c67 --- /dev/null +++ b/packages/federated/flower-superexec/Dockerfile @@ -0,0 +1,24 @@ +# Copyright(C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT + +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +# Install the example Flower app (quickstart-pytorch). The pyproject.toml +# shipped here has torch dependencies stripped so we don't overwrite the +# ROCm PyTorch provided by the base image. +WORKDIR /app +COPY app/ /app/ +RUN pip3 install --no-cache-dir --break-system-packages /app + +WORKDIR /ryzers +COPY test.sh /ryzers/test_flower-superexec.sh +RUN chmod +x /ryzers/test_flower-superexec.sh + +RUN mkdir -p /app/certificates + +# When running --plugin-type clientapp this connects out to the SuperNode +# at supernode:9094; when running --plugin-type serverapp it connects out +# to the SuperLink at superlink:9091. No inbound ports are exposed. + +CMD /ryzers/test_flower-superexec.sh diff --git a/packages/federated/flower-superexec/README.md b/packages/federated/flower-superexec/README.md new file mode 100644 index 00000000..f2c598ff --- /dev/null +++ b/packages/federated/flower-superexec/README.md @@ -0,0 +1,94 @@ +# Flower SuperExec Docker Setup + +`flower-superexec` is Flower's process runner. The same binary runs both +the **ServerApp** (server-side aggregation logic) and the **ClientApp** +(client-side training), selected with `--plugin-type {serverapp,clientapp}`. + +This Ryzer is the only one in the federation that actually executes +PyTorch training, so it sits on top of the ROCm/PyTorch base image and +inherits GPU access. It ships with the `quickstart-pytorch` example app +preinstalled at `/app/`. + +## Build + +```sh +ryzers build flower-base flower-superexec +``` + +## Run — ServerApp (one per federation, on the server machine) + +```sh +ryzers run flower-superexec \ + --insecure \ + --plugin-type serverapp \ + --appio-api-address ${SUPERLINK_HOST:-127.0.0.1}:9091 +``` + +## Run — ClientApp (one per SuperNode, on each client machine) + +```sh +ryzers run flower-superexec \ + --insecure \ + --plugin-type clientapp \ + --appio-api-address ${SUPERNODE_HOST:-127.0.0.1}:9094 +``` + +## Putting it together (mirrors the upstream tutorial) + +**On the server machine** (one SuperLink + one ServerApp superexec): + +```sh +ryzers build flower-base flower-superlink +ryzers build flower-base flower-superexec # separate image, ServerApp role + +# Terminal 1 — SuperLink +ryzers run flower-superlink --insecure --isolation process + +# Terminal 2 — ServerApp superexec +ryzers run flower-superexec \ + --insecure --plugin-type serverapp \ + --appio-api-address 127.0.0.1:9091 +``` + +**On each client machine** (one SuperNode + one ClientApp superexec): + +```sh +ryzers build flower-base flower-supernode +ryzers build flower-base flower-superexec + +# Terminal 1 — SuperNode (connects to remote SuperLink) +ryzers run flower-supernode \ + --insecure \ + --superlink ${SUPERLINK_IP}:9092 \ + --clientappio-api-address 0.0.0.0:9094 \ + --node-config "partition-id=0 num-partitions=2" \ + --isolation process + +# Terminal 2 — ClientApp superexec (runs the PyTorch training) +ryzers run flower-superexec \ + --insecure --plugin-type clientapp \ + --appio-api-address 127.0.0.1:9094 +``` + +Then from any machine with the `flwr` CLI installed: + +```sh +flwr run /app remote-deployment # see config.toml in upstream tutorial +``` + +For TLS (production), see `flower-superlink/gen-certs.sh` and pass +`--root-certificates /app/certificates/ca.crt` to the superexec instead +of `--insecure`. + +## Bundled example + +`/app/` contains a copy of [`examples/quickstart-pytorch`](https://github.com/adap/flower/tree/v1.26.1/examples/quickstart-pytorch), +modified to drop the `torch==2.8.0` pin so the ROCm PyTorch from the +base image is used unchanged. + +## References + +- [SuperExec / process isolation](https://flower.ai/docs/framework/how-to-deploy-flower-server-using-process-isolation.html) +- [Multi-machine Docker tutorial](https://flower.ai/docs/framework/docker/tutorial-deploy-on-multiple-machines.html) + +Copyright(C) 2026 Advanced Micro Devices, Inc. All rights reserved. diff --git a/packages/federated/flower-superexec/app/README.md b/packages/federated/flower-superexec/app/README.md new file mode 100644 index 00000000..dc92e10c --- /dev/null +++ b/packages/federated/flower-superexec/app/README.md @@ -0,0 +1,75 @@ +--- +tags: [quickstart, vision, fds] +dataset: [CIFAR-10] +framework: [torch, torchvision] +--- + +# Federated Learning with PyTorch and Flower (Quickstart Example) + +This introductory example to Flower uses PyTorch, but deep knowledge of PyTorch is not necessarily required to run the example. However, it will help you understand how to adapt Flower to your use case. Running this example in itself is quite easy. This example uses [Flower Datasets](https://flower.ai/docs/datasets/) to download, partition and preprocess the CIFAR-10 dataset. + +## Set up the project + +### Fetch the app + +Install Flower: + +```shell +pip install flwr +``` + +Fetch the app: + +```shell +flwr new @flwrlabs/quickstart-pytorch +``` + +This will create a new directory called `quickstart-pytorch` with the following structure: + +```shell +quickstart-pytorch +├── pytorchexample +│ ├── __init__.py +│ ├── client_app.py # Defines your ClientApp +│ ├── server_app.py # Defines your ServerApp +│ └── task.py # Defines your model, training and data loading +├── pyproject.toml # Project metadata like dependencies and configs +└── README.md +``` + +### Install dependencies and project + +Install the dependencies defined in `pyproject.toml` as well as the `pytorchexample` package. + +```bash +pip install -e . +``` + +## Run the project + +You can run your Flower project in both _simulation_ and _deployment_ mode without making changes to the code. If you are starting with Flower, we recommend you using the _simulation_ mode as it requires fewer components to be launched manually. By default, `flwr run` will make use of the Simulation Engine. + +### Run with the Simulation Engine + +> [!TIP] +> This example runs faster when the `ClientApp`s have access to a GPU. If your system has one, you can make use of it by configuring the `backend.client-resources` component in your Flower Configuration. Check the [Simulation Engine documentation](https://flower.ai/docs/framework/how-to-run-simulations.html) to learn more about Flower simulations and how to optimize them. + +```bash +# Run with the default federation (CPU only) +flwr run . +``` + +You can also override some of the settings for your `ClientApp` and `ServerApp` defined in `pyproject.toml`. For example: + +```bash +flwr run . --run-config "num-server-rounds=5 learning-rate=0.05" +``` + +> [!TIP] +> For a more detailed walk-through check our [quickstart PyTorch tutorial](https://flower.ai/docs/framework/tutorial-quickstart-pytorch.html) + +### Run with the Deployment Engine + +Follow this [how-to guide](https://flower.ai/docs/framework/how-to-run-flower-with-deployment-engine.html) to run the same app in this example but with Flower's Deployment Engine. After that, you might be intersted in setting up [secure TLS-enabled communications](https://flower.ai/docs/framework/how-to-enable-tls-connections.html) and [SuperNode authentication](https://flower.ai/docs/framework/how-to-authenticate-supernodes.html) in your federation. + +If you are already familiar with how the Deployment Engine works, you may want to learn how to run it using Docker. Check out the [Flower with Docker](https://flower.ai/docs/framework/docker/index.html) documentation. diff --git a/packages/federated/flower-superexec/app/pyproject.toml b/packages/federated/flower-superexec/app/pyproject.toml new file mode 100644 index 00000000..37a21dd2 --- /dev/null +++ b/packages/federated/flower-superexec/app/pyproject.toml @@ -0,0 +1,33 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "quickstart-pytorch" +version = "1.0.1" +description = "Federated Learning with PyTorch and Flower (Quickstart Example)" +license = "Apache-2.0" +# torch and torchvision are intentionally omitted: they are provided by +# the ROCm/PyTorch base image. Pinning them here would cause pip to +# replace the ROCm build with the upstream CUDA/CPU wheel. +dependencies = [ + "flwr>=1.26.0", + "flwr-datasets[vision]>=0.5.0", +] + +[tool.hatch.build.targets.wheel] +packages = ["."] + +[tool.flwr.app] +publisher = "flwrlabs" + +[tool.flwr.app.components] +serverapp = "pytorchexample.server_app:app" +clientapp = "pytorchexample.client_app:app" + +[tool.flwr.app.config] +num-server-rounds = 3 +fraction-evaluate = 0.5 +local-epochs = 1 +learning-rate = 0.1 +batch-size = 32 diff --git a/packages/federated/flower-superexec/app/pytorchexample/__init__.py b/packages/federated/flower-superexec/app/pytorchexample/__init__.py new file mode 100644 index 00000000..d29a98eb --- /dev/null +++ b/packages/federated/flower-superexec/app/pytorchexample/__init__.py @@ -0,0 +1 @@ +"""pytorchexample.""" diff --git a/packages/federated/flower-superexec/app/pytorchexample/client_app.py b/packages/federated/flower-superexec/app/pytorchexample/client_app.py new file mode 100644 index 00000000..95efa9b7 --- /dev/null +++ b/packages/federated/flower-superexec/app/pytorchexample/client_app.py @@ -0,0 +1,82 @@ +"""pytorchexample: A Flower / PyTorch app.""" + +import torch +from flwr.app import ArrayRecord, Context, Message, MetricRecord, RecordDict +from flwr.clientapp import ClientApp + +from pytorchexample.task import Net, load_data +from pytorchexample.task import test as test_fn +from pytorchexample.task import train as train_fn + +# Flower ClientApp +app = ClientApp() + + +@app.train() +def train(msg: Message, context: Context): + """Train the model on local data.""" + + # Load the model and initialize it with the received weights + model = Net() + model.load_state_dict(msg.content["arrays"].to_torch_state_dict()) + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + model.to(device) + + # Load the data + partition_id = context.node_config["partition-id"] + num_partitions = context.node_config["num-partitions"] + batch_size = context.run_config["batch-size"] + trainloader, _ = load_data(partition_id, num_partitions, batch_size) + + # Call the training function + train_loss = train_fn( + model, + trainloader, + context.run_config["local-epochs"], + msg.content["config"]["lr"], + device, + ) + + # Construct and return reply Message + model_record = ArrayRecord(model.state_dict()) + metrics = { + "train_loss": train_loss, + "num-examples": len(trainloader.dataset), + } + metric_record = MetricRecord(metrics) + content = RecordDict({"arrays": model_record, "metrics": metric_record}) + return Message(content=content, reply_to=msg) + + +@app.evaluate() +def evaluate(msg: Message, context: Context): + """Evaluate the model on local data.""" + + # Load the model and initialize it with the received weights + model = Net() + model.load_state_dict(msg.content["arrays"].to_torch_state_dict()) + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + model.to(device) + + # Load the data + partition_id = context.node_config["partition-id"] + num_partitions = context.node_config["num-partitions"] + batch_size = context.run_config["batch-size"] + _, valloader = load_data(partition_id, num_partitions, batch_size) + + # Call the evaluation function + eval_loss, eval_acc = test_fn( + model, + valloader, + device, + ) + + # Construct and return reply Message + metrics = { + "eval_loss": eval_loss, + "eval_acc": eval_acc, + "num-examples": len(valloader.dataset), + } + metric_record = MetricRecord(metrics) + content = RecordDict({"metrics": metric_record}) + return Message(content=content, reply_to=msg) diff --git a/packages/federated/flower-superexec/app/pytorchexample/server_app.py b/packages/federated/flower-superexec/app/pytorchexample/server_app.py new file mode 100644 index 00000000..2a6129e5 --- /dev/null +++ b/packages/federated/flower-superexec/app/pytorchexample/server_app.py @@ -0,0 +1,61 @@ +"""pytorchexample: A Flower / PyTorch app.""" + +import torch +from flwr.app import ArrayRecord, ConfigRecord, Context, MetricRecord +from flwr.serverapp import Grid, ServerApp +from flwr.serverapp.strategy import FedAvg + +from pytorchexample.task import Net, load_centralized_dataset, test + +# Create ServerApp +app = ServerApp() + + +@app.main() +def main(grid: Grid, context: Context) -> None: + """Main entry point for the ServerApp.""" + + # Read run config + fraction_evaluate: float = context.run_config["fraction-evaluate"] + num_rounds: int = context.run_config["num-server-rounds"] + lr: float = context.run_config["learning-rate"] + + # Load global model + global_model = Net() + arrays = ArrayRecord(global_model.state_dict()) + + # Initialize FedAvg strategy + strategy = FedAvg(fraction_evaluate=fraction_evaluate) + + # Start strategy, run FedAvg for `num_rounds` + result = strategy.start( + grid=grid, + initial_arrays=arrays, + train_config=ConfigRecord({"lr": lr}), + num_rounds=num_rounds, + evaluate_fn=global_evaluate, + ) + + # Save final model to disk + print("\nSaving final model to disk...") + state_dict = result.arrays.to_torch_state_dict() + torch.save(state_dict, "final_model.pt") + + +def global_evaluate(server_round: int, arrays: ArrayRecord) -> MetricRecord: + """Evaluate model on central data.""" + + # Load the model and initialize it with the received weights + model = Net() + model.load_state_dict(arrays.to_torch_state_dict()) + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + model.to(device) + + # Load entire test set + test_dataloader = load_centralized_dataset() + + # Evaluate the global model on the test set + test_loss, test_acc = test(model, test_dataloader, device) + + # Return the evaluation metrics + return MetricRecord({"accuracy": test_acc, "loss": test_loss}) diff --git a/packages/federated/flower-superexec/app/pytorchexample/task.py b/packages/federated/flower-superexec/app/pytorchexample/task.py new file mode 100644 index 00000000..f701ceb8 --- /dev/null +++ b/packages/federated/flower-superexec/app/pytorchexample/task.py @@ -0,0 +1,109 @@ +"""pytorchexample: A Flower / PyTorch app.""" + +import torch +import torch.nn as nn +import torch.nn.functional as F +from datasets import load_dataset +from flwr_datasets import FederatedDataset +from flwr_datasets.partitioner import IidPartitioner +from torch.utils.data import DataLoader +from torchvision.transforms import Compose, Normalize, ToTensor + + +class Net(nn.Module): + """Model (simple CNN adapted from 'PyTorch: A 60 Minute Blitz')""" + + def __init__(self): + super(Net, self).__init__() + self.conv1 = nn.Conv2d(3, 6, 5) + self.pool = nn.MaxPool2d(2, 2) + self.conv2 = nn.Conv2d(6, 16, 5) + self.fc1 = nn.Linear(16 * 5 * 5, 120) + self.fc2 = nn.Linear(120, 84) + self.fc3 = nn.Linear(84, 10) + + def forward(self, x): + x = self.pool(F.relu(self.conv1(x))) + x = self.pool(F.relu(self.conv2(x))) + x = x.view(-1, 16 * 5 * 5) + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + return self.fc3(x) + + +fds = None # Cache FederatedDataset + +pytorch_transforms = Compose([ToTensor(), Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) + + +def apply_transforms(batch): + """Apply transforms to the partition from FederatedDataset.""" + batch["img"] = [pytorch_transforms(img) for img in batch["img"]] + return batch + + +def load_data(partition_id: int, num_partitions: int, batch_size: int): + """Load partition CIFAR10 data.""" + # Only initialize `FederatedDataset` once + global fds + if fds is None: + partitioner = IidPartitioner(num_partitions=num_partitions) + fds = FederatedDataset( + dataset="uoft-cs/cifar10", + partitioners={"train": partitioner}, + ) + partition = fds.load_partition(partition_id) + # Divide data on each node: 80% train, 20% test + partition_train_test = partition.train_test_split(test_size=0.2, seed=42) + # Construct dataloaders + partition_train_test = partition_train_test.with_transform(apply_transforms) + trainloader = DataLoader( + partition_train_test["train"], batch_size=batch_size, shuffle=True + ) + testloader = DataLoader(partition_train_test["test"], batch_size=batch_size) + return trainloader, testloader + + +def load_centralized_dataset(): + """Load test set and return dataloader.""" + # Load entire test set + test_dataset = load_dataset("uoft-cs/cifar10", split="test") + dataset = test_dataset.with_format("torch").with_transform(apply_transforms) + return DataLoader(dataset, batch_size=128) + + +def train(net, trainloader, epochs, lr, device): + """Train the model on the training set.""" + net.to(device) # move model to GPU if available + criterion = torch.nn.CrossEntropyLoss().to(device) + optimizer = torch.optim.SGD(net.parameters(), lr=lr, momentum=0.9) + net.train() + running_loss = 0.0 + for _ in range(epochs): + for batch in trainloader: + images = batch["img"].to(device) + labels = batch["label"].to(device) + optimizer.zero_grad() + loss = criterion(net(images), labels) + loss.backward() + optimizer.step() + running_loss += loss.item() + avg_trainloss = running_loss / (epochs * len(trainloader)) + return avg_trainloss + + +def test(net, testloader, device): + """Validate the model on the test set.""" + net.to(device) + criterion = torch.nn.CrossEntropyLoss() + correct, loss = 0, 0.0 + with torch.no_grad(): + for batch in testloader: + images = batch["img"].to(device) + labels = batch["label"].to(device) + outputs = net(images) + loss += criterion(outputs, labels).item() + correct += (torch.max(outputs.data, 1)[1] == labels).sum().item() + accuracy = correct / len(testloader.dataset) + loss = loss / len(testloader) + return loss, accuracy diff --git a/packages/federated/flower-superexec/config.yaml b/packages/federated/flower-superexec/config.yaml new file mode 100644 index 00000000..18957e53 --- /dev/null +++ b/packages/federated/flower-superexec/config.yaml @@ -0,0 +1,13 @@ +# Copyright(C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT + +# gpu_support defaults to true — this is the only flower-* Ryzer that +# actually runs training, so ROCm access matters here. + +volume_mappings: +- "$PWD/workspace/flower/superlink-certificates:/app/certificates:ro" + +# Strix Point (gfx1150) compatibility shim — uncomment if your iGPU +# isn't directly recognised by the ROCm runtime. +# environment_variables: +# - "HSA_OVERRIDE_GFX_VERSION=11.0.0" diff --git a/packages/federated/flower-superexec/test.sh b/packages/federated/flower-superexec/test.sh new file mode 100644 index 00000000..79b18585 --- /dev/null +++ b/packages/federated/flower-superexec/test.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Smoke test: validate that the flower-superexec CLI is available, that +# the bundled quickstart-pytorch app imports cleanly, and that PyTorch +# can see the ROCm GPU. + +set -e + +echo "Running tests for flower-superexec..." + +flower-superexec --help >/dev/null +flower-superexec --help 2>&1 | grep -q -- "--plugin-type" \ + || { echo "FAIL: --plugin-type flag missing from help"; exit 1; } + +python3 - <<'PY' +import torch +import flwr +from pytorchexample import server_app, client_app + +print(f"flwr version: {flwr.__version__}") +print(f"torch version: {torch.__version__} (HIP: {torch.version.hip})") +print(f"torch.cuda.is_available(): {torch.cuda.is_available()}") +assert hasattr(server_app, "app"), "pytorchexample.server_app.app missing" +assert hasattr(client_app, "app"), "pytorchexample.client_app.app missing" +print("quickstart-pytorch ServerApp and ClientApp imported successfully.") +PY + +echo "Tests passed!" diff --git a/packages/federated/flower-superlink/Dockerfile b/packages/federated/flower-superlink/Dockerfile new file mode 100644 index 00000000..8f6a3836 --- /dev/null +++ b/packages/federated/flower-superlink/Dockerfile @@ -0,0 +1,26 @@ +# Copyright(C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT + +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +# Install openssl (used by gen-certs.sh) — base image normally has it, but be safe. +RUN apt-get update && apt-get install -y --no-install-recommends \ + openssl \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /ryzers +COPY test.sh /ryzers/test_flower-superlink.sh +COPY gen-certs.sh /ryzers/gen-certs.sh +RUN chmod +x /ryzers/test_flower-superlink.sh /ryzers/gen-certs.sh + +# Persisted SuperLink state and TLS certificates are expected to be +# mounted from the host (see config.yaml). +RUN mkdir -p /app/state /app/certificates + +# 9091 = ExecApi (control plane: flwr run / serverapp superexec) +# 9092 = FleetApi (data plane: supernodes connect here) +# 9093 = ServerAppIo (legacy / Exec API TLS) +EXPOSE 9091 9092 9093 + +CMD /ryzers/test_flower-superlink.sh diff --git a/packages/federated/flower-superlink/README.md b/packages/federated/flower-superlink/README.md new file mode 100644 index 00000000..60226a9d --- /dev/null +++ b/packages/federated/flower-superlink/README.md @@ -0,0 +1,65 @@ +# Flower SuperLink Docker Setup + +The SuperLink is the central coordinator in a Flower deployment. It accepts +connections from SuperNodes (Fleet API, port 9092), runs the control plane +(Exec API, port 9091), and persists run state to `/app/state`. + +This Ryzer runs on the **server machine** and is the first component you +start when bringing up a federation. + +## Build + +```sh +ryzers build flower-base flower-superlink +``` + +## Run (insecure — testing only) + +The default `CMD` runs `test_flower-superlink.sh`, which starts the +SuperLink with `--insecure` and verifies it binds port 9092. To run it +properly, override the CMD: + +```sh +ryzers run flower-superlink --insecure +``` + +## Run (with TLS — for real deployments) + +1. Generate certs on the server machine, specifying its routable IP: + + ```sh + SUPERLINK_IP=192.168.2.33 \ + bash packages/federated/flower-superlink/gen-certs.sh + ``` + + This produces `ca.crt`, `server.pem`, `server.key` under + `./workspace/flower/superlink-certificates/`, which the Ryzer's + `config.yaml` mounts read-only at `/app/certificates/`. + +2. Copy `ca.crt` to every client machine (the SuperNodes will need it). + +3. Start the SuperLink: + + ```sh + ryzers run flower-superlink \ + --ssl-ca-certfile=/app/certificates/ca.crt \ + --ssl-certfile=/app/certificates/server.pem \ + --ssl-keyfile=/app/certificates/server.key \ + --database=/app/state/state.db \ + --isolation=process + ``` + +## Ports + +| Port | API | Used by | +|------|-----|---------| +| 9091 | ExecApi | local `flower-superexec --plugin-type serverapp` | +| 9092 | FleetApi | remote SuperNodes | +| 9093 | ServerAppIo | (legacy / TLS control) | + +## References + +- [SuperLink reference](https://flower.ai/docs/framework/ref-api-cli.html#flower-superlink) +- [Multi-machine Docker tutorial](https://flower.ai/docs/framework/docker/tutorial-deploy-on-multiple-machines.html) + +Copyright(C) 2026 Advanced Micro Devices, Inc. All rights reserved. diff --git a/packages/federated/flower-superlink/config.yaml b/packages/federated/flower-superlink/config.yaml new file mode 100644 index 00000000..9e42d5f1 --- /dev/null +++ b/packages/federated/flower-superlink/config.yaml @@ -0,0 +1,16 @@ +# Copyright(C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT + +# SuperLink is the coordinator. It does not run training and does not +# need the GPU or an X display. +gpu_support: false +x11_display: false + +port_mappings: +- "9091:9091" # ExecApi (control plane) +- "9092:9092" # FleetApi (supernodes connect here) +- "9093:9093" # ServerAppIo + +volume_mappings: +- "$PWD/workspace/flower/state:/app/state" +- "$PWD/workspace/flower/superlink-certificates:/app/certificates:ro" diff --git a/packages/federated/flower-superlink/gen-certs.sh b/packages/federated/flower-superlink/gen-certs.sh new file mode 100644 index 00000000..fd69d44f --- /dev/null +++ b/packages/federated/flower-superlink/gen-certs.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Generate a self-signed CA + server certificate for the Flower SuperLink. +# +# Usage: SUPERLINK_IP=192.168.2.33 ./gen-certs.sh [output-dir] +# Default: SUPERLINK_IP=127.0.0.1, output-dir=$PWD/workspace/flower/superlink-certificates +# +# Outputs (matching the upstream tutorial's expected layout): +# /ca.crt — root cert; copy to each client machine +# /server.pem — server cert (SAN includes SUPERLINK_IP) +# /server.key — server private key +# +# After generation, the flower-superlink Ryzer will pick these up via its +# volume mount (config.yaml maps the output dir to /app/certificates:ro). + +set -euo pipefail + +SUPERLINK_IP="${SUPERLINK_IP:-127.0.0.1}" +OUT_DIR="${1:-$PWD/workspace/flower/superlink-certificates}" + +mkdir -p "$OUT_DIR" +cd "$OUT_DIR" + +echo "Generating Flower TLS certs for SUPERLINK_IP=${SUPERLINK_IP} in ${OUT_DIR}" + +# 1. CA +openssl genrsa -out ca.key 4096 +openssl req -x509 -new -nodes -key ca.key -sha256 -days 3650 \ + -subj "/CN=Flower SuperLink CA" \ + -out ca.crt + +# 2. Server key + CSR +openssl genrsa -out server.key 4096 +openssl req -new -key server.key \ + -subj "/CN=${SUPERLINK_IP}" \ + -out server.csr + +# 3. Server cert signed by the CA, with SAN +cat >server.ext </dev/null + +flower-superlink --insecure & +PID=$! +trap "kill $PID 2>/dev/null || true" EXIT + +# Wait up to 15s for port 9092 to be listening +for i in $(seq 1 15); do + if (echo >/dev/tcp/127.0.0.1/9092) >/dev/null 2>&1; then + echo "SuperLink is listening on 9092" + echo "Tests passed!" + exit 0 + fi + sleep 1 +done + +echo "FAIL: SuperLink did not bind port 9092 within 15s" +exit 1 diff --git a/packages/federated/flower-supernode/Dockerfile b/packages/federated/flower-supernode/Dockerfile new file mode 100644 index 00000000..0fa140ff --- /dev/null +++ b/packages/federated/flower-supernode/Dockerfile @@ -0,0 +1,16 @@ +# Copyright(C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT + +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +WORKDIR /ryzers +COPY test.sh /ryzers/test_flower-supernode.sh +RUN chmod +x /ryzers/test_flower-supernode.sh + +RUN mkdir -p /app/certificates + +# ClientAppIo port — a paired superexec (clientapp plugin) connects here. +EXPOSE 9094 + +CMD /ryzers/test_flower-supernode.sh diff --git a/packages/federated/flower-supernode/README.md b/packages/federated/flower-supernode/README.md new file mode 100644 index 00000000..e919cf15 --- /dev/null +++ b/packages/federated/flower-supernode/README.md @@ -0,0 +1,57 @@ +# Flower SuperNode Docker Setup + +The SuperNode runs on each **client machine** in a Flower deployment. It +connects to the SuperLink's Fleet API (port 9092) and exposes a local +ClientAppIo socket (port 9094) for the paired `flower-superexec +--plugin-type clientapp` to attach to. + +Pair this Ryzer with `flower-superexec` on every client host. The +SuperNode does the federation plumbing; the superexec runs the actual +PyTorch/ROCm training. + +## Build + +```sh +ryzers build flower-base flower-supernode +``` + +## Run + +The default `CMD` runs `test_flower-supernode.sh`, which only validates +the CLI. To start a real SuperNode, override the CMD and point it at +your SuperLink: + +### Insecure (testing only) + +```sh +ryzers run flower-supernode \ + --superlink ${SUPERLINK_IP}:9092 \ + --clientappio-api-address 0.0.0.0:9094 \ + --insecure \ + --node-config "partition-id=0 num-partitions=2" \ + --isolation process +``` + +### With TLS (using `ca.crt` copied from the server machine) + +Place `ca.crt` at `./workspace/flower/superlink-certificates/ca.crt` +(the volume mount in `config.yaml` makes it visible inside the container +at `/app/certificates/ca.crt`), then: + +```sh +ryzers run flower-supernode \ + --superlink ${SUPERLINK_IP}:9092 \ + --clientappio-api-address 0.0.0.0:9094 \ + --root-certificates /app/certificates/ca.crt \ + --node-config "partition-id=0 num-partitions=2" \ + --isolation process +``` + +`partition-id` should be unique per client; `num-partitions` is the total +number of clients across the federation. + +## References + +- [SuperNode reference](https://flower.ai/docs/framework/ref-api-cli.html#flower-supernode) + +Copyright(C) 2026 Advanced Micro Devices, Inc. All rights reserved. diff --git a/packages/federated/flower-supernode/config.yaml b/packages/federated/flower-supernode/config.yaml new file mode 100644 index 00000000..5969ef0f --- /dev/null +++ b/packages/federated/flower-supernode/config.yaml @@ -0,0 +1,18 @@ +# Copyright(C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT + +# The SuperNode itself does not run training; it just orchestrates the +# paired ClientApp superexec. GPU support is left enabled (default) so +# that running on the same host as the superexec is straightforward, but +# you can flip it off if running them in separate containers. + +port_mappings: +- "9094:9094" # ClientAppIo (local superexec connects here) + +volume_mappings: +- "$PWD/workspace/flower/superlink-certificates:/app/certificates:ro" + +# SUPERLINK_IP is consumed by the run-time CMD override: +# ryzers run flower-supernode --superlink ${SUPERLINK_IP}:9092 ... +environment_variables: +- "SUPERLINK_IP=127.0.0.1" diff --git a/packages/federated/flower-supernode/test.sh b/packages/federated/flower-supernode/test.sh new file mode 100644 index 00000000..d7e06417 --- /dev/null +++ b/packages/federated/flower-supernode/test.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Smoke test: verify the flower-supernode binary is installed and the +# --help output renders. We do NOT attempt to connect to a SuperLink in +# the test, since that would require a live federation. + +set -e + +echo "Running tests for flower-supernode..." + +flower-supernode --help >/dev/null + +# Sanity-check that the expected flags are documented +flower-supernode --help 2>&1 | grep -q -- "--superlink" \ + || { echo "FAIL: --superlink flag missing from help"; exit 1; } +flower-supernode --help 2>&1 | grep -q -- "--clientappio-api-address" \ + || { echo "FAIL: --clientappio-api-address flag missing from help"; exit 1; } + +echo "flower-supernode CLI looks good." +echo "Tests passed!" From 3c72a75f3fcf12a3a3986f35f76cf43fcb408641 Mon Sep 17 00:00:00 2001 From: Ken O'Brien Date: Tue, 9 Jun 2026 13:27:40 +0100 Subject: [PATCH 02/20] fixes for demo --- .../federated/flower-superexec/Dockerfile | 3 +- packages/federated/flower-superexec/README.md | 68 +++++++++++-------- .../federated/flower-superexec/config.yaml | 8 ++- .../flower-superexec/run-superexec.sh | 48 +++++++++++++ .../federated/flower-superlink/Dockerfile | 3 +- packages/federated/flower-superlink/README.md | 40 +++++++---- .../federated/flower-superlink/config.yaml | 7 ++ .../flower-superlink/run-superlink.sh | 34 ++++++++++ .../federated/flower-supernode/Dockerfile | 3 +- packages/federated/flower-supernode/README.md | 51 ++++++++------ .../federated/flower-supernode/config.yaml | 10 ++- .../flower-supernode/run-supernode.sh | 37 ++++++++++ 12 files changed, 242 insertions(+), 70 deletions(-) create mode 100644 packages/federated/flower-superexec/run-superexec.sh create mode 100644 packages/federated/flower-superlink/run-superlink.sh create mode 100644 packages/federated/flower-supernode/run-supernode.sh diff --git a/packages/federated/flower-superexec/Dockerfile b/packages/federated/flower-superexec/Dockerfile index e7843c67..45e180f9 100644 --- a/packages/federated/flower-superexec/Dockerfile +++ b/packages/federated/flower-superexec/Dockerfile @@ -13,7 +13,8 @@ RUN pip3 install --no-cache-dir --break-system-packages /app WORKDIR /ryzers COPY test.sh /ryzers/test_flower-superexec.sh -RUN chmod +x /ryzers/test_flower-superexec.sh +COPY run-superexec.sh /ryzers/run-superexec.sh +RUN chmod +x /ryzers/test_flower-superexec.sh /ryzers/run-superexec.sh RUN mkdir -p /app/certificates diff --git a/packages/federated/flower-superexec/README.md b/packages/federated/flower-superexec/README.md index f2c598ff..d2fd4f72 100644 --- a/packages/federated/flower-superexec/README.md +++ b/packages/federated/flower-superexec/README.md @@ -15,59 +15,62 @@ preinstalled at `/app/`. ryzers build flower-base flower-superexec ``` -## Run — ServerApp (one per federation, on the server machine) +## Run + +`ryzers run` only accepts a single-token CMD override, so all superexec +flags are wrapped by `/ryzers/run-superexec.sh`, which reads env vars +declared (with shell-expansion defaults) in `config.yaml`. The role +(`serverapp` vs `clientapp`) is selected via `FLOWER_PLUGIN_TYPE`. + +### Run as ServerApp (server machine) ```sh -ryzers run flower-superexec \ - --insecure \ - --plugin-type serverapp \ - --appio-api-address ${SUPERLINK_HOST:-127.0.0.1}:9091 +export FLOWER_PLUGIN_TYPE=serverapp +export FLOWER_APPIO_ADDR=127.0.0.1:9091 # local SuperLink +ryzers run /ryzers/run-superexec.sh ``` -## Run — ClientApp (one per SuperNode, on each client machine) +### Run as ClientApp (client machine) ```sh -ryzers run flower-superexec \ - --insecure \ - --plugin-type clientapp \ - --appio-api-address ${SUPERNODE_HOST:-127.0.0.1}:9094 +export FLOWER_PLUGIN_TYPE=clientapp +export FLOWER_APPIO_ADDR=127.0.0.1:9094 # local SuperNode +ryzers run /ryzers/run-superexec.sh ``` ## Putting it together (mirrors the upstream tutorial) -**On the server machine** (one SuperLink + one ServerApp superexec): +**On the server machine** (SuperLink + ServerApp superexec): ```sh ryzers build flower-base flower-superlink -ryzers build flower-base flower-superexec # separate image, ServerApp role +ryzers build flower-base flower-superexec # Terminal 1 — SuperLink -ryzers run flower-superlink --insecure --isolation process +ryzers run /ryzers/run-superlink.sh # Terminal 2 — ServerApp superexec -ryzers run flower-superexec \ - --insecure --plugin-type serverapp \ - --appio-api-address 127.0.0.1:9091 +export FLOWER_PLUGIN_TYPE=serverapp +export FLOWER_APPIO_ADDR=127.0.0.1:9091 +ryzers run /ryzers/run-superexec.sh ``` -**On each client machine** (one SuperNode + one ClientApp superexec): +**On each client machine** (SuperNode + ClientApp superexec): ```sh ryzers build flower-base flower-supernode ryzers build flower-base flower-superexec # Terminal 1 — SuperNode (connects to remote SuperLink) -ryzers run flower-supernode \ - --insecure \ - --superlink ${SUPERLINK_IP}:9092 \ - --clientappio-api-address 0.0.0.0:9094 \ - --node-config "partition-id=0 num-partitions=2" \ - --isolation process +export SUPERLINK_IP=192.168.2.33 +export FLOWER_PARTITION_ID=0 +export FLOWER_NUM_PARTITIONS=2 +ryzers run /ryzers/run-supernode.sh # Terminal 2 — ClientApp superexec (runs the PyTorch training) -ryzers run flower-superexec \ - --insecure --plugin-type clientapp \ - --appio-api-address 127.0.0.1:9094 +export FLOWER_PLUGIN_TYPE=clientapp +export FLOWER_APPIO_ADDR=127.0.0.1:9094 +ryzers run /ryzers/run-superexec.sh ``` Then from any machine with the `flwr` CLI installed: @@ -76,9 +79,16 @@ Then from any machine with the `flwr` CLI installed: flwr run /app remote-deployment # see config.toml in upstream tutorial ``` -For TLS (production), see `flower-superlink/gen-certs.sh` and pass -`--root-certificates /app/certificates/ca.crt` to the superexec instead -of `--insecure`. +For TLS, set `FLOWER_INSECURE=0` on each component and stage the certs +generated by `flower-superlink/gen-certs.sh` (see that Ryzer's README). + +### Env-var reference + +| Variable | Default | Purpose | +|----------|---------|---------| +| `FLOWER_PLUGIN_TYPE` | `serverapp` | `serverapp` or `clientapp` | +| `FLOWER_APPIO_ADDR` | `127.0.0.1:9091` | SuperLink :9091 (serverapp) or SuperNode :9094 (clientapp) | +| `FLOWER_INSECURE` | `1` | `1` = `--insecure`; `0` = TLS via `FLOWER_CA_CERT` | ## Bundled example diff --git a/packages/federated/flower-superexec/config.yaml b/packages/federated/flower-superexec/config.yaml index 18957e53..96ae2ae3 100644 --- a/packages/federated/flower-superexec/config.yaml +++ b/packages/federated/flower-superexec/config.yaml @@ -7,7 +7,13 @@ volume_mappings: - "$PWD/workspace/flower/superlink-certificates:/app/certificates:ro" +# Consumed by /ryzers/run-superexec.sh. The two important knobs are +# FLOWER_PLUGIN_TYPE (serverapp | clientapp) and FLOWER_APPIO_ADDR +# (the SuperLink :9091 for serverapp, the SuperNode :9094 for clientapp). +environment_variables: +- "FLOWER_PLUGIN_TYPE=${FLOWER_PLUGIN_TYPE:-serverapp}" +- "FLOWER_APPIO_ADDR=${FLOWER_APPIO_ADDR:-127.0.0.1:9091}" +- "FLOWER_INSECURE=${FLOWER_INSECURE:-1}" # Strix Point (gfx1150) compatibility shim — uncomment if your iGPU # isn't directly recognised by the ROCm runtime. -# environment_variables: # - "HSA_OVERRIDE_GFX_VERSION=11.0.0" diff --git a/packages/federated/flower-superexec/run-superexec.sh b/packages/federated/flower-superexec/run-superexec.sh new file mode 100644 index 00000000..3fec5c4a --- /dev/null +++ b/packages/federated/flower-superexec/run-superexec.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Entry point for a flower-superexec process (the runner used for both +# ServerApp and ClientApp roles in Flower's process-isolation model). +# +# Invoke via: +# ryzers run /ryzers/run-superexec.sh +# +# Required env vars: +# FLOWER_PLUGIN_TYPE — "serverapp" or "clientapp" (default serverapp) +# FLOWER_APPIO_ADDR — host:port of the paired SuperLink (ServerApp) +# or SuperNode (ClientApp). +# Defaults to 127.0.0.1:9091 (matches a local +# SuperLink). For ClientApp, set to your local +# SuperNode, e.g. 127.0.0.1:9094. + +set -e + +FLOWER_PLUGIN_TYPE="${FLOWER_PLUGIN_TYPE:-serverapp}" +FLOWER_INSECURE="${FLOWER_INSECURE:-1}" + +case "${FLOWER_PLUGIN_TYPE}" in + serverapp) DEFAULT_ADDR="127.0.0.1:9091" ;; + clientapp) DEFAULT_ADDR="127.0.0.1:9094" ;; + *) + echo "FLOWER_PLUGIN_TYPE must be 'serverapp' or 'clientapp' (got: ${FLOWER_PLUGIN_TYPE})" >&2 + exit 2 + ;; +esac + +FLOWER_APPIO_ADDR="${FLOWER_APPIO_ADDR:-${DEFAULT_ADDR}}" + +ARGS=( + --plugin-type "${FLOWER_PLUGIN_TYPE}" + --appio-api-address "${FLOWER_APPIO_ADDR}" +) + +if [ "${FLOWER_INSECURE}" = "1" ]; then + ARGS+=(--insecure) +else + ARGS+=(--root-certificates "${FLOWER_CA_CERT:-/app/certificates/ca.crt}") +fi + +echo "Starting: flower-superexec ${ARGS[*]}" +exec flower-superexec "${ARGS[@]}" diff --git a/packages/federated/flower-superlink/Dockerfile b/packages/federated/flower-superlink/Dockerfile index 8f6a3836..d7471570 100644 --- a/packages/federated/flower-superlink/Dockerfile +++ b/packages/federated/flower-superlink/Dockerfile @@ -12,7 +12,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ WORKDIR /ryzers COPY test.sh /ryzers/test_flower-superlink.sh COPY gen-certs.sh /ryzers/gen-certs.sh -RUN chmod +x /ryzers/test_flower-superlink.sh /ryzers/gen-certs.sh +COPY run-superlink.sh /ryzers/run-superlink.sh +RUN chmod +x /ryzers/test_flower-superlink.sh /ryzers/gen-certs.sh /ryzers/run-superlink.sh # Persisted SuperLink state and TLS certificates are expected to be # mounted from the host (see config.yaml). diff --git a/packages/federated/flower-superlink/README.md b/packages/federated/flower-superlink/README.md index 60226a9d..6f6e7346 100644 --- a/packages/federated/flower-superlink/README.md +++ b/packages/federated/flower-superlink/README.md @@ -13,17 +13,24 @@ start when bringing up a federation. ryzers build flower-base flower-superlink ``` -## Run (insecure — testing only) +## Run -The default `CMD` runs `test_flower-superlink.sh`, which starts the -SuperLink with `--insecure` and verifies it binds port 9092. To run it -properly, override the CMD: +`ryzers run` only accepts a single-token CMD override, so all SuperLink +flags are wrapped by an entry-point script (`/ryzers/run-superlink.sh`) +that reads its config from environment variables. The relevant env vars +are declared in `config.yaml` with shell-expansion defaults — override +them by exporting in your shell before `ryzers run`. + +The default `CMD` runs `test_flower-superlink.sh`, a one-shot install +check. For a real launch, use the entry-point script: + +### Insecure (local testing) ```sh -ryzers run flower-superlink --insecure +ryzers run /ryzers/run-superlink.sh ``` -## Run (with TLS — for real deployments) +### With TLS (real deployment) 1. Generate certs on the server machine, specifying its routable IP: @@ -38,17 +45,24 @@ ryzers run flower-superlink --insecure 2. Copy `ca.crt` to every client machine (the SuperNodes will need it). -3. Start the SuperLink: +3. Launch the SuperLink with TLS: ```sh - ryzers run flower-superlink \ - --ssl-ca-certfile=/app/certificates/ca.crt \ - --ssl-certfile=/app/certificates/server.pem \ - --ssl-keyfile=/app/certificates/server.key \ - --database=/app/state/state.db \ - --isolation=process + export FLOWER_INSECURE=0 + ryzers run /ryzers/run-superlink.sh ``` + To override the cert paths inside the container, also export + `FLOWER_CA_CERT`, `FLOWER_SERVER_CERT`, `FLOWER_SERVER_KEY`. + +### Env-var reference + +| Variable | Default | Purpose | +|----------|---------|---------| +| `FLOWER_INSECURE` | `1` | `1` = `--insecure`; `0` = enable TLS flags | +| `FLOWER_ISOLATION` | `process` | Passed to `--isolation` | +| `FLOWER_STATE_DB` | `/app/state/state.db` | Persisted run state | + ## Ports | Port | API | Used by | diff --git a/packages/federated/flower-superlink/config.yaml b/packages/federated/flower-superlink/config.yaml index 9e42d5f1..d7b31792 100644 --- a/packages/federated/flower-superlink/config.yaml +++ b/packages/federated/flower-superlink/config.yaml @@ -14,3 +14,10 @@ port_mappings: volume_mappings: - "$PWD/workspace/flower/state:/app/state" - "$PWD/workspace/flower/superlink-certificates:/app/certificates:ro" + +# These are consumed by /ryzers/run-superlink.sh. Override by exporting +# the same variables in your shell before `ryzers run`. +environment_variables: +- "FLOWER_INSECURE=${FLOWER_INSECURE:-1}" +- "FLOWER_ISOLATION=${FLOWER_ISOLATION:-process}" +- "FLOWER_STATE_DB=${FLOWER_STATE_DB:-/app/state/state.db}" diff --git a/packages/federated/flower-superlink/run-superlink.sh b/packages/federated/flower-superlink/run-superlink.sh new file mode 100644 index 00000000..a88894a6 --- /dev/null +++ b/packages/federated/flower-superlink/run-superlink.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Entry point for the SuperLink. Invoke via: +# ryzers run /ryzers/run-superlink.sh +# +# All knobs are env vars (defaults make insecure local testing work +# out of the box). Override by exporting them in your shell before +# calling `ryzers run` — config.yaml propagates them via shell expansion. + +set -e + +FLOWER_INSECURE="${FLOWER_INSECURE:-1}" +FLOWER_ISOLATION="${FLOWER_ISOLATION:-process}" +FLOWER_STATE_DB="${FLOWER_STATE_DB:-/app/state/state.db}" + +ARGS=(--isolation "${FLOWER_ISOLATION}" --database "${FLOWER_STATE_DB}") + +if [ "${FLOWER_INSECURE}" = "1" ]; then + ARGS+=(--insecure) +else + ARGS+=( + --ssl-ca-certfile "${FLOWER_CA_CERT:-/app/certificates/ca.crt}" + --ssl-certfile "${FLOWER_SERVER_CERT:-/app/certificates/server.pem}" + --ssl-keyfile "${FLOWER_SERVER_KEY:-/app/certificates/server.key}" + ) +fi + +mkdir -p "$(dirname "${FLOWER_STATE_DB}")" + +echo "Starting: flower-superlink ${ARGS[*]}" +exec flower-superlink "${ARGS[@]}" diff --git a/packages/federated/flower-supernode/Dockerfile b/packages/federated/flower-supernode/Dockerfile index 0fa140ff..befc1835 100644 --- a/packages/federated/flower-supernode/Dockerfile +++ b/packages/federated/flower-supernode/Dockerfile @@ -6,7 +6,8 @@ FROM ${BASE_IMAGE} WORKDIR /ryzers COPY test.sh /ryzers/test_flower-supernode.sh -RUN chmod +x /ryzers/test_flower-supernode.sh +COPY run-supernode.sh /ryzers/run-supernode.sh +RUN chmod +x /ryzers/test_flower-supernode.sh /ryzers/run-supernode.sh RUN mkdir -p /app/certificates diff --git a/packages/federated/flower-supernode/README.md b/packages/federated/flower-supernode/README.md index e919cf15..9489d6e4 100644 --- a/packages/federated/flower-supernode/README.md +++ b/packages/federated/flower-supernode/README.md @@ -17,38 +17,47 @@ ryzers build flower-base flower-supernode ## Run -The default `CMD` runs `test_flower-supernode.sh`, which only validates -the CLI. To start a real SuperNode, override the CMD and point it at -your SuperLink: +`ryzers run` only accepts a single-token CMD override, so the SuperNode +flags are wrapped by `/ryzers/run-supernode.sh`, which reads env vars +declared (with shell-expansion defaults) in `config.yaml`. Export them +in your shell before `ryzers run` to override. -### Insecure (testing only) +The default `CMD` runs `test_flower-supernode.sh`, an install check. +For a real launch: + +### Insecure (testing) ```sh -ryzers run flower-supernode \ - --superlink ${SUPERLINK_IP}:9092 \ - --clientappio-api-address 0.0.0.0:9094 \ - --insecure \ - --node-config "partition-id=0 num-partitions=2" \ - --isolation process +export SUPERLINK_IP=192.168.2.33 +export FLOWER_PARTITION_ID=0 +export FLOWER_NUM_PARTITIONS=2 +ryzers run /ryzers/run-supernode.sh ``` -### With TLS (using `ca.crt` copied from the server machine) +### With TLS Place `ca.crt` at `./workspace/flower/superlink-certificates/ca.crt` -(the volume mount in `config.yaml` makes it visible inside the container -at `/app/certificates/ca.crt`), then: +(the volume mount in `config.yaml` exposes it at `/app/certificates/ca.crt`), +then: ```sh -ryzers run flower-supernode \ - --superlink ${SUPERLINK_IP}:9092 \ - --clientappio-api-address 0.0.0.0:9094 \ - --root-certificates /app/certificates/ca.crt \ - --node-config "partition-id=0 num-partitions=2" \ - --isolation process +export SUPERLINK_IP=192.168.2.33 +export FLOWER_INSECURE=0 +export FLOWER_PARTITION_ID=0 +export FLOWER_NUM_PARTITIONS=2 +ryzers run /ryzers/run-supernode.sh ``` -`partition-id` should be unique per client; `num-partitions` is the total -number of clients across the federation. +### Env-var reference + +| Variable | Default | Purpose | +|----------|---------|---------| +| `SUPERLINK_IP` | `127.0.0.1` | Routable IP of the SuperLink host | +| `FLOWER_INSECURE` | `1` | `1` = `--insecure`; `0` = TLS via `FLOWER_CA_CERT` | +| `FLOWER_PARTITION_ID` | `0` | Unique partition for this node (0..N-1) | +| `FLOWER_NUM_PARTITIONS` | `2` | Total clients across the federation | +| `FLOWER_CLIENTAPPIO` | `0.0.0.0:9094` | Local socket the paired ClientApp connects to | +| `FLOWER_ISOLATION` | `process` | Passed to `--isolation` | ## References diff --git a/packages/federated/flower-supernode/config.yaml b/packages/federated/flower-supernode/config.yaml index 5969ef0f..fab615e7 100644 --- a/packages/federated/flower-supernode/config.yaml +++ b/packages/federated/flower-supernode/config.yaml @@ -12,7 +12,11 @@ port_mappings: volume_mappings: - "$PWD/workspace/flower/superlink-certificates:/app/certificates:ro" -# SUPERLINK_IP is consumed by the run-time CMD override: -# ryzers run flower-supernode --superlink ${SUPERLINK_IP}:9092 ... +# Consumed by /ryzers/run-supernode.sh. Override by exporting the same +# variables in your shell before `ryzers run`. SUPERLINK_IP is the most +# important one — set it to the routable IP of your SuperLink host. environment_variables: -- "SUPERLINK_IP=127.0.0.1" +- "SUPERLINK_IP=${SUPERLINK_IP:-127.0.0.1}" +- "FLOWER_INSECURE=${FLOWER_INSECURE:-1}" +- "FLOWER_PARTITION_ID=${FLOWER_PARTITION_ID:-0}" +- "FLOWER_NUM_PARTITIONS=${FLOWER_NUM_PARTITIONS:-2}" diff --git a/packages/federated/flower-supernode/run-supernode.sh b/packages/federated/flower-supernode/run-supernode.sh new file mode 100644 index 00000000..ba082607 --- /dev/null +++ b/packages/federated/flower-supernode/run-supernode.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Entry point for a SuperNode. Invoke via: +# ryzers run /ryzers/run-supernode.sh +# +# Required: +# SUPERLINK_IP — IP of the remote SuperLink (defaults to 127.0.0.1) +# FLOWER_PARTITION_ID — this node's partition (default 0) +# FLOWER_NUM_PARTITIONS — total nodes in the federation (default 2) + +set -e + +SUPERLINK_IP="${SUPERLINK_IP:-127.0.0.1}" +FLOWER_INSECURE="${FLOWER_INSECURE:-1}" +FLOWER_PARTITION_ID="${FLOWER_PARTITION_ID:-0}" +FLOWER_NUM_PARTITIONS="${FLOWER_NUM_PARTITIONS:-2}" +FLOWER_ISOLATION="${FLOWER_ISOLATION:-process}" +FLOWER_CLIENTAPPIO="${FLOWER_CLIENTAPPIO:-0.0.0.0:9094}" + +ARGS=( + --superlink "${SUPERLINK_IP}:9092" + --clientappio-api-address "${FLOWER_CLIENTAPPIO}" + --isolation "${FLOWER_ISOLATION}" + --node-config "partition-id=${FLOWER_PARTITION_ID} num-partitions=${FLOWER_NUM_PARTITIONS}" +) + +if [ "${FLOWER_INSECURE}" = "1" ]; then + ARGS+=(--insecure) +else + ARGS+=(--root-certificates "${FLOWER_CA_CERT:-/app/certificates/ca.crt}") +fi + +echo "Starting: flower-supernode ${ARGS[*]}" +exec flower-supernode "${ARGS[@]}" From 1aaeea01fe60afe425e6c68ad8e0a4b05ca10006 Mon Sep 17 00:00:00 2001 From: Ken O'Brien Date: Tue, 9 Jun 2026 13:35:22 +0100 Subject: [PATCH 03/20] fixes for demo --- .../federated/flower-superexec/Dockerfile | 5 ++++- packages/federated/flower-superexec/README.md | 19 +++++++++---------- .../federated/flower-superlink/Dockerfile | 5 ++++- packages/federated/flower-superlink/README.md | 15 +++++---------- .../federated/flower-supernode/Dockerfile | 5 ++++- packages/federated/flower-supernode/README.md | 14 +++++--------- 6 files changed, 31 insertions(+), 32 deletions(-) diff --git a/packages/federated/flower-superexec/Dockerfile b/packages/federated/flower-superexec/Dockerfile index 45e180f9..84cfa260 100644 --- a/packages/federated/flower-superexec/Dockerfile +++ b/packages/federated/flower-superexec/Dockerfile @@ -22,4 +22,7 @@ RUN mkdir -p /app/certificates # at supernode:9094; when running --plugin-type serverapp it connects out # to the SuperLink at superlink:9091. No inbound ports are exposed. -CMD /ryzers/test_flower-superexec.sh +# Default CMD is the role launcher (reads FLOWER_PLUGIN_TYPE etc. from +# config.yaml). To run the install-validation smoke test instead: +# ryzers run /ryzers/test_flower-superexec.sh +CMD /ryzers/run-superexec.sh diff --git a/packages/federated/flower-superexec/README.md b/packages/federated/flower-superexec/README.md index d2fd4f72..723d0492 100644 --- a/packages/federated/flower-superexec/README.md +++ b/packages/federated/flower-superexec/README.md @@ -17,9 +17,8 @@ ryzers build flower-base flower-superexec ## Run -`ryzers run` only accepts a single-token CMD override, so all superexec -flags are wrapped by `/ryzers/run-superexec.sh`, which reads env vars -declared (with shell-expansion defaults) in `config.yaml`. The role +SuperExec flags are driven by environment variables declared in +`config.yaml` (with shell-expansion defaults). The role (`serverapp` vs `clientapp`) is selected via `FLOWER_PLUGIN_TYPE`. ### Run as ServerApp (server machine) @@ -27,7 +26,7 @@ declared (with shell-expansion defaults) in `config.yaml`. The role ```sh export FLOWER_PLUGIN_TYPE=serverapp export FLOWER_APPIO_ADDR=127.0.0.1:9091 # local SuperLink -ryzers run /ryzers/run-superexec.sh +ryzers run ``` ### Run as ClientApp (client machine) @@ -35,7 +34,7 @@ ryzers run /ryzers/run-superexec.sh ```sh export FLOWER_PLUGIN_TYPE=clientapp export FLOWER_APPIO_ADDR=127.0.0.1:9094 # local SuperNode -ryzers run /ryzers/run-superexec.sh +ryzers run ``` ## Putting it together (mirrors the upstream tutorial) @@ -46,13 +45,13 @@ ryzers run /ryzers/run-superexec.sh ryzers build flower-base flower-superlink ryzers build flower-base flower-superexec -# Terminal 1 — SuperLink -ryzers run /ryzers/run-superlink.sh +# Terminal 1 — SuperLink (last built image) +ryzers run --name flower-superlink # Terminal 2 — ServerApp superexec export FLOWER_PLUGIN_TYPE=serverapp export FLOWER_APPIO_ADDR=127.0.0.1:9091 -ryzers run /ryzers/run-superexec.sh +ryzers run --name flower-superexec ``` **On each client machine** (SuperNode + ClientApp superexec): @@ -65,12 +64,12 @@ ryzers build flower-base flower-superexec export SUPERLINK_IP=192.168.2.33 export FLOWER_PARTITION_ID=0 export FLOWER_NUM_PARTITIONS=2 -ryzers run /ryzers/run-supernode.sh +ryzers run --name flower-supernode # Terminal 2 — ClientApp superexec (runs the PyTorch training) export FLOWER_PLUGIN_TYPE=clientapp export FLOWER_APPIO_ADDR=127.0.0.1:9094 -ryzers run /ryzers/run-superexec.sh +ryzers run --name flower-superexec ``` Then from any machine with the `flwr` CLI installed: diff --git a/packages/federated/flower-superlink/Dockerfile b/packages/federated/flower-superlink/Dockerfile index d7471570..5c197f10 100644 --- a/packages/federated/flower-superlink/Dockerfile +++ b/packages/federated/flower-superlink/Dockerfile @@ -24,4 +24,7 @@ RUN mkdir -p /app/state /app/certificates # 9093 = ServerAppIo (legacy / Exec API TLS) EXPOSE 9091 9092 9093 -CMD /ryzers/test_flower-superlink.sh +# Default CMD is the role launcher (reads env vars from config.yaml). +# To run the install-validation smoke test instead: +# ryzers run /ryzers/test_flower-superlink.sh +CMD /ryzers/run-superlink.sh diff --git a/packages/federated/flower-superlink/README.md b/packages/federated/flower-superlink/README.md index 6f6e7346..c03f4a24 100644 --- a/packages/federated/flower-superlink/README.md +++ b/packages/federated/flower-superlink/README.md @@ -15,19 +15,14 @@ ryzers build flower-base flower-superlink ## Run -`ryzers run` only accepts a single-token CMD override, so all SuperLink -flags are wrapped by an entry-point script (`/ryzers/run-superlink.sh`) -that reads its config from environment variables. The relevant env vars -are declared in `config.yaml` with shell-expansion defaults — override -them by exporting in your shell before `ryzers run`. - -The default `CMD` runs `test_flower-superlink.sh`, a one-shot install -check. For a real launch, use the entry-point script: +SuperLink flags are driven by environment variables declared in +`config.yaml` (with shell-expansion defaults). Export them in your +shell before `ryzers run` to override. ### Insecure (local testing) ```sh -ryzers run /ryzers/run-superlink.sh +ryzers run ``` ### With TLS (real deployment) @@ -49,7 +44,7 @@ ryzers run /ryzers/run-superlink.sh ```sh export FLOWER_INSECURE=0 - ryzers run /ryzers/run-superlink.sh + ryzers run ``` To override the cert paths inside the container, also export diff --git a/packages/federated/flower-supernode/Dockerfile b/packages/federated/flower-supernode/Dockerfile index befc1835..defbb1b1 100644 --- a/packages/federated/flower-supernode/Dockerfile +++ b/packages/federated/flower-supernode/Dockerfile @@ -14,4 +14,7 @@ RUN mkdir -p /app/certificates # ClientAppIo port — a paired superexec (clientapp plugin) connects here. EXPOSE 9094 -CMD /ryzers/test_flower-supernode.sh +# Default CMD is the role launcher (reads env vars from config.yaml). +# To run the install-validation smoke test instead: +# ryzers run /ryzers/test_flower-supernode.sh +CMD /ryzers/run-supernode.sh diff --git a/packages/federated/flower-supernode/README.md b/packages/federated/flower-supernode/README.md index 9489d6e4..cb9da14f 100644 --- a/packages/federated/flower-supernode/README.md +++ b/packages/federated/flower-supernode/README.md @@ -17,13 +17,9 @@ ryzers build flower-base flower-supernode ## Run -`ryzers run` only accepts a single-token CMD override, so the SuperNode -flags are wrapped by `/ryzers/run-supernode.sh`, which reads env vars -declared (with shell-expansion defaults) in `config.yaml`. Export them -in your shell before `ryzers run` to override. - -The default `CMD` runs `test_flower-supernode.sh`, an install check. -For a real launch: +SuperNode flags are driven by environment variables declared in +`config.yaml` (with shell-expansion defaults). Export them in your +shell before `ryzers run` to override. ### Insecure (testing) @@ -31,7 +27,7 @@ For a real launch: export SUPERLINK_IP=192.168.2.33 export FLOWER_PARTITION_ID=0 export FLOWER_NUM_PARTITIONS=2 -ryzers run /ryzers/run-supernode.sh +ryzers run ``` ### With TLS @@ -45,7 +41,7 @@ export SUPERLINK_IP=192.168.2.33 export FLOWER_INSECURE=0 export FLOWER_PARTITION_ID=0 export FLOWER_NUM_PARTITIONS=2 -ryzers run /ryzers/run-supernode.sh +ryzers run ``` ### Env-var reference From 9c420b6c4b425f0e639df44c33234e9521267b73 Mon Sep 17 00:00:00 2001 From: Ken O'Brien Date: Tue, 9 Jun 2026 13:51:59 +0100 Subject: [PATCH 04/20] fixes for demo --- packages/federated/flower-base/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/federated/flower-base/README.md b/packages/federated/flower-base/README.md index 41569f4d..e2bd0696 100644 --- a/packages/federated/flower-base/README.md +++ b/packages/federated/flower-base/README.md @@ -26,6 +26,13 @@ The default `CMD` runs `test_flower-base.sh`, which verifies the CLI binaries are installed and that `torch.cuda.is_available()` returns true under ROCm. +## Local single-machine smoke test + +For an end-to-end test on one host (SuperLink + ServerApp + two +SuperNode/ClientApp pairs) see +[`../local/README.md`](../local/README.md), which ships a docker-compose +file and a `run-local.sh` one-shot script. + ## References - [Flower documentation](https://flower.ai/docs/framework/) From bf1ed8fb0ee98d32673e5e7b527c6d3e81f7c3bb Mon Sep 17 00:00:00 2001 From: Ken O'Brien Date: Tue, 9 Jun 2026 13:58:12 +0100 Subject: [PATCH 05/20] fixes for demo --- packages/federated/local/README.md | 97 +++++++++++++++++++++ packages/federated/local/compose.yml | 117 ++++++++++++++++++++++++++ packages/federated/local/run-local.sh | 49 +++++++++++ 3 files changed, 263 insertions(+) create mode 100644 packages/federated/local/README.md create mode 100644 packages/federated/local/compose.yml create mode 100755 packages/federated/local/run-local.sh diff --git a/packages/federated/local/README.md b/packages/federated/local/README.md new file mode 100644 index 00000000..e32e9a6d --- /dev/null +++ b/packages/federated/local/README.md @@ -0,0 +1,97 @@ +# Flower Local Single-Machine Deployment + +Run the entire federation — SuperLink, ServerApp, and two SuperNode + +ClientApp pairs — on a single host for smoke-testing and development. +Mirrors the upstream multi-machine tutorial structure but collapses +everything onto one box on a shared docker bridge network. + +## Topology + +``` + ┌─────────────┐ + │ superlink │ + └──┬───┬───┬──┘ + 9091 ─────┘ │ │ + (ExecApi) │ │ 9092 (FleetApi) + │ │ + ┌─────┘ └─────┐ + │ │ + ┌─────────┐ ┌─────────┐ + │serverapp│ │supernode│ ×2 + └─────────┘ └────┬────┘ + │ 9094 + │ + ┌────────┐ + │clientapp│ ×2 + └────────┘ +``` + +## Prerequisites + +Build the three role images (they all layer on `flower-base`): + +```sh +ryzers build flower-base flower-superlink +ryzers build flower-base flower-supernode +ryzers build flower-base flower-superexec +``` + +The compose file references them by name (`flower-superlink:latest` +etc.) so the local docker daemon must have all three tagged. + +## Bring up the federation + +```sh +cd packages/federated/local +docker compose up # add -d to detach +``` + +Tear down with `docker compose down -v` (the `-v` clears the SuperLink +state volume too). + +## Submit a training run + +From the host (requires `pip install "flwr==1.26.1"` on the host): + +1. Add a federation entry pointing at the local SuperLink in + `../flower-superexec/app/pyproject.toml`: + + ```toml + [tool.flwr.federations.local] + address = "127.0.0.1:9093" + insecure = true + ``` + +2. Submit the run: + + ```sh + flwr run ../flower-superexec/app local + ``` + +You should see two clients pick up partitions, train one round each, +report metrics back to the ServerApp, and exit after three rounds (per +the example's `num-server-rounds = 3` default). + +## One-shot smoke test + +`run-local.sh` builds the images (if needed), brings the stack up, runs +the example, then tears down: + +```sh +bash run-local.sh +``` + +## Notes + +- All components run with `FLOWER_INSECURE=1` (no TLS). For a TLS + rehearsal, generate certs with `flower-superlink/gen-certs.sh` and + add `FLOWER_INSECURE=0` plus the cert volume mounts to the compose + file. +- Each `clientapp-*` service mounts `/dev/kfd` and `/dev/dri` so PyTorch + can use the ROCm GPU. If your host has no AMD GPU, comment those + blocks out — the example will fall back to CPU. +- `partition-id` is set per SuperNode (0 and 1). Add more + `supernode-N` + `clientapp-N` pairs and bump `num-partitions` + accordingly to scale the federation. + +Copyright(C) 2026 Advanced Micro Devices, Inc. All rights reserved. diff --git a/packages/federated/local/compose.yml b/packages/federated/local/compose.yml new file mode 100644 index 00000000..5092034b --- /dev/null +++ b/packages/federated/local/compose.yml @@ -0,0 +1,117 @@ +# Copyright(C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +# +# Local single-machine deployment of the full Flower federation: +# +# superlink ─┬─ serverapp (flower-superexec --plugin-type serverapp) +# ├─ supernode-0 ── clientapp-0 (flower-superexec --plugin-type clientapp) +# └─ supernode-1 ── clientapp-1 (flower-superexec --plugin-type clientapp) +# +# All components run insecure (no TLS) on a shared bridge network so they +# can address each other by service name. +# +# Prerequisites: build the three role images first (the chained builds +# also build flower-base implicitly): +# +# ryzers build flower-base flower-superlink +# ryzers build flower-base flower-supernode +# ryzers build flower-base flower-superexec +# +# Then from this directory: +# +# docker compose up --abort-on-container-exit +# +# Submit a run from the host (requires `pip install flwr` on the host): +# +# flwr run ../flower-superexec/app local-federation +# +# (Add a `[tool.flwr.federations.local-federation]` block in the app's +# pyproject.toml pointing at superlink address 127.0.0.1:9093.) + +services: + + superlink: + image: flower-superlink:latest + container_name: flower-superlink + networks: [flower] + ports: + - "9091:9091" # ExecApi (host-side flwr CLI) + - "9092:9092" # FleetApi + - "9093:9093" # ServerAppIo + environment: + FLOWER_INSECURE: "1" + FLOWER_ISOLATION: process + FLOWER_STATE_DB: /app/state/state.db + volumes: + - ./state:/app/state + + serverapp: + image: flower-superexec:latest + container_name: flower-serverapp + networks: [flower] + depends_on: [superlink] + environment: + FLOWER_PLUGIN_TYPE: serverapp + FLOWER_APPIO_ADDR: superlink:9091 + FLOWER_INSECURE: "1" + + supernode-0: + image: flower-supernode:latest + container_name: flower-supernode-0 + networks: [flower] + depends_on: [superlink] + environment: + SUPERLINK_IP: superlink + FLOWER_INSECURE: "1" + FLOWER_PARTITION_ID: "0" + FLOWER_NUM_PARTITIONS: "2" + FLOWER_CLIENTAPPIO: 0.0.0.0:9094 + FLOWER_ISOLATION: process + + clientapp-0: + image: flower-superexec:latest + container_name: flower-clientapp-0 + networks: [flower] + depends_on: [supernode-0] + # GPU passthrough so PyTorch ROCm is usable + devices: + - /dev/kfd + - /dev/dri + group_add: [video, render] + security_opt: [seccomp=unconfined] + environment: + FLOWER_PLUGIN_TYPE: clientapp + FLOWER_APPIO_ADDR: supernode-0:9094 + FLOWER_INSECURE: "1" + + supernode-1: + image: flower-supernode:latest + container_name: flower-supernode-1 + networks: [flower] + depends_on: [superlink] + environment: + SUPERLINK_IP: superlink + FLOWER_INSECURE: "1" + FLOWER_PARTITION_ID: "1" + FLOWER_NUM_PARTITIONS: "2" + FLOWER_CLIENTAPPIO: 0.0.0.0:9094 + FLOWER_ISOLATION: process + + clientapp-1: + image: flower-superexec:latest + container_name: flower-clientapp-1 + networks: [flower] + depends_on: [supernode-1] + devices: + - /dev/kfd + - /dev/dri + group_add: [video, render] + security_opt: [seccomp=unconfined] + environment: + FLOWER_PLUGIN_TYPE: clientapp + FLOWER_APPIO_ADDR: supernode-1:9094 + FLOWER_INSECURE: "1" + +networks: + flower: + driver: bridge diff --git a/packages/federated/local/run-local.sh b/packages/federated/local/run-local.sh new file mode 100755 index 00000000..26cd4a4e --- /dev/null +++ b/packages/federated/local/run-local.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +# +# One-shot local smoke test for the Flower federation: builds the three +# role images (if needed), brings the compose stack up, runs the +# quickstart-pytorch example end-to-end, then tears the stack down. +# +# Usage: bash run-local.sh + +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "${HERE}/../../.." && pwd)" +APP_DIR="${HERE}/../flower-superexec/app" + +cd "${REPO}" + +echo "==> Building role images (chained on flower-base)" +ryzers build flower-base flower-superlink +ryzers build flower-base flower-supernode +ryzers build flower-base flower-superexec + +cd "${HERE}" +mkdir -p state + +echo "==> Bringing the federation up" +docker compose up -d + +cleanup() { + echo "==> Tearing down" + docker compose down -v || true +} +trap cleanup EXIT + +echo "==> Waiting for SuperLink ExecApi (127.0.0.1:9091)" +for i in $(seq 1 30); do + (echo >/dev/tcp/127.0.0.1/9091) >/dev/null 2>&1 && break + sleep 1 +done + +echo "==> Submitting quickstart-pytorch run via flwr CLI" +# The host must have `flwr` installed: pip install "flwr==1.26.1" +# Federation name "local" must be configured in app/pyproject.toml — see +# packages/federated/local/README.md for the snippet. +flwr run "${APP_DIR}" local + +echo "==> Smoke test complete" From 448be56256f568f6c549d9289398cc2c0272461e Mon Sep 17 00:00:00 2001 From: Ken O'Brien Date: Tue, 9 Jun 2026 14:09:14 +0100 Subject: [PATCH 06/20] fixes for demo --- packages/federated/flower-base/README.md | 12 +- packages/federated/flower-superexec/README.md | 14 ++- .../flower-superexec/app/pyproject.toml | 10 ++ .../federated/flower-superexec/config.yaml | 20 ++- .../flower-superexec/run-superexec.sh | 27 ++-- packages/federated/flower-superlink/README.md | 21 ++-- .../federated/flower-superlink/config.yaml | 13 +- packages/federated/flower-supernode/README.md | 8 +- .../federated/flower-supernode/config.yaml | 18 ++- packages/federated/local/README.md | 97 --------------- packages/federated/local/compose.yml | 117 ------------------ packages/federated/local/run-local.sh | 49 -------- packages/federated/run-local.sh | 106 ++++++++++++++++ 13 files changed, 212 insertions(+), 300 deletions(-) delete mode 100644 packages/federated/local/README.md delete mode 100644 packages/federated/local/compose.yml delete mode 100755 packages/federated/local/run-local.sh create mode 100755 packages/federated/run-local.sh diff --git a/packages/federated/flower-base/README.md b/packages/federated/flower-base/README.md index e2bd0696..8a552a1b 100644 --- a/packages/federated/flower-base/README.md +++ b/packages/federated/flower-base/README.md @@ -29,9 +29,15 @@ under ROCm. ## Local single-machine smoke test For an end-to-end test on one host (SuperLink + ServerApp + two -SuperNode/ClientApp pairs) see -[`../local/README.md`](../local/README.md), which ships a docker-compose -file and a `run-local.sh` one-shot script. +SuperNode/ClientApp pairs) run [`../run-local.sh`](../run-local.sh). It +builds the three role Ryzers, brings everything up under `--network host` +on the loopback (with ClientAppIo ports offset by partition ID), submits +the quickstart-pytorch run, and tears down on exit. + +The same three role Ryzers (`flower-superlink`, `flower-supernode`, +`flower-superexec`) are also what you run in a distributed deployment — +just set `SUPERLINK_IP` to the server's IP on the client boxes. See each +role Ryzer's README for the multi-machine flow. ## References diff --git a/packages/federated/flower-superexec/README.md b/packages/federated/flower-superexec/README.md index 723d0492..ad46bf3a 100644 --- a/packages/federated/flower-superexec/README.md +++ b/packages/federated/flower-superexec/README.md @@ -9,6 +9,15 @@ PyTorch training, so it sits on top of the ROCm/PyTorch base image and inherits GPU access. It ships with the `quickstart-pytorch` example app preinstalled at `/app/`. +A third plugin type, `submit`, runs `flwr run /app local` once and +exits — the `local` federation in `/app/pyproject.toml` points at the +SuperLink ExecApi on `127.0.0.1:9093`. Used by +[`../run-local.sh`](../run-local.sh) to kick off the example run after +everything else is up. The container uses `--network host`, so the +ServerApp/ClientApp/submit roles all reach SuperLink and SuperNode via +loopback in single-machine deployments and via the host network +directly in multi-machine ones. + ## Build ```sh @@ -85,8 +94,9 @@ generated by `flower-superlink/gen-certs.sh` (see that Ryzer's README). | Variable | Default | Purpose | |----------|---------|---------| -| `FLOWER_PLUGIN_TYPE` | `serverapp` | `serverapp` or `clientapp` | -| `FLOWER_APPIO_ADDR` | `127.0.0.1:9091` | SuperLink :9091 (serverapp) or SuperNode :9094 (clientapp) | +| `FLOWER_PLUGIN_TYPE` | `serverapp` | `serverapp`, `clientapp`, or `submit` | +| `FLOWER_PARTITION_ID` | `0` | For `clientapp`, picks which SuperNode ClientAppIo port to attach to | +| `FLOWER_APPIO_ADDR` | derived | `127.0.0.1:9091` (serverapp) or `127.0.0.1:$((9094 + FLOWER_PARTITION_ID))` (clientapp); ignored for `submit` | | `FLOWER_INSECURE` | `1` | `1` = `--insecure`; `0` = TLS via `FLOWER_CA_CERT` | ## Bundled example diff --git a/packages/federated/flower-superexec/app/pyproject.toml b/packages/federated/flower-superexec/app/pyproject.toml index 37a21dd2..bbf2c5fa 100644 --- a/packages/federated/flower-superexec/app/pyproject.toml +++ b/packages/federated/flower-superexec/app/pyproject.toml @@ -31,3 +31,13 @@ fraction-evaluate = 0.5 local-epochs = 1 learning-rate = 0.1 batch-size = 32 + +# `local` federation: targets the SuperLink ExecApi on the host loopback +# (works under --network host). Used by `flwr run /app local`, which the +# `submit` plugin type in run-superexec.sh invokes. +[tool.flwr.federations.local] +address = "127.0.0.1:9093" +insecure = true + +# SuperLink default ports: 9091 = ServerAppIo (serverapp superexec), +# 9092 = Fleet (SuperNode), 9093 = ExecApi (flwr run submitter). diff --git a/packages/federated/flower-superexec/config.yaml b/packages/federated/flower-superexec/config.yaml index 96ae2ae3..367f257d 100644 --- a/packages/federated/flower-superexec/config.yaml +++ b/packages/federated/flower-superexec/config.yaml @@ -4,16 +4,28 @@ # gpu_support defaults to true — this is the only flower-* Ryzer that # actually runs training, so ROCm access matters here. +# Host networking so the superexec can reach its paired SuperLink (for +# serverapp role) or SuperNode (for clientapp role) over 127.0.0.1 on a +# single-host deployment, and bind/connect directly on the host network +# in a distributed deployment. +docker_extra_run_flags: "--network host --rm" + volume_mappings: - "$PWD/workspace/flower/superlink-certificates:/app/certificates:ro" -# Consumed by /ryzers/run-superexec.sh. The two important knobs are -# FLOWER_PLUGIN_TYPE (serverapp | clientapp) and FLOWER_APPIO_ADDR -# (the SuperLink :9091 for serverapp, the SuperNode :9094 for clientapp). +# Consumed by /ryzers/run-superexec.sh. +# +# Plugin types: +# serverapp — runs the ServerApp; talks to SuperLink ExecApi (default :9091) +# clientapp — runs a ClientApp; talks to its paired SuperNode (default :9094+PARTITION_ID) +# submit — one-shot `flwr run /app local`, then exits (no daemon) environment_variables: - "FLOWER_PLUGIN_TYPE=${FLOWER_PLUGIN_TYPE:-serverapp}" -- "FLOWER_APPIO_ADDR=${FLOWER_APPIO_ADDR:-127.0.0.1:9091}" +- "FLOWER_PARTITION_ID=${FLOWER_PARTITION_ID:-0}" - "FLOWER_INSECURE=${FLOWER_INSECURE:-1}" +# FLOWER_APPIO_ADDR is computed by run-superexec.sh from PLUGIN_TYPE + +# PARTITION_ID by default; export it explicitly to override. +- "FLOWER_APPIO_ADDR=${FLOWER_APPIO_ADDR:-}" # Strix Point (gfx1150) compatibility shim — uncomment if your iGPU # isn't directly recognised by the ROCm runtime. # - "HSA_OVERRIDE_GFX_VERSION=11.0.0" diff --git a/packages/federated/flower-superexec/run-superexec.sh b/packages/federated/flower-superexec/run-superexec.sh index 3fec5c4a..b4d19688 100644 --- a/packages/federated/flower-superexec/run-superexec.sh +++ b/packages/federated/flower-superexec/run-superexec.sh @@ -10,23 +10,34 @@ # ryzers run /ryzers/run-superexec.sh # # Required env vars: -# FLOWER_PLUGIN_TYPE — "serverapp" or "clientapp" (default serverapp) -# FLOWER_APPIO_ADDR — host:port of the paired SuperLink (ServerApp) -# or SuperNode (ClientApp). -# Defaults to 127.0.0.1:9091 (matches a local -# SuperLink). For ClientApp, set to your local -# SuperNode, e.g. 127.0.0.1:9094. +# FLOWER_PLUGIN_TYPE — "serverapp", "clientapp", or "submit" +# (default serverapp) +# FLOWER_PARTITION_ID — for clientapp, offsets the default +# SuperNode ClientAppIo port (9094 + ID). +# FLOWER_APPIO_ADDR — host:port of the paired SuperLink (ServerApp) +# or SuperNode (ClientApp). Default depends on +# plugin type. +# +# "submit" plugin type is a one-shot helper: it runs `flwr run /app local` +# against the local SuperLink (using the `local` federation pre-baked +# into /app/pyproject.toml) and then exits. set -e FLOWER_PLUGIN_TYPE="${FLOWER_PLUGIN_TYPE:-serverapp}" FLOWER_INSECURE="${FLOWER_INSECURE:-1}" +FLOWER_PARTITION_ID="${FLOWER_PARTITION_ID:-0}" + +if [ "${FLOWER_PLUGIN_TYPE}" = "submit" ]; then + echo "Submitting: flwr run /app local" + exec flwr run /app local +fi case "${FLOWER_PLUGIN_TYPE}" in serverapp) DEFAULT_ADDR="127.0.0.1:9091" ;; - clientapp) DEFAULT_ADDR="127.0.0.1:9094" ;; + clientapp) DEFAULT_ADDR="127.0.0.1:$((9094 + FLOWER_PARTITION_ID))" ;; *) - echo "FLOWER_PLUGIN_TYPE must be 'serverapp' or 'clientapp' (got: ${FLOWER_PLUGIN_TYPE})" >&2 + echo "FLOWER_PLUGIN_TYPE must be 'serverapp', 'clientapp', or 'submit' (got: ${FLOWER_PLUGIN_TYPE})" >&2 exit 2 ;; esac diff --git a/packages/federated/flower-superlink/README.md b/packages/federated/flower-superlink/README.md index c03f4a24..a003b00d 100644 --- a/packages/federated/flower-superlink/README.md +++ b/packages/federated/flower-superlink/README.md @@ -1,11 +1,18 @@ # Flower SuperLink Docker Setup -The SuperLink is the central coordinator in a Flower deployment. It accepts -connections from SuperNodes (Fleet API, port 9092), runs the control plane -(Exec API, port 9091), and persists run state to `/app/state`. +The SuperLink is the central coordinator in a Flower deployment. It +accepts connections from the ServerApp superexec (ServerAppIo, port +9091), SuperNodes (Fleet API, port 9092), and the `flwr run` submitter +(Exec API, port 9093). Run state is persisted to `/app/state`. This Ryzer runs on the **server machine** and is the first component you -start when bringing up a federation. +start when bringing up a federation. The container uses `--network host` +so all three ports bind directly on the host — local components reach +them via `127.0.0.1`, remote ones via the server's IP. + +For a single-machine smoke test that brings up SuperLink + ServerApp + +two SuperNode/ClientApp pairs and submits the example run, use +[`../run-local.sh`](../run-local.sh). ## Build @@ -62,9 +69,9 @@ ryzers run | Port | API | Used by | |------|-----|---------| -| 9091 | ExecApi | local `flower-superexec --plugin-type serverapp` | -| 9092 | FleetApi | remote SuperNodes | -| 9093 | ServerAppIo | (legacy / TLS control) | +| 9091 | ServerAppIo | local `flower-superexec --plugin-type serverapp` | +| 9092 | Fleet | remote SuperNodes | +| 9093 | Exec | `flwr run` submitter (and the `submit` plugin-type) | ## References diff --git a/packages/federated/flower-superlink/config.yaml b/packages/federated/flower-superlink/config.yaml index d7b31792..0fa3d296 100644 --- a/packages/federated/flower-superlink/config.yaml +++ b/packages/federated/flower-superlink/config.yaml @@ -6,17 +6,18 @@ gpu_support: false x11_display: false -port_mappings: -- "9091:9091" # ExecApi (control plane) -- "9092:9092" # FleetApi (supernodes connect here) -- "9093:9093" # ServerAppIo +# Use host networking so that local single-machine deployments can +# address every component over 127.0.0.1 and distributed deployments +# bind ports directly on the host (no -p hop needed). SuperLink will +# listen on host 9091/9092/9093. +docker_extra_run_flags: "--network host --rm" volume_mappings: - "$PWD/workspace/flower/state:/app/state" - "$PWD/workspace/flower/superlink-certificates:/app/certificates:ro" -# These are consumed by /ryzers/run-superlink.sh. Override by exporting -# the same variables in your shell before `ryzers run`. +# Consumed by /ryzers/run-superlink.sh. Override by exporting the same +# variables in your shell before `ryzers run`. environment_variables: - "FLOWER_INSECURE=${FLOWER_INSECURE:-1}" - "FLOWER_ISOLATION=${FLOWER_ISOLATION:-process}" diff --git a/packages/federated/flower-supernode/README.md b/packages/federated/flower-supernode/README.md index cb9da14f..2cfa2c7a 100644 --- a/packages/federated/flower-supernode/README.md +++ b/packages/federated/flower-supernode/README.md @@ -9,6 +9,12 @@ Pair this Ryzer with `flower-superexec` on every client host. The SuperNode does the federation plumbing; the superexec runs the actual PyTorch/ROCm training. +The container uses `--network host`, so `SUPERLINK_IP` is the only +network knob you usually need to set. The ClientAppIo socket defaults +to `0.0.0.0:$((9094 + FLOWER_PARTITION_ID))` so multiple SuperNodes can +share a host for local testing without colliding (see +[`../run-local.sh`](../run-local.sh)). + ## Build ```sh @@ -52,7 +58,7 @@ ryzers run | `FLOWER_INSECURE` | `1` | `1` = `--insecure`; `0` = TLS via `FLOWER_CA_CERT` | | `FLOWER_PARTITION_ID` | `0` | Unique partition for this node (0..N-1) | | `FLOWER_NUM_PARTITIONS` | `2` | Total clients across the federation | -| `FLOWER_CLIENTAPPIO` | `0.0.0.0:9094` | Local socket the paired ClientApp connects to | +| `FLOWER_CLIENTAPPIO` | `0.0.0.0:$((9094 + FLOWER_PARTITION_ID))` | Local socket the paired ClientApp connects to (auto-offset for co-located nodes) | | `FLOWER_ISOLATION` | `process` | Passed to `--isolation` | ## References diff --git a/packages/federated/flower-supernode/config.yaml b/packages/federated/flower-supernode/config.yaml index fab615e7..89a039b6 100644 --- a/packages/federated/flower-supernode/config.yaml +++ b/packages/federated/flower-supernode/config.yaml @@ -3,20 +3,26 @@ # The SuperNode itself does not run training; it just orchestrates the # paired ClientApp superexec. GPU support is left enabled (default) so -# that running on the same host as the superexec is straightforward, but -# you can flip it off if running them in separate containers. +# colocating with the superexec is straightforward. -port_mappings: -- "9094:9094" # ClientAppIo (local superexec connects here) +# Host networking so we can run multiple instances on one box (each on +# a different ClientAppIo port) for local testing, and so distributed +# deployments don't need explicit port mapping. +docker_extra_run_flags: "--network host --rm" volume_mappings: - "$PWD/workspace/flower/superlink-certificates:/app/certificates:ro" # Consumed by /ryzers/run-supernode.sh. Override by exporting the same -# variables in your shell before `ryzers run`. SUPERLINK_IP is the most -# important one — set it to the routable IP of your SuperLink host. +# variables in your shell before `ryzers run`. +# +# Local (single-machine) testing: defaults are correct — SUPERLINK_IP +# stays 127.0.0.1, FLOWER_CLIENTAPPIO auto-offsets by PARTITION_ID so +# multiple SuperNodes don't collide on port 9094. +# Distributed testing: export SUPERLINK_IP=. environment_variables: - "SUPERLINK_IP=${SUPERLINK_IP:-127.0.0.1}" - "FLOWER_INSECURE=${FLOWER_INSECURE:-1}" - "FLOWER_PARTITION_ID=${FLOWER_PARTITION_ID:-0}" - "FLOWER_NUM_PARTITIONS=${FLOWER_NUM_PARTITIONS:-2}" +- "FLOWER_CLIENTAPPIO=${FLOWER_CLIENTAPPIO:-0.0.0.0:$((9094 + ${FLOWER_PARTITION_ID:-0}))}" diff --git a/packages/federated/local/README.md b/packages/federated/local/README.md deleted file mode 100644 index e32e9a6d..00000000 --- a/packages/federated/local/README.md +++ /dev/null @@ -1,97 +0,0 @@ -# Flower Local Single-Machine Deployment - -Run the entire federation — SuperLink, ServerApp, and two SuperNode + -ClientApp pairs — on a single host for smoke-testing and development. -Mirrors the upstream multi-machine tutorial structure but collapses -everything onto one box on a shared docker bridge network. - -## Topology - -``` - ┌─────────────┐ - │ superlink │ - └──┬───┬───┬──┘ - 9091 ─────┘ │ │ - (ExecApi) │ │ 9092 (FleetApi) - │ │ - ┌─────┘ └─────┐ - │ │ - ┌─────────┐ ┌─────────┐ - │serverapp│ │supernode│ ×2 - └─────────┘ └────┬────┘ - │ 9094 - │ - ┌────────┐ - │clientapp│ ×2 - └────────┘ -``` - -## Prerequisites - -Build the three role images (they all layer on `flower-base`): - -```sh -ryzers build flower-base flower-superlink -ryzers build flower-base flower-supernode -ryzers build flower-base flower-superexec -``` - -The compose file references them by name (`flower-superlink:latest` -etc.) so the local docker daemon must have all three tagged. - -## Bring up the federation - -```sh -cd packages/federated/local -docker compose up # add -d to detach -``` - -Tear down with `docker compose down -v` (the `-v` clears the SuperLink -state volume too). - -## Submit a training run - -From the host (requires `pip install "flwr==1.26.1"` on the host): - -1. Add a federation entry pointing at the local SuperLink in - `../flower-superexec/app/pyproject.toml`: - - ```toml - [tool.flwr.federations.local] - address = "127.0.0.1:9093" - insecure = true - ``` - -2. Submit the run: - - ```sh - flwr run ../flower-superexec/app local - ``` - -You should see two clients pick up partitions, train one round each, -report metrics back to the ServerApp, and exit after three rounds (per -the example's `num-server-rounds = 3` default). - -## One-shot smoke test - -`run-local.sh` builds the images (if needed), brings the stack up, runs -the example, then tears down: - -```sh -bash run-local.sh -``` - -## Notes - -- All components run with `FLOWER_INSECURE=1` (no TLS). For a TLS - rehearsal, generate certs with `flower-superlink/gen-certs.sh` and - add `FLOWER_INSECURE=0` plus the cert volume mounts to the compose - file. -- Each `clientapp-*` service mounts `/dev/kfd` and `/dev/dri` so PyTorch - can use the ROCm GPU. If your host has no AMD GPU, comment those - blocks out — the example will fall back to CPU. -- `partition-id` is set per SuperNode (0 and 1). Add more - `supernode-N` + `clientapp-N` pairs and bump `num-partitions` - accordingly to scale the federation. - -Copyright(C) 2026 Advanced Micro Devices, Inc. All rights reserved. diff --git a/packages/federated/local/compose.yml b/packages/federated/local/compose.yml deleted file mode 100644 index 5092034b..00000000 --- a/packages/federated/local/compose.yml +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright(C) 2026 Advanced Micro Devices, Inc. All rights reserved. -# SPDX-License-Identifier: MIT -# -# Local single-machine deployment of the full Flower federation: -# -# superlink ─┬─ serverapp (flower-superexec --plugin-type serverapp) -# ├─ supernode-0 ── clientapp-0 (flower-superexec --plugin-type clientapp) -# └─ supernode-1 ── clientapp-1 (flower-superexec --plugin-type clientapp) -# -# All components run insecure (no TLS) on a shared bridge network so they -# can address each other by service name. -# -# Prerequisites: build the three role images first (the chained builds -# also build flower-base implicitly): -# -# ryzers build flower-base flower-superlink -# ryzers build flower-base flower-supernode -# ryzers build flower-base flower-superexec -# -# Then from this directory: -# -# docker compose up --abort-on-container-exit -# -# Submit a run from the host (requires `pip install flwr` on the host): -# -# flwr run ../flower-superexec/app local-federation -# -# (Add a `[tool.flwr.federations.local-federation]` block in the app's -# pyproject.toml pointing at superlink address 127.0.0.1:9093.) - -services: - - superlink: - image: flower-superlink:latest - container_name: flower-superlink - networks: [flower] - ports: - - "9091:9091" # ExecApi (host-side flwr CLI) - - "9092:9092" # FleetApi - - "9093:9093" # ServerAppIo - environment: - FLOWER_INSECURE: "1" - FLOWER_ISOLATION: process - FLOWER_STATE_DB: /app/state/state.db - volumes: - - ./state:/app/state - - serverapp: - image: flower-superexec:latest - container_name: flower-serverapp - networks: [flower] - depends_on: [superlink] - environment: - FLOWER_PLUGIN_TYPE: serverapp - FLOWER_APPIO_ADDR: superlink:9091 - FLOWER_INSECURE: "1" - - supernode-0: - image: flower-supernode:latest - container_name: flower-supernode-0 - networks: [flower] - depends_on: [superlink] - environment: - SUPERLINK_IP: superlink - FLOWER_INSECURE: "1" - FLOWER_PARTITION_ID: "0" - FLOWER_NUM_PARTITIONS: "2" - FLOWER_CLIENTAPPIO: 0.0.0.0:9094 - FLOWER_ISOLATION: process - - clientapp-0: - image: flower-superexec:latest - container_name: flower-clientapp-0 - networks: [flower] - depends_on: [supernode-0] - # GPU passthrough so PyTorch ROCm is usable - devices: - - /dev/kfd - - /dev/dri - group_add: [video, render] - security_opt: [seccomp=unconfined] - environment: - FLOWER_PLUGIN_TYPE: clientapp - FLOWER_APPIO_ADDR: supernode-0:9094 - FLOWER_INSECURE: "1" - - supernode-1: - image: flower-supernode:latest - container_name: flower-supernode-1 - networks: [flower] - depends_on: [superlink] - environment: - SUPERLINK_IP: superlink - FLOWER_INSECURE: "1" - FLOWER_PARTITION_ID: "1" - FLOWER_NUM_PARTITIONS: "2" - FLOWER_CLIENTAPPIO: 0.0.0.0:9094 - FLOWER_ISOLATION: process - - clientapp-1: - image: flower-superexec:latest - container_name: flower-clientapp-1 - networks: [flower] - depends_on: [supernode-1] - devices: - - /dev/kfd - - /dev/dri - group_add: [video, render] - security_opt: [seccomp=unconfined] - environment: - FLOWER_PLUGIN_TYPE: clientapp - FLOWER_APPIO_ADDR: supernode-1:9094 - FLOWER_INSECURE: "1" - -networks: - flower: - driver: bridge diff --git a/packages/federated/local/run-local.sh b/packages/federated/local/run-local.sh deleted file mode 100755 index 26cd4a4e..00000000 --- a/packages/federated/local/run-local.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. -# SPDX-License-Identifier: MIT -# -# One-shot local smoke test for the Flower federation: builds the three -# role images (if needed), brings the compose stack up, runs the -# quickstart-pytorch example end-to-end, then tears the stack down. -# -# Usage: bash run-local.sh - -set -euo pipefail - -HERE="$(cd "$(dirname "$0")" && pwd)" -REPO="$(cd "${HERE}/../../.." && pwd)" -APP_DIR="${HERE}/../flower-superexec/app" - -cd "${REPO}" - -echo "==> Building role images (chained on flower-base)" -ryzers build flower-base flower-superlink -ryzers build flower-base flower-supernode -ryzers build flower-base flower-superexec - -cd "${HERE}" -mkdir -p state - -echo "==> Bringing the federation up" -docker compose up -d - -cleanup() { - echo "==> Tearing down" - docker compose down -v || true -} -trap cleanup EXIT - -echo "==> Waiting for SuperLink ExecApi (127.0.0.1:9091)" -for i in $(seq 1 30); do - (echo >/dev/tcp/127.0.0.1/9091) >/dev/null 2>&1 && break - sleep 1 -done - -echo "==> Submitting quickstart-pytorch run via flwr CLI" -# The host must have `flwr` installed: pip install "flwr==1.26.1" -# Federation name "local" must be configured in app/pyproject.toml — see -# packages/federated/local/README.md for the snippet. -flwr run "${APP_DIR}" local - -echo "==> Smoke test complete" diff --git a/packages/federated/run-local.sh b/packages/federated/run-local.sh new file mode 100755 index 00000000..d535909c --- /dev/null +++ b/packages/federated/run-local.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +# +# One-shot single-machine smoke test for the Flower federated-learning +# Ryzers. Brings up SuperLink + ServerApp superexec + 2×SuperNode + +# 2×ClientApp superexec on the host network (127.0.0.1 + offset ports), +# submits the quickstart-pytorch run, and tears everything down. +# +# Multi-machine deployment uses the same three Ryzers — just point +# SUPERLINK_IP at the server and run the appropriate role on each box. +# See each Ryzer's README for the distributed flow. +# +# Usage: +# cd packages/federated +# ./run-local.sh # default 2 partitions +# FLOWER_NUM_PARTITIONS=4 ./run-local.sh + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +NUM_PARTITIONS="${FLOWER_NUM_PARTITIONS:-2}" + +cd "${REPO_ROOT}" + +echo "== Building Ryzers ==" +ryzers build flower-base flower-superlink +ryzers build flower-base flower-supernode +ryzers build flower-base flower-superexec + +echo "== Cleaning up stale containers ==" +for name in flower-superlink flower-supernode flower-superexec; do + docker ps -aq --filter "ancestor=${name}" | xargs -r docker rm -f >/dev/null 2>&1 || true +done + +# All background PIDs and per-instance log files so we can clean up on exit. +BG_PIDS=() +LOG_DIR="$(mktemp -d -t flower-local-XXXXXX)" +echo "Logs: ${LOG_DIR}" + +cleanup() { + echo "== Tearing down ==" + for pid in "${BG_PIDS[@]}"; do + kill "${pid}" 2>/dev/null || true + done + # --rm in each config.yaml means containers self-clean on exit, + # but kill any stragglers just in case. + for name in flower-superlink flower-supernode flower-superexec; do + docker ps -q --filter "ancestor=${name}" | xargs -r docker kill >/dev/null 2>&1 || true + done +} +trap cleanup EXIT INT TERM + +wait_for_port() { + local port="$1" + local timeout="${2:-30}" + for _ in $(seq 1 "${timeout}"); do + if (echo > /dev/tcp/127.0.0.1/"${port}") 2>/dev/null; then + return 0 + fi + sleep 1 + done + echo "Timed out waiting for 127.0.0.1:${port}" >&2 + return 1 +} + +echo "== Starting SuperLink ==" +ryzers run --name flower-superlink >"${LOG_DIR}/superlink.log" 2>&1 & +BG_PIDS+=($!) +wait_for_port 9092 30 # FleetApi — last of SuperLink's ports to bind + +echo "== Starting ServerApp superexec ==" +FLOWER_PLUGIN_TYPE=serverapp \ +FLOWER_APPIO_ADDR=127.0.0.1:9091 \ + ryzers run --name flower-superexec >"${LOG_DIR}/serverapp.log" 2>&1 & +BG_PIDS+=($!) + +for i in $(seq 0 $((NUM_PARTITIONS - 1))); do + echo "== Starting SuperNode partition ${i} ==" + FLOWER_PARTITION_ID="${i}" \ + FLOWER_NUM_PARTITIONS="${NUM_PARTITIONS}" \ + ryzers run --name flower-supernode >"${LOG_DIR}/supernode-${i}.log" 2>&1 & + BG_PIDS+=($!) + + echo "== Starting ClientApp superexec partition ${i} ==" + FLOWER_PLUGIN_TYPE=clientapp \ + FLOWER_PARTITION_ID="${i}" \ + FLOWER_APPIO_ADDR="127.0.0.1:$((9094 + i))" \ + ryzers run --name flower-superexec >"${LOG_DIR}/clientapp-${i}.log" 2>&1 & + BG_PIDS+=($!) +done + +echo "== Waiting for SuperLink ExecApi (9093) ==" +wait_for_port 9093 30 + +# Give the supernodes a beat to register with the SuperLink before +# submitting — otherwise `flwr run` can race the fleet handshake. +sleep 5 + +echo "== Submitting quickstart-pytorch run ==" +FLOWER_PLUGIN_TYPE=submit ryzers run --name flower-superexec | tee "${LOG_DIR}/submit.log" + +echo +echo "== Done. Per-component logs: ${LOG_DIR} ==" From ac9019d876186d3314561fbd534e1eebc551b0e6 Mon Sep 17 00:00:00 2001 From: Ken O'Brien Date: Thu, 11 Jun 2026 11:28:09 +0100 Subject: [PATCH 07/20] demo fixes --- packages/federated/flower-base/README.md | 8 +- packages/federated/flower-base/config.yaml | 6 +- packages/federated/flower-superexec/README.md | 15 +- packages/federated/flower-superlink/README.md | 2 +- packages/federated/flower-supernode/README.md | 2 +- packages/federated/run-local.sh | 180 ++++++++++++------ 6 files changed, 142 insertions(+), 71 deletions(-) diff --git a/packages/federated/flower-base/README.md b/packages/federated/flower-base/README.md index 8a552a1b..913afa99 100644 --- a/packages/federated/flower-base/README.md +++ b/packages/federated/flower-base/README.md @@ -10,9 +10,11 @@ You usually don't build or run this Ryzer on its own — chain it with one of the role Ryzers: ```sh -ryzers build flower-base flower-superlink # server box -ryzers build flower-base flower-supernode # client box -ryzers build flower-base flower-superexec # serverapp or clientapp runner +# --name sets the final image tag (it defaults to "ryzerdocker", not the +# last package name), so each role gets its own image + run-script. +ryzers build --name flower-superlink flower-base flower-superlink # server box +ryzers build --name flower-supernode flower-base flower-supernode # client box +ryzers build --name flower-superexec flower-base flower-superexec # serverapp or clientapp runner ``` ## Build & Run (standalone smoke test) diff --git a/packages/federated/flower-base/config.yaml b/packages/federated/flower-base/config.yaml index c3b2b261..9b70209a 100644 --- a/packages/federated/flower-base/config.yaml +++ b/packages/federated/flower-base/config.yaml @@ -4,9 +4,9 @@ # flower-base is a composable layer. It provides the flwr CLI on top of the # default ROCm/PyTorch base image. Most users will not run it directly; # instead chain it with one of: -# ryzers build flower-base flower-superlink -# ryzers build flower-base flower-supernode -# ryzers build flower-base flower-superexec +# ryzers build --name flower-superlink flower-base flower-superlink +# ryzers build --name flower-supernode flower-base flower-supernode +# ryzers build --name flower-superexec flower-base flower-superexec build_arguments: - "FLWR_VERSION=1.26.1" diff --git a/packages/federated/flower-superexec/README.md b/packages/federated/flower-superexec/README.md index ad46bf3a..8f9c8531 100644 --- a/packages/federated/flower-superexec/README.md +++ b/packages/federated/flower-superexec/README.md @@ -21,7 +21,7 @@ directly in multi-machine ones. ## Build ```sh -ryzers build flower-base flower-superexec +ryzers build --name flower-superexec flower-base flower-superexec ``` ## Run @@ -51,10 +51,13 @@ ryzers run **On the server machine** (SuperLink + ServerApp superexec): ```sh -ryzers build flower-base flower-superlink -ryzers build flower-base flower-superexec +# --name sets the final image tag; without it both builds would clobber +# the default "ryzerdocker" image and `ryzers run --name ` would +# not find its generated run-script. +ryzers build --name flower-superlink flower-base flower-superlink +ryzers build --name flower-superexec flower-base flower-superexec -# Terminal 1 — SuperLink (last built image) +# Terminal 1 — SuperLink ryzers run --name flower-superlink # Terminal 2 — ServerApp superexec @@ -66,8 +69,8 @@ ryzers run --name flower-superexec **On each client machine** (SuperNode + ClientApp superexec): ```sh -ryzers build flower-base flower-supernode -ryzers build flower-base flower-superexec +ryzers build --name flower-supernode flower-base flower-supernode +ryzers build --name flower-superexec flower-base flower-superexec # Terminal 1 — SuperNode (connects to remote SuperLink) export SUPERLINK_IP=192.168.2.33 diff --git a/packages/federated/flower-superlink/README.md b/packages/federated/flower-superlink/README.md index a003b00d..93436444 100644 --- a/packages/federated/flower-superlink/README.md +++ b/packages/federated/flower-superlink/README.md @@ -17,7 +17,7 @@ two SuperNode/ClientApp pairs and submits the example run, use ## Build ```sh -ryzers build flower-base flower-superlink +ryzers build --name flower-superlink flower-base flower-superlink ``` ## Run diff --git a/packages/federated/flower-supernode/README.md b/packages/federated/flower-supernode/README.md index 2cfa2c7a..b28cac16 100644 --- a/packages/federated/flower-supernode/README.md +++ b/packages/federated/flower-supernode/README.md @@ -18,7 +18,7 @@ share a host for local testing without colliding (see ## Build ```sh -ryzers build flower-base flower-supernode +ryzers build --name flower-supernode flower-base flower-supernode ``` ## Run diff --git a/packages/federated/run-local.sh b/packages/federated/run-local.sh index d535909c..5b791471 100755 --- a/packages/federated/run-local.sh +++ b/packages/federated/run-local.sh @@ -3,104 +3,170 @@ # SPDX-License-Identifier: MIT # # One-shot single-machine smoke test for the Flower federated-learning -# Ryzers. Brings up SuperLink + ServerApp superexec + 2×SuperNode + -# 2×ClientApp superexec on the host network (127.0.0.1 + offset ports), -# submits the quickstart-pytorch run, and tears everything down. +# Ryzers. Each long-running component (SuperLink, ServerApp superexec, +# and one SuperNode + ClientApp superexec per partition) is launched in +# its own terminal window so you can watch them individually. Once the +# federation is up, the quickstart-pytorch run is submitted from this +# terminal. # -# Multi-machine deployment uses the same three Ryzers — just point +# Multi-machine deployment uses the same three Ryzers — point # SUPERLINK_IP at the server and run the appropriate role on each box. # See each Ryzer's README for the distributed flow. # # Usage: # cd packages/federated -# ./run-local.sh # default 2 partitions -# FLOWER_NUM_PARTITIONS=4 ./run-local.sh +# ./run-local.sh # 2 partitions, auto-pick terminal +# FLOWER_NUM_PARTITIONS=4 ./run-local.sh # 4 partitions +# RYZERS_TERMINAL=gnome-terminal ./run-local.sh # force a terminal emulator set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" - NUM_PARTITIONS="${FLOWER_NUM_PARTITIONS:-2}" cd "${REPO_ROOT}" -echo "== Building Ryzers ==" -ryzers build flower-base flower-superlink -ryzers build flower-base flower-supernode -ryzers build flower-base flower-superexec - -echo "== Cleaning up stale containers ==" -for name in flower-superlink flower-supernode flower-superexec; do - docker ps -aq --filter "ancestor=${name}" | xargs -r docker rm -f >/dev/null 2>&1 || true -done +# --------------------------------------------------------------------------- +# Pick a terminal emulator. xterm is preferred because it honours $DISPLAY +# reliably over SSH X-forwarding; the others are tried as fallbacks. +# --------------------------------------------------------------------------- +if [ -z "${DISPLAY}" ]; then + echo "ERROR: \$DISPLAY is not set — can't open terminal windows." >&2 + echo " Run over X-forwarding (ssh -X) or set DISPLAY." >&2 + exit 1 +fi + +TERM_EMU="${RYZERS_TERMINAL:-}" +if [ -z "${TERM_EMU}" ]; then + for t in xterm gnome-terminal xfce4-terminal konsole x-terminal-emulator; do + if command -v "${t}" >/dev/null 2>&1; then + TERM_EMU="${t}" + break + fi + done +fi +if [ -z "${TERM_EMU}" ] || ! command -v "${TERM_EMU}" >/dev/null 2>&1; then + echo "ERROR: no terminal emulator found (tried xterm, gnome-terminal, ...)." >&2 + echo " Install one or set RYZERS_TERMINAL." >&2 + exit 1 +fi +echo "Using terminal emulator: ${TERM_EMU}" -# All background PIDs and per-instance log files so we can clean up on exit. -BG_PIDS=() LOG_DIR="$(mktemp -d -t flower-local-XXXXXX)" -echo "Logs: ${LOG_DIR}" - -cleanup() { - echo "== Tearing down ==" - for pid in "${BG_PIDS[@]}"; do - kill "${pid}" 2>/dev/null || true - done - # --rm in each config.yaml means containers self-clean on exit, - # but kill any stragglers just in case. - for name in flower-superlink flower-supernode flower-superexec; do - docker ps -q --filter "ancestor=${name}" | xargs -r docker kill >/dev/null 2>&1 || true - done +echo "Per-component launch scripts: ${LOG_DIR}" + +# spawn TITLE "shell-command" +# Writes a small wrapper script (so we avoid cross-emulator quoting +# hell), then opens it in a new terminal window that stays open after +# the component exits. +spawn() { + local title="$1" + local body="$2" + local script="${LOG_DIR}/${title}.sh" + cat > "${script}" < /dev/tcp/127.0.0.1/"${port}") 2>/dev/null; then return 0 fi + if ((i % 10 == 0)); then + echo " ... still waiting for 127.0.0.1:${port} (${i}/${timeout}s)" + fi sleep 1 done - echo "Timed out waiting for 127.0.0.1:${port}" >&2 return 1 } +# --------------------------------------------------------------------------- +# Build the three role images. NB: `ryzers build` names the *final* image +# after --name (default "ryzerdocker"), NOT after the last package — so +# --name is required here, otherwise all three would clobber the same +# "ryzerdocker" image and `ryzers run --name ` would not find its +# generated run-script. +# --------------------------------------------------------------------------- +echo "== Building Ryzers ==" +ryzers build --name flower-superlink flower-base flower-superlink +ryzers build --name flower-supernode flower-base flower-supernode +ryzers build --name flower-superexec flower-base flower-superexec + +echo "== Cleaning up stale containers ==" +for name in flower-superlink flower-supernode flower-superexec; do + docker ps -aq --filter "ancestor=${name}" | xargs -r docker rm -f >/dev/null 2>&1 || true +done + echo "== Starting SuperLink ==" -ryzers run --name flower-superlink >"${LOG_DIR}/superlink.log" 2>&1 & -BG_PIDS+=($!) -wait_for_port 9092 30 # FleetApi — last of SuperLink's ports to bind +spawn "flower-superlink" "ryzers run --name flower-superlink" +if ! wait_for_port 9092 120; then + echo "ERROR: SuperLink Fleet API (9092) never came up." >&2 + echo " Check the 'flower-superlink' terminal window for the error." >&2 + exit 1 +fi +echo " SuperLink up (9092 bound)." echo "== Starting ServerApp superexec ==" -FLOWER_PLUGIN_TYPE=serverapp \ -FLOWER_APPIO_ADDR=127.0.0.1:9091 \ - ryzers run --name flower-superexec >"${LOG_DIR}/serverapp.log" 2>&1 & -BG_PIDS+=($!) +spawn "flower-serverapp" \ + "FLOWER_PLUGIN_TYPE=serverapp FLOWER_APPIO_ADDR=127.0.0.1:9091 ryzers run --name flower-superexec" -for i in $(seq 0 $((NUM_PARTITIONS - 1))); do +for ((i = 0; i < NUM_PARTITIONS; i++)); do echo "== Starting SuperNode partition ${i} ==" - FLOWER_PARTITION_ID="${i}" \ - FLOWER_NUM_PARTITIONS="${NUM_PARTITIONS}" \ - ryzers run --name flower-supernode >"${LOG_DIR}/supernode-${i}.log" 2>&1 & - BG_PIDS+=($!) + spawn "flower-supernode-${i}" \ + "FLOWER_PARTITION_ID=${i} FLOWER_NUM_PARTITIONS=${NUM_PARTITIONS} ryzers run --name flower-supernode" echo "== Starting ClientApp superexec partition ${i} ==" - FLOWER_PLUGIN_TYPE=clientapp \ - FLOWER_PARTITION_ID="${i}" \ - FLOWER_APPIO_ADDR="127.0.0.1:$((9094 + i))" \ - ryzers run --name flower-superexec >"${LOG_DIR}/clientapp-${i}.log" 2>&1 & - BG_PIDS+=($!) + spawn "flower-clientapp-${i}" \ + "FLOWER_PLUGIN_TYPE=clientapp FLOWER_PARTITION_ID=${i} FLOWER_APPIO_ADDR=127.0.0.1:$((9094 + i)) ryzers run --name flower-superexec" done echo "== Waiting for SuperLink ExecApi (9093) ==" -wait_for_port 9093 30 +if ! wait_for_port 9093 120; then + echo "ERROR: SuperLink ExecApi (9093) never came up." >&2 + exit 1 +fi -# Give the supernodes a beat to register with the SuperLink before -# submitting — otherwise `flwr run` can race the fleet handshake. -sleep 5 +# Give the SuperNodes a beat to finish the Fleet handshake before +# submitting — otherwise `flwr run` can race node registration. +echo " Waiting for SuperNodes to register..." +sleep 8 echo "== Submitting quickstart-pytorch run ==" -FLOWER_PLUGIN_TYPE=submit ryzers run --name flower-superexec | tee "${LOG_DIR}/submit.log" +FLOWER_PLUGIN_TYPE=submit ryzers run --name flower-superexec echo -echo "== Done. Per-component logs: ${LOG_DIR} ==" +echo "== Run submitted. Component windows are still open. ==" +echo "To stop everything:" +echo " for n in flower-superlink flower-supernode flower-superexec; do \\" +echo " docker ps -q --filter ancestor=\$n | xargs -r docker kill; done" From 9214ebf515545fd168e8ae191b222fdc66dadfe1 Mon Sep 17 00:00:00 2001 From: Ken O'Brien Date: Thu, 11 Jun 2026 11:30:41 +0100 Subject: [PATCH 08/20] demo fixes --- packages/federated/flower-superexec/config.yaml | 12 ++++++------ packages/federated/flower-superlink/config.yaml | 10 +++++----- packages/federated/flower-supernode/config.yaml | 9 +++++---- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/packages/federated/flower-superexec/config.yaml b/packages/federated/flower-superexec/config.yaml index 367f257d..fe93b599 100644 --- a/packages/federated/flower-superexec/config.yaml +++ b/packages/federated/flower-superexec/config.yaml @@ -4,11 +4,11 @@ # gpu_support defaults to true — this is the only flower-* Ryzer that # actually runs training, so ROCm access matters here. -# Host networking so the superexec can reach its paired SuperLink (for -# serverapp role) or SuperNode (for clientapp role) over 127.0.0.1 on a -# single-host deployment, and bind/connect directly on the host network -# in a distributed deployment. -docker_extra_run_flags: "--network host --rm" +# Host networking (and --rm) already come from RYZERS_DEFAULT_RUN_FLAGS, +# so we don't repeat them here — Docker rejects "--network host" twice. +# Host networking lets the superexec reach its paired SuperLink (serverapp +# role) or SuperNode (clientapp role) over 127.0.0.1 on a single host, and +# bind/connect directly on the host network in a distributed deployment. volume_mappings: - "$PWD/workspace/flower/superlink-certificates:/app/certificates:ro" @@ -16,7 +16,7 @@ volume_mappings: # Consumed by /ryzers/run-superexec.sh. # # Plugin types: -# serverapp — runs the ServerApp; talks to SuperLink ExecApi (default :9091) +# serverapp — runs the ServerApp; talks to SuperLink ServerAppIo (default :9091) # clientapp — runs a ClientApp; talks to its paired SuperNode (default :9094+PARTITION_ID) # submit — one-shot `flwr run /app local`, then exits (no daemon) environment_variables: diff --git a/packages/federated/flower-superlink/config.yaml b/packages/federated/flower-superlink/config.yaml index 0fa3d296..e6371512 100644 --- a/packages/federated/flower-superlink/config.yaml +++ b/packages/federated/flower-superlink/config.yaml @@ -6,11 +6,11 @@ gpu_support: false x11_display: false -# Use host networking so that local single-machine deployments can -# address every component over 127.0.0.1 and distributed deployments -# bind ports directly on the host (no -p hop needed). SuperLink will -# listen on host 9091/9092/9093. -docker_extra_run_flags: "--network host --rm" +# Host networking (and --rm) already come from RYZERS_DEFAULT_RUN_FLAGS, +# so we don't repeat them here — Docker rejects "--network host" twice. +# Host networking lets local deployments address every component over +# 127.0.0.1 and distributed ones bind ports directly on the host (no -p +# hop). SuperLink listens on host 9091/9092/9093. volume_mappings: - "$PWD/workspace/flower/state:/app/state" diff --git a/packages/federated/flower-supernode/config.yaml b/packages/federated/flower-supernode/config.yaml index 89a039b6..2613cf92 100644 --- a/packages/federated/flower-supernode/config.yaml +++ b/packages/federated/flower-supernode/config.yaml @@ -5,10 +5,11 @@ # paired ClientApp superexec. GPU support is left enabled (default) so # colocating with the superexec is straightforward. -# Host networking so we can run multiple instances on one box (each on -# a different ClientAppIo port) for local testing, and so distributed -# deployments don't need explicit port mapping. -docker_extra_run_flags: "--network host --rm" +# Host networking (and --rm) already come from RYZERS_DEFAULT_RUN_FLAGS, +# so we don't repeat them here — Docker rejects "--network host" twice. +# Host networking lets us run multiple instances on one box (each on a +# different ClientAppIo port) for local testing, and lets distributed +# deployments skip explicit port mapping. volume_mappings: - "$PWD/workspace/flower/superlink-certificates:/app/certificates:ro" From 244c5599beaedfb716e69d35f0eb22bd8169384f Mon Sep 17 00:00:00 2001 From: Ken O'Brien Date: Thu, 11 Jun 2026 11:33:55 +0100 Subject: [PATCH 09/20] demo fixes --- packages/federated/flower-superexec/app/pyproject.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/federated/flower-superexec/app/pyproject.toml b/packages/federated/flower-superexec/app/pyproject.toml index bbf2c5fa..8d4b0abb 100644 --- a/packages/federated/flower-superexec/app/pyproject.toml +++ b/packages/federated/flower-superexec/app/pyproject.toml @@ -32,6 +32,12 @@ local-epochs = 1 learning-rate = 0.1 batch-size = 32 +# Flower requires the [tool.flwr.federations] table with a `default` +# key; omitting it makes flwr fall back to its legacy-config migration +# path, which fails with "failed to migrate legacy toml configuration". +[tool.flwr.federations] +default = "local" + # `local` federation: targets the SuperLink ExecApi on the host loopback # (works under --network host). Used by `flwr run /app local`, which the # `submit` plugin type in run-superexec.sh invokes. From f275f4384b380fffed368a858eeebddc98ba2dcf Mon Sep 17 00:00:00 2001 From: Ken O'Brien Date: Thu, 11 Jun 2026 12:13:50 +0100 Subject: [PATCH 10/20] fixes for demo --- packages/federated/run-local.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/federated/run-local.sh b/packages/federated/run-local.sh index 5b791471..875906cd 100755 --- a/packages/federated/run-local.sh +++ b/packages/federated/run-local.sh @@ -124,10 +124,21 @@ ryzers build --name flower-supernode flower-base flower-supernode ryzers build --name flower-superexec flower-base flower-superexec echo "== Cleaning up stale containers ==" -for name in flower-superlink flower-supernode flower-superexec; do +# Include the default "ryzerdocker" tag: runs from before per-role --name +# builds created containers under that image and, with --network host, +# they hold the 909x ports and break the new components. +for name in flower-superlink flower-supernode flower-superexec ryzerdocker; do docker ps -aq --filter "ancestor=${name}" | xargs -r docker rm -f >/dev/null 2>&1 || true done +# Start from a clean SuperLink state. A partial run left over from a +# previous (e.g. failed) submit can make the SuperLink raise +# KeyError('config') -> "Exception calling application: 'config'" when a +# SuperNode connects. This matches the volume mount in +# flower-superlink/config.yaml ($PWD/workspace/flower/state). +echo "== Resetting SuperLink state ==" +rm -rf "${REPO_ROOT}/workspace/flower/state"/* 2>/dev/null || true + echo "== Starting SuperLink ==" spawn "flower-superlink" "ryzers run --name flower-superlink" if ! wait_for_port 9092 120; then From c9f76e356cb5d128713fd58c76d9a8f9bd3ead3f Mon Sep 17 00:00:00 2001 From: Ken O'Brien Date: Thu, 11 Jun 2026 13:27:14 +0100 Subject: [PATCH 11/20] demo fixes --- packages/federated/flower-superlink/config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/federated/flower-superlink/config.yaml b/packages/federated/flower-superlink/config.yaml index e6371512..7583d837 100644 --- a/packages/federated/flower-superlink/config.yaml +++ b/packages/federated/flower-superlink/config.yaml @@ -22,3 +22,7 @@ environment_variables: - "FLOWER_INSECURE=${FLOWER_INSECURE:-1}" - "FLOWER_ISOLATION=${FLOWER_ISOLATION:-process}" - "FLOWER_STATE_DB=${FLOWER_STATE_DB:-/app/state/state.db}" +# Set FLWR_LOG_LEVEL=DEBUG in your shell before `ryzers run` to make the +# SuperLink print full tracebacks (e.g. the server-side KeyError behind a +# gRPC "Exception calling application: ..." seen by SuperNode/superexec). +- "FLWR_LOG_LEVEL=${FLWR_LOG_LEVEL:-INFO}" From 9756174034674be1d7933a20864519f823bae071 Mon Sep 17 00:00:00 2001 From: Ken O'Brien Date: Thu, 11 Jun 2026 13:30:42 +0100 Subject: [PATCH 12/20] demo fixes --- .../federated/flower-superexec/config.yaml | 4 +++ .../federated/flower-superlink/config.yaml | 5 ++++ .../federated/flower-supernode/config.yaml | 4 +++ packages/federated/run-local.sh | 25 +++++++++++++------ 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/packages/federated/flower-superexec/config.yaml b/packages/federated/flower-superexec/config.yaml index fe93b599..61823583 100644 --- a/packages/federated/flower-superexec/config.yaml +++ b/packages/federated/flower-superexec/config.yaml @@ -9,6 +9,10 @@ # Host networking lets the superexec reach its paired SuperLink (serverapp # role) or SuperNode (clientapp role) over 127.0.0.1 on a single host, and # bind/connect directly on the host network in a distributed deployment. +# +# The label lets run-local.sh reliably clean up flower containers across +# image rebuilds. +docker_extra_run_flags: "--label ryzers-flower-local=1" volume_mappings: - "$PWD/workspace/flower/superlink-certificates:/app/certificates:ro" diff --git a/packages/federated/flower-superlink/config.yaml b/packages/federated/flower-superlink/config.yaml index 7583d837..d929cc57 100644 --- a/packages/federated/flower-superlink/config.yaml +++ b/packages/federated/flower-superlink/config.yaml @@ -11,6 +11,11 @@ x11_display: false # Host networking lets local deployments address every component over # 127.0.0.1 and distributed ones bind ports directly on the host (no -p # hop). SuperLink listens on host 9091/9092/9093. +# +# The label lets run-local.sh reliably clean up flower containers across +# image rebuilds (an `--filter ancestor=` cleanup misses containers +# whose image was orphaned by a rebuild). +docker_extra_run_flags: "--label ryzers-flower-local=1" volume_mappings: - "$PWD/workspace/flower/state:/app/state" diff --git a/packages/federated/flower-supernode/config.yaml b/packages/federated/flower-supernode/config.yaml index 2613cf92..a99eb956 100644 --- a/packages/federated/flower-supernode/config.yaml +++ b/packages/federated/flower-supernode/config.yaml @@ -10,6 +10,10 @@ # Host networking lets us run multiple instances on one box (each on a # different ClientAppIo port) for local testing, and lets distributed # deployments skip explicit port mapping. +# +# The label lets run-local.sh reliably clean up flower containers across +# image rebuilds. +docker_extra_run_flags: "--label ryzers-flower-local=1" volume_mappings: - "$PWD/workspace/flower/superlink-certificates:/app/certificates:ro" diff --git a/packages/federated/run-local.sh b/packages/federated/run-local.sh index 875906cd..95b74a06 100755 --- a/packages/federated/run-local.sh +++ b/packages/federated/run-local.sh @@ -111,6 +111,23 @@ wait_for_port() { return 1 } +# --------------------------------------------------------------------------- +# Clean up stale containers FIRST, before building. A rebuild moves the +# flower-* tags onto new image IDs and orphans the old ones, so an +# `--filter ancestor=` cleanup run *after* the build would no longer +# match containers from the previous run (they still reference the old +# image ID) and they'd keep holding the 909x ports. +# +# Primary match is the ryzers-flower-local label (set via each role's +# docker_extra_run_flags) which survives rebuilds. The ancestor/ryzerdocker +# pass is a fallback for containers created before the label existed. +# --------------------------------------------------------------------------- +echo "== Cleaning up stale containers ==" +docker ps -aq --filter "label=ryzers-flower-local=1" | xargs -r docker rm -f >/dev/null 2>&1 || true +for name in flower-superlink flower-supernode flower-superexec ryzerdocker; do + docker ps -aq --filter "ancestor=${name}" | xargs -r docker rm -f >/dev/null 2>&1 || true +done + # --------------------------------------------------------------------------- # Build the three role images. NB: `ryzers build` names the *final* image # after --name (default "ryzerdocker"), NOT after the last package — so @@ -123,14 +140,6 @@ ryzers build --name flower-superlink flower-base flower-superlink ryzers build --name flower-supernode flower-base flower-supernode ryzers build --name flower-superexec flower-base flower-superexec -echo "== Cleaning up stale containers ==" -# Include the default "ryzerdocker" tag: runs from before per-role --name -# builds created containers under that image and, with --network host, -# they hold the 909x ports and break the new components. -for name in flower-superlink flower-supernode flower-superexec ryzerdocker; do - docker ps -aq --filter "ancestor=${name}" | xargs -r docker rm -f >/dev/null 2>&1 || true -done - # Start from a clean SuperLink state. A partial run left over from a # previous (e.g. failed) submit can make the SuperLink raise # KeyError('config') -> "Exception calling application: 'config'" when a From fe23e81ce039d9889e1ab2314d6356e7e949c168 Mon Sep 17 00:00:00 2001 From: Ken O'Brien Date: Thu, 11 Jun 2026 14:49:08 +0100 Subject: [PATCH 13/20] demo fixes --- packages/federated/run-local.sh | 74 +++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 4 deletions(-) diff --git a/packages/federated/run-local.sh b/packages/federated/run-local.sh index 95b74a06..1d188233 100755 --- a/packages/federated/run-local.sh +++ b/packages/federated/run-local.sh @@ -111,6 +111,41 @@ wait_for_port() { return 1 } +port_in_use() { + # 0 (true) if something is already listening on 127.0.0.1:. + local port="$1" + (echo > /dev/tcp/127.0.0.1/"${port}") 2>/dev/null +} + +# require_ports_free PORT... +# Abort if any of the given ports already has a listener. Because gRPC +# uses SO_REUSEPORT, a leftover listener would be silently co-bound to by +# the component we are about to start — the exact failure mode this script +# guards against — so we fail fast with diagnostics instead of starting on +# top of it. `wait_for_port` cannot catch this: it treats a stale listener +# as "the service is up". +require_ports_free() { + local p busy=() + for p in "$@"; do + if port_in_use "${p}"; then + busy+=("${p}") + fi + done + if ((${#busy[@]} > 0)); then + echo "ERROR: these ports are still in use after cleanup: ${busy[*]}" >&2 + echo " A stale listener here would be silently co-bound via" >&2 + echo " SO_REUSEPORT and answer some calls with errors like" >&2 + echo " \"Exception calling application: 'config'\"." >&2 + echo >&2 + echo " Find and remove what is holding them, e.g.:" >&2 + echo " docker ps -a --format '{{.ID}} {{.Image}} {{.Names}}'" >&2 + for p in "${busy[@]}"; do + echo " ss -tlnp 'sport = :${p}' # (or: lsof -iTCP:${p} -sTCP:LISTEN)" >&2 + done + exit 1 + fi +} + # --------------------------------------------------------------------------- # Clean up stale containers FIRST, before building. A rebuild moves the # flower-* tags onto new image IDs and orphans the old ones, so an @@ -118,15 +153,33 @@ wait_for_port() { # match containers from the previous run (they still reference the old # image ID) and they'd keep holding the 909x ports. # -# Primary match is the ryzers-flower-local label (set via each role's -# docker_extra_run_flags) which survives rebuilds. The ancestor/ryzerdocker -# pass is a fallback for containers created before the label existed. +# This MUST be thorough. Every component runs with `--network host`, and +# gRPC enables SO_REUSEPORT by default (flwr does not disable it), so a +# stale SuperLink/SuperNode from a previous run can silently CO-BIND 909x +# alongside the freshly started one. The kernel then load-balances +# connections across both, and the older / half-broken instance answers +# some calls with cryptic gRPC errors such as +# "Exception calling application: 'config'" (a server-side KeyError). +# That is invisible in the new SuperLink's window (it starts fine), which +# makes it very hard to diagnose — so we remove flower containers by EVERY +# signal we have, not just the label. # --------------------------------------------------------------------------- echo "== Cleaning up stale containers ==" +# 1) Primary: the label set via each role's docker_extra_run_flags +# (survives image rebuilds). docker ps -aq --filter "label=ryzers-flower-local=1" | xargs -r docker rm -f >/dev/null 2>&1 || true -for name in flower-superlink flower-supernode flower-superexec ryzerdocker; do +# 2) Fallback: anything whose image references a flower-* tag (or the +# default "ryzerdocker" tag) — for containers created before the label +# existed, or built from a differently-tagged cached base. +for name in flower-superlink flower-supernode flower-superexec flower-base ryzerdocker; do docker ps -aq --filter "ancestor=${name}" | xargs -r docker rm -f >/dev/null 2>&1 || true done +# 3) Last resort: any remaining container whose image name contains +# "flower" (catches odd tags from earlier iterations). `--filter` has no +# image wildcard, so match on the formatted list instead. +docker ps -a --format '{{.ID}} {{.Image}}' \ + | awk 'tolower($2) ~ /flower/ {print $1}' \ + | xargs -r docker rm -f >/dev/null 2>&1 || true # --------------------------------------------------------------------------- # Build the three role images. NB: `ryzers build` names the *final* image @@ -148,6 +201,19 @@ ryzers build --name flower-superexec flower-base flower-superexec echo "== Resetting SuperLink state ==" rm -rf "${REPO_ROOT}/workspace/flower/state"/* 2>/dev/null || true +# Confirm cleanup actually freed every host port we are about to bind. +# 9091 ServerAppIo, 9092 Fleet, 9093 ExecApi, plus one ClientAppIo per +# partition (9094 + i). If any is still held, abort before we start — a +# survivor would be co-bound via SO_REUSEPORT and intermittently serve +# stale responses. +echo "== Verifying ports are free ==" +PORTS_TO_CHECK=(9091 9092 9093) +for ((i = 0; i < NUM_PARTITIONS; i++)); do + PORTS_TO_CHECK+=($((9094 + i))) +done +require_ports_free "${PORTS_TO_CHECK[@]}" +echo " All required ports are free: ${PORTS_TO_CHECK[*]}" + echo "== Starting SuperLink ==" spawn "flower-superlink" "ryzers run --name flower-superlink" if ! wait_for_port 9092 120; then From 16ba0119d96cc33811223245b84fa1d03cdd8df1 Mon Sep 17 00:00:00 2001 From: Ken O'Brien Date: Fri, 12 Jun 2026 16:00:59 +0100 Subject: [PATCH 14/20] demo fixes --- packages/federated/run-local.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/federated/run-local.sh b/packages/federated/run-local.sh index 1d188233..93bfd854 100755 --- a/packages/federated/run-local.sh +++ b/packages/federated/run-local.sh @@ -181,6 +181,19 @@ docker ps -a --format '{{.ID}} {{.Image}}' \ | awk 'tolower($2) ~ /flower/ {print $1}' \ | xargs -r docker rm -f >/dev/null 2>&1 || true +# --------------------------------------------------------------------------- +# Remove stale generated run-scripts BEFORE building. `ryzers run` does NOT +# regenerate `ryzers.run..sh` — it blindly `bash`-executes whatever is +# already in the cwd (runner.py). `ryzers build` overwrites the three current +# scripts, but a script left over from an earlier package layout or image +# name is never touched and can be picked up by a stray `ryzers run`, +# reintroducing the cryptic gRPC "Exception calling application: 'config'" / +# "'script'" failures. Wipe them so every run starts from freshly generated +# scripts. +# --------------------------------------------------------------------------- +echo "== Removing stale ryzers run-scripts ==" +rm -f "${REPO_ROOT}"/ryzers.run.*.sh 2>/dev/null || true + # --------------------------------------------------------------------------- # Build the three role images. NB: `ryzers build` names the *final* image # after --name (default "ryzerdocker"), NOT after the last package — so From 2b5abb3e0f342cdc0ebc5201c52cf83566e34db7 Mon Sep 17 00:00:00 2001 From: Ken O'Brien Date: Fri, 12 Jun 2026 16:14:24 +0100 Subject: [PATCH 15/20] demo fixes --- .../flower-superexec/run-superexec.sh | 8 ++++-- packages/federated/run-local.sh | 25 +++++++++++++++---- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/packages/federated/flower-superexec/run-superexec.sh b/packages/federated/flower-superexec/run-superexec.sh index b4d19688..44ed94b4 100644 --- a/packages/federated/flower-superexec/run-superexec.sh +++ b/packages/federated/flower-superexec/run-superexec.sh @@ -29,8 +29,12 @@ FLOWER_INSECURE="${FLOWER_INSECURE:-1}" FLOWER_PARTITION_ID="${FLOWER_PARTITION_ID:-0}" if [ "${FLOWER_PLUGIN_TYPE}" = "submit" ]; then - echo "Submitting: flwr run /app local" - exec flwr run /app local + # `--stream` keeps this process attached to the run and returns only once + # the run has finished (i.e. the ServerApp has written final_model.pt to + # disk). This lets the local orchestrator (run-local.sh) detect completion + # and tear the federation down afterwards instead of leaving it running. + echo "Submitting: flwr run /app local --stream" + exec flwr run /app local --stream fi case "${FLOWER_PLUGIN_TYPE}" in diff --git a/packages/federated/run-local.sh b/packages/federated/run-local.sh index 93bfd854..afd8a07c 100755 --- a/packages/federated/run-local.sh +++ b/packages/federated/run-local.sh @@ -71,6 +71,14 @@ echo "### ${title}" ${body} status=\$? echo +# If the orchestrator dropped the shutdown sentinel, this component was +# stopped as part of a normal end-of-run teardown — exit immediately so the +# terminal window closes on its own. Otherwise (e.g. an early crash) stay +# open so the error stays readable. +if [ -f "${LOG_DIR}/.shutdown" ]; then + echo "[${title} stopped for shutdown (status \${status}) — closing window]" + exit \${status} +fi echo "[${title} exited (status \${status}) — press Enter to close]" read -r EOF @@ -261,11 +269,18 @@ fi echo " Waiting for SuperNodes to register..." sleep 8 -echo "== Submitting quickstart-pytorch run ==" +# The submit uses `flwr run ... --stream`, so this call blocks until the run +# finishes and the ServerApp has written final_model.pt to disk. +echo "== Submitting quickstart-pytorch run (streaming until complete) ==" FLOWER_PLUGIN_TYPE=submit ryzers run --name flower-superexec +# Run finished — tear the whole federation down so every per-component window +# closes on its own. Dropping the sentinel tells each spawned wrapper to exit +# (closing its terminal) instead of waiting on a keypress; killing the +# containers makes each wrapper's `docker run` return so it reaches that check. echo -echo "== Run submitted. Component windows are still open. ==" -echo "To stop everything:" -echo " for n in flower-superlink flower-supernode flower-superexec; do \\" -echo " docker ps -q --filter ancestor=\$n | xargs -r docker kill; done" +echo "== Run complete. Shutting down all components... ==" +touch "${LOG_DIR}/.shutdown" +docker ps -q --filter "label=ryzers-flower-local=1" | xargs -r docker kill >/dev/null 2>&1 || true + +echo "== Done. Model written to disk; all component windows are closing. ==" From effc18c786a7e2858e518c8bedb7b06bc3c6df83 Mon Sep 17 00:00:00 2001 From: Ken O'Brien Date: Fri, 26 Jun 2026 15:10:10 +0100 Subject: [PATCH 16/20] demo fix --- packages/federated/flower-superexec/app/pyproject.toml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/federated/flower-superexec/app/pyproject.toml b/packages/federated/flower-superexec/app/pyproject.toml index 8d4b0abb..e31e21f2 100644 --- a/packages/federated/flower-superexec/app/pyproject.toml +++ b/packages/federated/flower-superexec/app/pyproject.toml @@ -10,8 +10,16 @@ license = "Apache-2.0" # torch and torchvision are intentionally omitted: they are provided by # the ROCm/PyTorch base image. Pinning them here would cause pip to # replace the ROCm build with the upstream CUDA/CPU wheel. +# flwr is pinned to exactly the version installed by flower-base +# (FLWR_VERSION in flower-base/Dockerfile). A loose lower bound like +# "flwr>=1.26.0" lets `pip install /app` silently UPGRADE flwr in the +# superexec image to whatever is newest on PyPI at build time, while +# flower-superlink/flower-supernode stay at the pinned version. The +# resulting version skew makes the SuperLink/SuperNode raise cryptic +# server-side errors ("Exception calling application: 'config'" / "'script'") +# on the Fleet handshake. Keep this pin in lockstep with flower-base. dependencies = [ - "flwr>=1.26.0", + "flwr==1.26.1", "flwr-datasets[vision]>=0.5.0", ] From b71c599baa2f23d0aac7a1b02cf2374f13b9385f Mon Sep 17 00:00:00 2001 From: Ken O'Brien Date: Fri, 26 Jun 2026 15:55:06 +0100 Subject: [PATCH 17/20] demo fix --- packages/federated/flower-base/Dockerfile | 2 +- packages/federated/flower-superexec/app/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/federated/flower-base/Dockerfile b/packages/federated/flower-base/Dockerfile index 83c2bcf2..16c38d42 100644 --- a/packages/federated/flower-base/Dockerfile +++ b/packages/federated/flower-base/Dockerfile @@ -9,7 +9,7 @@ ENV DEBIAN_FRONTEND=noninteractive # Flower CLI binaries (flower-superlink, flower-supernode, flower-superexec) # all come from this single pip package. Torch + ROCm is provided by the # default rocm/pytorch base image, so no torch install is needed here. -ARG FLWR_VERSION=1.26.1 +ARG FLWR_VERSION=1.30.0 RUN pip3 install --no-cache-dir --break-system-packages \ "flwr[simulation]==${FLWR_VERSION}" \ "flwr-datasets[vision]>=0.5.0" diff --git a/packages/federated/flower-superexec/app/pyproject.toml b/packages/federated/flower-superexec/app/pyproject.toml index e31e21f2..2d5f5dca 100644 --- a/packages/federated/flower-superexec/app/pyproject.toml +++ b/packages/federated/flower-superexec/app/pyproject.toml @@ -19,7 +19,7 @@ license = "Apache-2.0" # server-side errors ("Exception calling application: 'config'" / "'script'") # on the Fleet handshake. Keep this pin in lockstep with flower-base. dependencies = [ - "flwr==1.26.1", + "flwr==1.30.0", "flwr-datasets[vision]>=0.5.0", ] From 361fbde206ebc9d86979224d400f071d96ce5844 Mon Sep 17 00:00:00 2001 From: Ken O'Brien Date: Mon, 29 Jun 2026 14:28:24 +0100 Subject: [PATCH 18/20] fix to flower version mismatch --- packages/federated/flower-base/config.yaml | 2 +- packages/federated/flower-superexec/app/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/federated/flower-base/config.yaml b/packages/federated/flower-base/config.yaml index 9b70209a..979668fa 100644 --- a/packages/federated/flower-base/config.yaml +++ b/packages/federated/flower-base/config.yaml @@ -9,4 +9,4 @@ # ryzers build --name flower-superexec flower-base flower-superexec build_arguments: -- "FLWR_VERSION=1.26.1" +- "FLWR_VERSION=1.32.0" diff --git a/packages/federated/flower-superexec/app/pyproject.toml b/packages/federated/flower-superexec/app/pyproject.toml index 2d5f5dca..317a0b46 100644 --- a/packages/federated/flower-superexec/app/pyproject.toml +++ b/packages/federated/flower-superexec/app/pyproject.toml @@ -19,7 +19,7 @@ license = "Apache-2.0" # server-side errors ("Exception calling application: 'config'" / "'script'") # on the Fleet handshake. Keep this pin in lockstep with flower-base. dependencies = [ - "flwr==1.30.0", + "flwr==1.32.0", "flwr-datasets[vision]>=0.5.0", ] From f3fb22f80f1c480a9a617f3e45fc40cad155fda1 Mon Sep 17 00:00:00 2001 From: Ken O'Brien Date: Tue, 18 Aug 2026 16:10:12 +0100 Subject: [PATCH 19/20] bump to 7.14 --- ryzers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ryzers/__init__.py b/ryzers/__init__.py index 7335304b..9be24ac1 100644 --- a/ryzers/__init__.py +++ b/ryzers/__init__.py @@ -3,7 +3,7 @@ import os -RYZERS_DEFAULT_INIT_IMAGE = "rocm/pytorch:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0" +RYZERS_DEFAULT_INIT_IMAGE = "rocm/pytorch:rocm7.14_ubuntu26.04_py3.14_pytorch_release_2.12.0" RYZERS_DEFAULT_RUN_FLAGS = "-it --rm --shm-size 16G --cap-add=SYS_PTRACE --network=host --ipc=host" # Auto-detect packages path in editable mode From 9632398836a32dff5e22c0efe1cd3b68e3947a50 Mon Sep 17 00:00:00 2001 From: Ken O'Brien Date: Tue, 18 Aug 2026 16:46:45 +0100 Subject: [PATCH 20/20] fixes for rocm 7.14 --- packages/federated/flower-base/Dockerfile | 16 ++++++++++++++++ packages/federated/flower-superexec/Dockerfile | 10 ++++++++++ 2 files changed, 26 insertions(+) diff --git a/packages/federated/flower-base/Dockerfile b/packages/federated/flower-base/Dockerfile index 16c38d42..71a663a7 100644 --- a/packages/federated/flower-base/Dockerfile +++ b/packages/federated/flower-base/Dockerfile @@ -14,6 +14,22 @@ RUN pip3 install --no-cache-dir --break-system-packages \ "flwr[simulation]==${FLWR_VERSION}" \ "flwr-datasets[vision]>=0.5.0" +# Python 3.14 fix. The base image is now py3.14, but flwr-datasets 0.5.0 +# hard-pins datasets<=3.1.0, which drags in dill 0.3.8. That old +# datasets/dill combo runs a fingerprinting path that trips over py3.14's +# changed pickle.Pickler._batch_setitems() signature: +# TypeError: _batch_setitems() takes 2 positional arguments but 3 were given +# raised the moment a run loads a dataset (e.g. the ServerApp's CIFAR-10 +# eval set), aborting the whole federation. datasets 4.x drops that legacy +# code path and works under py3.14. It must be installed in a SEPARATE step: +# resolving it together with flwr-datasets fails (its <=3.1.0 cap), and +# bumping flwr-datasets to 0.6.0 instead is blocked by a rich pin conflict +# with flwr 1.32.0. The residual pip "incompatible" notice about the 3.1.0 +# cap is cosmetic — 0.5.0 works with datasets 4.x at runtime. +ARG DATASETS_VERSION=4.8.5 +RUN pip3 install --no-cache-dir --break-system-packages \ + "datasets==${DATASETS_VERSION}" + WORKDIR /ryzers COPY test.sh /ryzers/test_flower-base.sh RUN chmod +x /ryzers/test_flower-base.sh diff --git a/packages/federated/flower-superexec/Dockerfile b/packages/federated/flower-superexec/Dockerfile index 84cfa260..84764f1c 100644 --- a/packages/federated/flower-superexec/Dockerfile +++ b/packages/federated/flower-superexec/Dockerfile @@ -11,6 +11,16 @@ WORKDIR /app COPY app/ /app/ RUN pip3 install --no-cache-dir --break-system-packages /app +# Re-assert the py3.14-compatible datasets. `pip install /app` above +# re-resolves the app's deps and, because flwr-datasets 0.5.0 hard-pins +# datasets<=3.1.0, downgrades datasets from the 4.x pinned in flower-base +# back to 3.1.0 — which reintroduces the py3.14 dill/pickle crash +# (_batch_setitems() TypeError) on the first dataset load. Keep this in +# lockstep with DATASETS_VERSION in flower-base/Dockerfile. +ARG DATASETS_VERSION=4.8.5 +RUN pip3 install --no-cache-dir --break-system-packages \ + "datasets==${DATASETS_VERSION}" + WORKDIR /ryzers COPY test.sh /ryzers/test_flower-superexec.sh COPY run-superexec.sh /ryzers/run-superexec.sh