From d5cbb1bf00988d7a043e351c7eeb62c9fb7bc233 Mon Sep 17 00:00:00 2001 From: "Ahmet (Buray)" Date: Thu, 3 Sep 2026 10:01:02 +0000 Subject: [PATCH 1/3] docs: add Podman deployment guide (v1.1.1) Podman is not mentioned anywhere in the repo despite being a common rootless container runtime for self-hosted node/validator operators. Adds docs/versions/v1.1.1/07_podman.md covering the parts that differ from Quick Start (Docker): why --network=host is required (same root cause as the Kubernetes hostNetwork requirement - no announce-address override), the :Z SELinux volume label, and a production systemd Quadlet unit using a Podman secret for the API key. Targets v1.1.1 per #99's 'Add to v1.1.1 (flows to latest)' - not v1.2.0, which I mistakenly used in an earlier draft. Scope note: this covers the production single-host Podman deployment gap. It does NOT attempt the examples/compose// self-configuring reference stack from #99's follow-up comment (a tiny entrypoint that polls self_info and dynamically injects peer IDs to avoid templating them on both sides up front) - that's a separate, larger deliverable serving a different scenario (local dev/test with CL+gateway on the same compose network) than this page (production gateway needing public reachability on 33213). Ref #99. --- docs/.vitepress/config.mts | 1 + docs/versions/v1.1.1/01_quick_start.md | 4 + docs/versions/v1.1.1/07_podman.md | 195 +++++++++++++++++++++++++ docs/versions/v1.1.1/index.md | 2 +- 4 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 docs/versions/v1.1.1/07_podman.md diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index e6137ce..df01898 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -40,6 +40,7 @@ const securityAudit = { const possibleItems = [ { text: 'Network Requirements', file: '00_network_requirements.md' }, { text: 'Quick Start', file: '01_quick_start.md' }, + { text: 'Podman', file: '07_podman.md' }, { text: 'Kubernetes (Helm)', file: '05_kubernetes.md' }, { text: 'Configuration', file: '02_configuration.md' }, { text: 'Consumer Block Stream', file: '06_block_stream.md' }, diff --git a/docs/versions/v1.1.1/01_quick_start.md b/docs/versions/v1.1.1/01_quick_start.md index 674c552..2b0a9b1 100644 --- a/docs/versions/v1.1.1/01_quick_start.md +++ b/docs/versions/v1.1.1/01_quick_start.md @@ -3,6 +3,10 @@ Get the Optimum Gateway running with Docker. > **Running on Kubernetes?** See [Kubernetes (Helm)](05_kubernetes.md) for the official Helm chart. +> +> **Running rootless Podman?** See [Podman](07_podman.md) — same binary and +> config, but Podman's default rootless network needs `--network=host` or the +> gateway advertises an unreachable address. > **Prerequisites:** [Requirements](index.md#requirements) and [Network Requirements](00_network_requirements.md). You also need an **API key** — see [Generate your API key](#generate-your-api-key) below. diff --git a/docs/versions/v1.1.1/07_podman.md b/docs/versions/v1.1.1/07_podman.md new file mode 100644 index 0000000..6ffee05 --- /dev/null +++ b/docs/versions/v1.1.1/07_podman.md @@ -0,0 +1,195 @@ +# Podman + +Run the Optimum Gateway with rootless Podman. Same binary and config as +[Quick Start (Docker)](01_quick_start.md) — this page only covers where +Podman's defaults differ from Docker's and will silently break the gateway +if you don't account for them. + +> **Prerequisites:** [Requirements](index.md#requirements) and +> [Network Requirements](00_network_requirements.md). You also need an +> **API key** — see [Generate your API key](01_quick_start.md#generate-your-api-key). + +> **Running on Kubernetes instead?** See [Kubernetes (Helm)](05_kubernetes.md). + +## Why this page exists: rootless networking + +Docker's default bridge network and Podman's default rootless network are +**not the same thing**. Rootless Podman without `--network=host` runs the +container behind `slirp4netns`/`pasta` — a user-mode NAT. The gateway inside +that namespace detects a private, non-routable address, advertises *that* to +the Optimum bootstrap registry, registers as "reachable," and then silently +receives **no inbound mump2p peers** — publishing port `33213` on the Podman +CLI does not fix this, because the gateway never advertises the address that +port mapping makes reachable. + +This is the same underlying problem the [Kubernetes guide](05_kubernetes.md#networking-why-hostnetwork-is-required) +documents for `hostNetwork` — the gateway has no announce-address override. +The fix here is the same shape: run the container in the host's network +namespace. + +```sh +--network=host +``` + +With `--network=host` there is nothing to publish — the container simply +listens on the host's ports directly. Do not combine `-p`/`--publish` with +`--network=host`; Podman will ignore or reject it. + +## Requirements + +* **Podman 4.4+** (rootless) +* A host with a **public IP** +* Inbound **TCP 33213** open to that host from the internet — the Optimum + mump2p port; the gateway is unusable without it +* The CL client able to reach the gateway on **TCP 33212** +* Outbound HTTPS to `bootstrap.getoptimum.io` and `auth.getoptimum.io` +* SELinux-enforcing hosts (Fedora, RHEL, CentOS Stream): see the volume note below + +## Run + +```sh +mkdir -p config data/libp2p data/mump2p +# fill in config/app_conf.yml — see Quick Start (Docker) for the minimal example +``` + +```sh +podman run --name optimum-gateway -d \ + --network=host \ + -e OPT_API_KEY=$OPT_API_KEY \ + -v $(pwd)/config:/app/config:Z \ + -v $(pwd)/data/libp2p:/tmp/libp2p:Z \ + -v $(pwd)/data/mump2p:/tmp/mump2p:Z \ + getoptimum/gateway:v1.1.1 \ + -config=/app/config/app_conf.yml +``` + +### The `:Z` suffix + +On SELinux-enforcing hosts, a bind mount without a label suffix fails with a +generic `permission denied` — nothing in the gateway's own logs points at +SELinux. `:Z` applies a private, container-specific label to the directory +so **only this container** can access it. Use `:z` (lowercase) instead only +if the same host directory must be shared with another container running at +the same time. On non-SELinux hosts (Debian, Ubuntu without SELinux) the +suffix is accepted and ignored — safe to leave in either way. + +If you see `permission denied` on the identity or config directories despite +the `:Z` suffix, confirm the host directory is owned by your user, not root: + +```sh +podman unshare chown -R $(id -u):$(id -g) data/libp2p data/mump2p +``` + +> Persist `data/libp2p` and `data/mump2p` across restarts — as with Docker, +> without them the gateway's peer ID changes every run and your CL client's +> configured multiaddr goes stale. + +## Verify + +Same checks as Docker, since `--network=host` means the ports are just the +host's ports: + +```sh +curl -s http://localhost:48123/health | jq +curl -s http://localhost:48123/api/v1/self_info | jq '{peer_id, multiaddrs: .libp2p.multiaddrs}' +``` + +`multiaddrs` must show a **public** IP, not `10.x`/`172.x`/`192.168.x`. A +private-only address here means `--network=host` was dropped somewhere, or +the host itself has no public interface. + +```sh +podman logs optimum-gateway +``` + +## Connect CL Client + +Identical to [Quick Start — Connect CL Client](01_quick_start.md#connect-cl-client): +get `peer_id` and a reachable multiaddr from `self_info`, then use the +Prysm/Teku/Lighthouse/Nimbus/Lodestar flags documented there. Nothing about +peering changes with Podman — only the container networking above does. + +## Production: systemd Quadlet + +For a restart-on-boot, restart-on-crash deployment, use a +[Quadlet](https://docs.podman.io/en/latest/markdown/podman-systemd.unit.5.html) +unit instead of a bare `podman run`. Quadlet generates a systemd service from +a declarative `.container` file — the Podman equivalent of Compose's +`restart: unless-stopped`, but managed by systemd (`systemctl status`, +`journalctl`, boot-time start). + +`~/.config/containers/systemd/optimum-gateway.container`: + +```ini +[Unit] +Description=Optimum Gateway +After=network-online.target +Wants=network-online.target + +[Container] +Image=getoptimum/gateway:v1.1.1 +ContainerName=optimum-gateway +Network=host +Exec=-config=/app/config/app_conf.yml +Secret=optimum-gateway-api-key,type=env,target=OPT_API_KEY +Volume=%h/optimum-gateway/config:/app/config:Z +Volume=%h/optimum-gateway/data/libp2p:/tmp/libp2p:Z +Volume=%h/optimum-gateway/data/mump2p:/tmp/mump2p:Z +HealthCmd=curl -f http://localhost:48123/api/v1/self_info || exit 1 +HealthInterval=30s + +[Service] +Restart=always + +[Install] +WantedBy=default.target +``` + +The API key is passed as a **Podman secret**, not a plaintext env line in +the unit file — mirrors the Kubernetes guide's use of a +[Secret](05_kubernetes.md#install) rather than a value in `values.yaml`: + +```sh +mkdir -p ~/.config/containers/systemd +echo -n 'ogw_live_...' | podman secret create optimum-gateway-api-key - +``` + +Then, for a user session that should keep running after logout: + +```sh +loginctl enable-linger $(whoami) +systemctl --user daemon-reload +systemctl --user start optimum-gateway.service +systemctl --user status optimum-gateway.service +journalctl --user -u optimum-gateway.service -f +``` + +Editing the `.container` file requires `daemon-reload` before it takes +effect, same as any systemd unit change. + +## When it doesn't work + +Podman-specific symptoms below. For gateway behaviour that isn't +Podman-specific (CL peering, PeerDAS, identity, log noise), see +[Troubleshooting](04_troubleshoot.md). + +| Symptom | Cause | +|---|---| +| `permission denied` on `/app/config` or `/tmp/libp2p` at startup | Missing `:Z` on the volume mount, or the host directory isn't owned by your user — see [The `:Z` suffix](#the-z-suffix) | +| `mump2p_peers: 0` after a few minutes, but `cl_peers` is fine | Container is not on `--network=host` — check `podman inspect optimum-gateway --format '{{.HostConfig.NetworkMode}}'`, expect `host` | +| `self_info` shows only a private IP in `multiaddrs` | Same as above, or the host itself has no public interface | +| `Error: address already in use` on `podman run` | Another process (or a previous gateway container) already holds `33212`/`33213`/`48123` on the host — with `--network=host` there is no port remapping to fall back on | +| Peer ID changes every restart | `data/libp2p` / `data/mump2p` not persisted, or pointed at the wrong host path — confirm the bind mount source matches across restarts | +| Quadlet service won't start; `systemctl --user status` shows nothing | Run `systemctl --user daemon-reload` after creating/editing the `.container` file | +| Quadlet service stops when you log out | `loginctl enable-linger $(whoami)` was not run | + +## Getting help + +Send us this — it answers most of the first round of questions: + +```sh +podman inspect optimum-gateway --format '{{.HostConfig.NetworkMode}}' +podman logs optimum-gateway --tail 100 +curl -s http://localhost:48123/health +curl -s http://localhost:48123/api/v1/self_info +``` diff --git a/docs/versions/v1.1.1/index.md b/docs/versions/v1.1.1/index.md index dad3632..9f906c8 100644 --- a/docs/versions/v1.1.1/index.md +++ b/docs/versions/v1.1.1/index.md @@ -82,7 +82,7 @@ The **Optimum Gateway** bridges your **Ethereum Consensus Layer (CL) client** wi * **CL Client**: Prysm, Lighthouse, Teku, Nimbus, or Lodestar running * **API key**: Issued from the [Optimum Partner Console](https://console.getoptimum.io/) after onboarding (see [Quick Start](01_quick_start.md#generate-your-api-key)) -* **Docker**: Docker Desktop or Docker Engine +* **Container runtime**: Docker Desktop/Engine, or rootless Podman 4.4+ (see [Podman](07_podman.md)) * **Firewall**: Required ports open (see [Network Requirements](00_network_requirements.md)) ## Getting Started From 7e29b9ec107eb7cbef7e04cbf1f6748130262211 Mon Sep 17 00:00:00 2001 From: "Ahmet (Buray)" Date: Thu, 3 Sep 2026 11:06:36 +0000 Subject: [PATCH 2/3] fix: address CodeRabbit review on #107 7 findings, all confirmed legitimate: - config bind mount: add ,ro (config is never written by the gateway) - ownership-repair command was missing 'config' despite the surrounding text saying it fixes /app/config permission errors - Quadlet section now sets up ~/optimum-gateway/{config,data/...} explicitly before showing the unit file - the earlier podman run example uses paths relative to the current directory, which don't match the %h/optimum-gateway root the Quadlet Volume= lines assume - removed HealthCmd=curl ... - verified against the actual published getoptimum/gateway:v1.1.1 image manifests (both linux/amd64 and linux/arm64): no curl binary in either. Replaced with host-side health check guidance instead of an in-container command that would silently never succeed - podman secret create example no longer puts the literal API key on the command line (shell history); reads it via 'read -s' instead - systemctl --user start -> enable --now, matching the lingering + WantedBy=default.target setup earlier in the same section - daemon-reload does not restart an already-running unit; added the restart step after it --- docs/versions/v1.1.1/07_podman.md | 46 +++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/docs/versions/v1.1.1/07_podman.md b/docs/versions/v1.1.1/07_podman.md index 6ffee05..e687d8c 100644 --- a/docs/versions/v1.1.1/07_podman.md +++ b/docs/versions/v1.1.1/07_podman.md @@ -56,7 +56,7 @@ mkdir -p config data/libp2p data/mump2p podman run --name optimum-gateway -d \ --network=host \ -e OPT_API_KEY=$OPT_API_KEY \ - -v $(pwd)/config:/app/config:Z \ + -v $(pwd)/config:/app/config:Z,ro \ -v $(pwd)/data/libp2p:/tmp/libp2p:Z \ -v $(pwd)/data/mump2p:/tmp/mump2p:Z \ getoptimum/gateway:v1.1.1 \ @@ -77,7 +77,7 @@ If you see `permission denied` on the identity or config directories despite the `:Z` suffix, confirm the host directory is owned by your user, not root: ```sh -podman unshare chown -R $(id -u):$(id -g) data/libp2p data/mump2p +podman unshare chown -R $(id -u):$(id -g) config data/libp2p data/mump2p ``` > Persist `data/libp2p` and `data/mump2p` across restarts — as with Docker, @@ -120,6 +120,15 @@ a declarative `.container` file — the Podman equivalent of Compose's `~/.config/containers/systemd/optimum-gateway.container`: +Quadlet volume paths are rooted at `%h` (your home directory), not the +current directory the earlier `podman run` example used — set up matching +directories first: + +```sh +mkdir -p ~/optimum-gateway/{config,data/libp2p,data/mump2p} +cp config/app_conf.yml ~/optimum-gateway/config/ # or write it fresh here +``` + ```ini [Unit] Description=Optimum Gateway @@ -132,11 +141,9 @@ ContainerName=optimum-gateway Network=host Exec=-config=/app/config/app_conf.yml Secret=optimum-gateway-api-key,type=env,target=OPT_API_KEY -Volume=%h/optimum-gateway/config:/app/config:Z +Volume=%h/optimum-gateway/config:/app/config:Z,ro Volume=%h/optimum-gateway/data/libp2p:/tmp/libp2p:Z Volume=%h/optimum-gateway/data/mump2p:/tmp/mump2p:Z -HealthCmd=curl -f http://localhost:48123/api/v1/self_info || exit 1 -HealthInterval=30s [Service] Restart=always @@ -145,27 +152,50 @@ Restart=always WantedBy=default.target ``` +No in-container `HealthCmd` here — the published `getoptimum/gateway` +images don't include `curl` (or any shell utilities beyond the binary +itself), so a `HealthCmd` that shells out to `curl` never runs. Check +health from the host instead: + +```sh +curl -s http://localhost:48123/health | jq +``` + +or point a systemd timer / your own monitoring at that same URL from +outside the container. + The API key is passed as a **Podman secret**, not a plaintext env line in the unit file — mirrors the Kubernetes guide's use of a [Secret](05_kubernetes.md#install) rather than a value in `values.yaml`: ```sh mkdir -p ~/.config/containers/systemd -echo -n 'ogw_live_...' | podman secret create optimum-gateway-api-key - +read -rsp 'API key: ' OPT_API_KEY_VALUE && echo +printf '%s' "$OPT_API_KEY_VALUE" | podman secret create optimum-gateway-api-key - +unset OPT_API_KEY_VALUE ``` +`read -s` keeps the key out of your shell history — pasting the literal +`ogw_live_...` value into a command line puts it there permanently. + Then, for a user session that should keep running after logout: ```sh loginctl enable-linger $(whoami) systemctl --user daemon-reload -systemctl --user start optimum-gateway.service +systemctl --user enable --now optimum-gateway.service systemctl --user status optimum-gateway.service journalctl --user -u optimum-gateway.service -f ``` Editing the `.container` file requires `daemon-reload` before it takes -effect, same as any systemd unit change. +effect, same as any systemd unit change — but `daemon-reload` alone does +**not** restart an already-running service with the new definition: + +```sh +systemctl --user daemon-reload +systemctl --user restart optimum-gateway.service +``` ## When it doesn't work From b5cb045ab429b8a7a7bbe0f17149e5ae3333c759 Mon Sep 17 00:00:00 2001 From: "Ahmet (Buray)" Date: Thu, 3 Sep 2026 13:23:56 +0000 Subject: [PATCH 3/3] fix: address remaining CodeRabbit findings on #107 - MD028: merged the two intro blockquotes into one (internal blank '>' paragraph break) instead of two blockquotes separated by an external blank line - security note: --network=host also exposes the telemetry HTTP server (health/metrics/self_info on 48123) beyond localhost - verified in cmd/main.go, it binds to a bare ':48123' (wildcard, all interfaces), not 127.0.0.1. Out of scope to change in a docs PR, so documented the risk and a host-firewall mitigation instead - mkdir brace expansion and 'read -rsp' were bash-specific inside a ```sh (POSIX) fenced block; replaced with three plain mkdir -p calls and a POSIX read + stty -echo/echo prompt - the API-key read now aborts before podman secret create if the input is empty, instead of silently creating an empty secret (flagged as the PR's top merge-risk item - an empty secret would leave gateway token verification effectively disabled) --- docs/versions/v1.1.1/07_podman.md | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/docs/versions/v1.1.1/07_podman.md b/docs/versions/v1.1.1/07_podman.md index e687d8c..945d14b 100644 --- a/docs/versions/v1.1.1/07_podman.md +++ b/docs/versions/v1.1.1/07_podman.md @@ -8,7 +8,7 @@ if you don't account for them. > **Prerequisites:** [Requirements](index.md#requirements) and > [Network Requirements](00_network_requirements.md). You also need an > **API key** — see [Generate your API key](01_quick_start.md#generate-your-api-key). - +> > **Running on Kubernetes instead?** See [Kubernetes (Helm)](05_kubernetes.md). ## Why this page exists: rootless networking @@ -35,6 +35,14 @@ With `--network=host` there is nothing to publish — the container simply listens on the host's ports directly. Do not combine `-p`/`--publish` with `--network=host`; Podman will ignore or reject it. +> **Security note:** the gateway's telemetry HTTP server (`/health`, +> `/metrics`, `/api/v1/self_info` on port 48123) listens on all interfaces, +> not just localhost. With `--network=host` that means it's reachable from +> the network the host is on — the public internet, if the host has a +> public IP with no firewall in front. Restrict port 48123 with a host +> firewall (e.g. `ufw deny 48123` / only allow from your monitoring +> source) unless you specifically want it externally reachable. + ## Requirements * **Podman 4.4+** (rootless) @@ -125,7 +133,7 @@ current directory the earlier `podman run` example used — set up matching directories first: ```sh -mkdir -p ~/optimum-gateway/{config,data/libp2p,data/mump2p} +mkdir -p ~/optimum-gateway/config ~/optimum-gateway/data/libp2p ~/optimum-gateway/data/mump2p cp config/app_conf.yml ~/optimum-gateway/config/ # or write it fresh here ``` @@ -170,8 +178,16 @@ the unit file — mirrors the Kubernetes guide's use of a ```sh mkdir -p ~/.config/containers/systemd -read -rsp 'API key: ' OPT_API_KEY_VALUE && echo -printf '%s' "$OPT_API_KEY_VALUE" | podman secret create optimum-gateway-api-key - +printf 'API key: ' +stty -echo +read -r OPT_API_KEY_VALUE +stty echo +echo +if [ -z "$OPT_API_KEY_VALUE" ]; then + echo "no key entered, aborting" >&2 +else + printf '%s' "$OPT_API_KEY_VALUE" | podman secret create optimum-gateway-api-key - +fi unset OPT_API_KEY_VALUE ```