diff --git a/docs/versions/v1.1.1/00_network_requirements.md b/docs/versions/v1.1.1/00_network_requirements.md index 3b5cc2c..508bb50 100644 --- a/docs/versions/v1.1.1/00_network_requirements.md +++ b/docs/versions/v1.1.1/00_network_requirements.md @@ -7,15 +7,15 @@ Before starting the Optimum Gateway, ensure your firewall allows these ports: | Port | Direction | Purpose | | ----- | --------- | ------------------------------------------------------------------- | | 33212 | Inbound | libp2p agent - CL clients connect here | -| 33213 TCP + UDP | Inbound | mump2p agent - Optimum network peers connect here (UDP is QUIC-v1) | +| 33213 | Inbound | mump2p agent - Optimum network peers connect here | | 48123 | Localhost | Telemetry / Health / API — **local access only** (see Docker below) | ## Docker -Publish **33212/TCP** (CL clients) and **33213/TCP + 33213/UDP** (Optimum mump2p mesh; UDP is QUIC-v1) to the network. Bind **48123** to localhost only — metrics and health stay available on the host (e.g. local Grafana) but are not reachable from the public internet. Optimum receives telemetry via remote push; you do not need to expose 48123 externally. +Publish **33212** (CL clients) and **33213** (Optimum mump2p mesh) to the network. Bind **48123** to localhost only — metrics and health stay available on the host (e.g. local Grafana) but are not reachable from the public internet. Optimum receives telemetry via remote push; you do not need to expose 48123 externally. ```bash -docker run -p 33212:33212/tcp -p 33213:33213/tcp -p 33213:33213/udp -p 127.0.0.1:48123:48123/tcp \ +docker run -p 33212:33212 -p 33213:33213 -p 127.0.0.1:48123:48123 \ --name optimum-gateway \ -e OPT_API_KEY=ogw_live_xxx \ getoptimum/gateway:v1.1.1 @@ -27,7 +27,7 @@ If Prometheus runs in another container on the same Docker network, omit the `48 ```bash # Check listening ports (Linux) -netstat -tulnp | grep -E "(33212|33213|48123)" +netstat -tlnp | grep -E "(33212|33213|48123)" # Check listening ports (macOS) netstat -an | grep -E "(33212|33213|48123)" diff --git a/docs/versions/v1.1.1/01_quick_start.md b/docs/versions/v1.1.1/01_quick_start.md index b59a942..674c552 100644 --- a/docs/versions/v1.1.1/01_quick_start.md +++ b/docs/versions/v1.1.1/01_quick_start.md @@ -81,7 +81,6 @@ mkdir -p config data/libp2p data/mump2p docker run --name optimum-gateway --rm \ -p 33212:33212/tcp \ -p 33213:33213/tcp \ - -p 33213:33213/udp \ -p 127.0.0.1:48123:48123/tcp \ -e OPT_API_KEY=$OPT_API_KEY \ -v $(pwd)/config:/app/config \ @@ -93,8 +92,6 @@ docker run --name optimum-gateway --rm \ > Persist the identity volumes (`/tmp/libp2p`, `/tmp/mump2p`) across restarts — otherwise the gateway's peer ID changes on every run and your CL client config breaks. -> `agent_mump2p_port` must be published and allowed through the firewall for both TCP and UDP (QUIC-v1). The example uses port `33213` for both transports. - ## Verify Health check: diff --git a/docs/versions/v1.1.1/02_configuration.md b/docs/versions/v1.1.1/02_configuration.md index 3041f99..42d2aec 100644 --- a/docs/versions/v1.1.1/02_configuration.md +++ b/docs/versions/v1.1.1/02_configuration.md @@ -110,7 +110,7 @@ The gateway receives automatic config updates from bootstrap. | `api_key` | `OPT_API_KEY` | *(required)* | Gateway API key (`ogw_live_...`). **Set via env, not YAML.** Drives gateway_id, chain, and validator scope | | `gateway_cluster_id` | `OPT_GATEWAY_CLUSTER_ID` | *(required)* | Cluster ID assigned by Optimum during onboarding | | `agent_lib_p2p_port` | `OPT_AGENT_LIB_P2P_PORT` | 33212 | CL clients connect here (inbound) | -| `agent_mump2p_port` | `OPT_AGENT_MUMP2P_PORT` | 33213 | mump2p agent port for TCP and UDP (QUIC-v1); allow inbound reachability for both. Sample config uses `43213` | +| `agent_mump2p_port` | `OPT_AGENT_MUMP2P_PORT` | 33213 | mump2p agent port (outbound). Sample config uses `43213` | | `telemetry_enable` | `OPT_ENABLE_TELEMETRY` | false | Enable metrics / health endpoint | | `telemetry_port` | `OPT_TELEMETRY_PORT` | 48123 | Telemetry HTTP port (`/health`, `/metrics`, `/api/v1/self_info`) | | `identity_libp2p_dir` | `OPT_IDENTITY_LIBP2P_DIR` | /tmp/libp2p | libp2p identity dir — **persist as a volume** | @@ -143,7 +143,7 @@ docker logs optimum-gateway | grep "subscribed to topic" * **Wrong network** — Chain comes from the API key, not YAML. If `/api/v1/self_info` shows the wrong `chain`, you are using the wrong key. Get the right key from Optimum * **Missing `gateway_cluster_id`** — Required; use the ID assigned during onboarding * **API key in YAML / image** — Move it to `OPT_API_KEY` in the environment -* **Port conflicts** — Ensure 33212/TCP, 43213/TCP + UDP, and 48123 are free +* **Port conflicts** — Ensure 33212, 43213, 48123 are free * **Peer ID changes after restart** — Persist `identity_libp2p_dir` and `identity_mump2p_dir` as volumes See [Troubleshooting](04_troubleshoot.md) for more. diff --git a/docs/versions/v1.1.1/04_troubleshoot.md b/docs/versions/v1.1.1/04_troubleshoot.md index 2ca27f5..ccf57ef 100644 --- a/docs/versions/v1.1.1/04_troubleshoot.md +++ b/docs/versions/v1.1.1/04_troubleshoot.md @@ -20,7 +20,7 @@ This guide covers the operational issues you can hit running the Optimum Gateway | `gateway_cluster_id` | Must match onboarding (Hoodi vs Mainnet) | | `identity_libp2p_dir` / `identity_mump2p_dir` | **Persist as volumes** — without them, peer ID changes every restart | | `agent_lib_p2p_port` | Default `33212`; CL connects here | -| `agent_mump2p_port` | Default `43213`; mump2p TCP + UDP (QUIC-v1), reachable from mesh peers | +| `agent_mump2p_port` | Default `43213`; mump2p egress | | `telemetry_enable` / `telemetry_port` | Default port `48123` | | `direct_cl_peers` | Optional, **strongly recommended** for Lighthouse / Nimbus | | `remote_push_enable` | Optional; pushes logs/metrics to Optimum for support visibility | @@ -60,8 +60,8 @@ docker logs optimum-gateway --tail=50 | Failing check | What it means | Fix | | -------------------- | -------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | -| `cl_peers` | No CL client connected on `:33212` | Configure CL peering; add `direct_cl_peers`; allow the intended CL to reach **33212/TCP**; use a reachable IP in the multiaddr; restart CL after a gateway restart | -| `mump2p_peers` | Not connected to the Optimum mesh | Check **43213/TCP inbound** and HTTPS to bootstrap; also check **43213/UDP** when QUIC is expected; verify `api_key` + `gateway_cluster_id`; wait 2-5 min after start | +| `cl_peers` | No CL client connected on `:33212` | Configure CL peering; add `direct_cl_peers`; open **33212 inbound**; use a reachable IP in the multiaddr; restart CL after a gateway restart | +| `mump2p_peers` | Not connected to the Optimum mesh | Check **43213 outbound** + HTTPS to bootstrap; verify `api_key` + `gateway_cluster_id`; wait 2-5 min after start | | `subscribed_topics` | Topic subscription failed (expect ~65) | Usually a chain/cluster mismatch (wrong API key for the network); check logs for subscribe errors | | `last_block_age_sec` | No beacon block in ~60s | CL is connected but **silent** — CL not synced, EL stuck, or CL OOM/restart; fix the CL first | | `cl_health` | No CL gossip traffic in the last 30s | Same as silent CL — peer count alone can be misleading | @@ -276,19 +276,17 @@ Lodestar is supported. Add the gateway as a trusted/direct peer in Lodestar, and | Port | Direction | Action | | ------- | --------- | --------------------------------------------------------------------------------------------------- | -| `33212` | Inbound from intended CL only | CL clients connect here; do not expose it publicly | +| `33212` | Inbound | CL clients connect here | | `48123` | Localhost | `/health`, `/metrics`, `/api/v1/self_info` — see [Network Requirements](00_network_requirements.md) | -| `43213` | Inbound TCP + UDP | mump2p mesh; UDP is QUIC-v1 | +| `43213` | Outbound | mump2p mesh egress (no inbound rule needed) | | `443` | Outbound | auth, bootstrap, Loki, Mimir | ```bash -sudo lsof -iTCP:33212 -iTCP:43213 -iUDP:43213 -iTCP:48123 -sudo ufw allow from to any port 33212 proto tcp -sudo ufw allow 43213/tcp -sudo ufw allow 43213/udp +sudo lsof -i :33212 -i :43213 -i :48123 +sudo ufw allow 33212/tcp ``` -Expose `agent_mump2p_port` publicly over **TCP + UDP** to enable QUIC; TCP remains a fallback if UDP is blocked. Allow **33212/TCP** only from the intended/local CL. **Docker tip:** `--network host` exposes all ports on the host network interface. +Only **33212** needs a public inbound firewall rule. **Docker tip:** `--network host` exposes all ports on the host network interface. ## Identity / persistence diff --git a/docs/versions/v1.1.1/05_kubernetes.md b/docs/versions/v1.1.1/05_kubernetes.md index 33e5f13..f7cb74e 100644 --- a/docs/versions/v1.1.1/05_kubernetes.md +++ b/docs/versions/v1.1.1/05_kubernetes.md @@ -26,9 +26,8 @@ for a **digest**, not a tag: a tag can be repointed, a digest cannot. * **Kubernetes 1.25+**, **Helm 3.8+** * A node with a **public IP** -* Inbound **TCP and UDP 33213** open to that node from the internet — this is - the Optimum network (mump2p) port; UDP carries QUIC-v1. TCP remains a - fallback if UDP is unavailable, but opening both enables QUIC. +* Inbound **TCP 33213** open to that node from the internet — this is the + Optimum network (mump2p) port and the gateway is unusable without it * The CL client able to reach the gateway on **TCP 33212** (usually in-cluster; no public firewall hole needed for this) * A namespace permitting `hostNetwork` and `hostPort`. Under Pod Security @@ -53,8 +52,7 @@ tell the gateway to announce that external address. Reachability comes from So the chart defaults to `networking.hostNetwork: true` (the K8s equivalent of Docker host-mode) and one gateway per node via anti-affinity, since it binds host -ports. Schedule it on a node with a public IP and open inbound **33213/TCP + UDP** -to enable QUIC (TCP remains a fallback). +ports. Schedule it on a node with a public IP and open inbound **33213**. ## Install @@ -227,7 +225,7 @@ Kubernetes-specific symptoms below. For gateway behaviour that isn't K8s-specifi | `helm install` says deployed but there are no pods | Pod Security is rejecting them. `kubectl -n optimum describe rs -l app.kubernetes.io/instance=gateway` — if it mentions `violates PodSecurity`, the namespace needs `privileged` | | `CrashLoopBackOff` immediately | Usually a rejected API key — `kubectl logs` will show `api key not recognized (401)` | | `cl_peers: 0` | `directClPeers` wrong, **or** your CL client was never pointed at the gateway (see above) | -| `mump2p_peers: 0` after a few minutes | Inbound **33213/TCP** is not reachable from the internet; also open UDP to enable QUIC | +| `mump2p_peers: 0` after a few minutes | Inbound **33213** is not reachable from the internet | | `multiaddrs` shows only private IPs | The node has no public IP, or `networking.hostNetwork` was disabled — it must stay `true` | | Pod stuck `Pending` | No node with capacity; the chart also keeps one gateway per node | diff --git a/integration/ethereum/docker-compose.yml b/integration/ethereum/docker-compose.yml index 4252c0c..a8f81b2 100644 --- a/integration/ethereum/docker-compose.yml +++ b/integration/ethereum/docker-compose.yml @@ -11,7 +11,6 @@ services: ports: - "33212:33212" # libp2p port (CL clients connect here) - "43213:43213" # mump2p agent port - - "43213:43213/udp" # mump2p agent QUIC port - "48123:48123" # Telemetry/metrics port volumes: - ./config:/app/config # Config directory (must contain app_conf.yml) diff --git a/integration/obol/compose-optimum.yml b/integration/obol/compose-optimum.yml index 3f9593b..648ae29 100644 --- a/integration/obol/compose-optimum.yml +++ b/integration/obol/compose-optimum.yml @@ -9,7 +9,6 @@ services: ports: - "${GATEWAY_PORT_LIBP2P:-33212}:33212" - "${GATEWAY_PORT_MUMP2P:-43213}:43213" - - "${GATEWAY_PORT_MUMP2P:-43213}:43213/udp" - "${GATEWAY_PORT_TELEMETRY:-48123}:48123" volumes: - ./optimum/config:/app/config diff --git a/pkg/service/mum_p2p/service.go b/pkg/service/mum_p2p/service.go index 7aeda65..201cf16 100644 --- a/pkg/service/mum_p2p/service.go +++ b/pkg/service/mum_p2p/service.go @@ -14,7 +14,6 @@ import ( "github.com/libp2p/go-libp2p/p2p/net/connmgr" "github.com/libp2p/go-libp2p/p2p/security/noise" libp2pquic "github.com/libp2p/go-libp2p/p2p/transport/quic" - "github.com/libp2p/go-libp2p/p2p/transport/quicreuse" "github.com/libp2p/go-libp2p/p2p/transport/tcp" gomplex "github.com/libp2p/go-mplex" "github.com/multiformats/go-multiaddr" @@ -97,7 +96,6 @@ func NewNode( libP2POpts := []libp2p.Option{ libp2p.ConnectionManager(cn), - libp2p.QUICReuse(quicreuse.NewConnManager), libp2p.ListenAddrStrings( fmt.Sprintf("/ip4/0.0.0.0/tcp/%d", cfg.ListenPort), fmt.Sprintf("/ip6/::/tcp/%d", cfg.ListenPort), diff --git a/pkg/service/mum_p2p/transport_test.go b/pkg/service/mum_p2p/transport_test.go index 1cbcd84..a9cb03a 100644 --- a/pkg/service/mum_p2p/transport_test.go +++ b/pkg/service/mum_p2p/transport_test.go @@ -22,75 +22,74 @@ import ( ) func TestNewNodeAcceptsInboundTCPAndQUIC(t *testing.T) { - // NewNode normally discovers public addresses. Keep this production - // constructor test deterministic while still exercising its host setup. + // NewNode calls GetExternalIPs for advertised addrs. Stub it so this + // only exercises listen + transport registration. oldGetExternalIPs := getExternalIPs getExternalIPs = func() (string, string, error) { return "127.0.0.1", "", nil } t.Cleanup(func() { getExternalIPs = oldGetExternalIPs }) - for _, tc := range []struct { - name string - transport string - addr string - dialOpts []libp2p.Option - }{ - { - name: "tcp", - transport: "tcp", - addr: "/ip4/127.0.0.1/tcp/%d/p2p/%s", - dialOpts: []libp2p.Option{libp2p.Transport(tcp.NewTCPTransport)}, - }, - { - name: "quic", - transport: "quic-v1", - addr: "/ip4/127.0.0.1/udp/%d/quic-v1/p2p/%s", - dialOpts: []libp2p.Option{libp2p.Transport(libp2pquic.NewTransport)}, - }, - } { - t.Run(tc.name, func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) - defer cancel() - log := commonlogger.NewAppSLogger(commonlogger.Error) - cfg := newTransportTestConfig(ctx, log, "transport-test-"+tc.name) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + log := commonlogger.NewAppSLogger(commonlogger.Error) + cfg := newTransportTestConfig(ctx, t, log) - target, err := NewNode(ctx, log, cfg, t.TempDir()) - require.NoError(t, err) - t.Cleanup(target.Stop) + target, err := NewNode(ctx, log, cfg, t.TempDir()) + require.NoError(t, err) + t.Cleanup(target.Stop) - dialOpts := append([]libp2p.Option{}, tc.dialOpts...) - dialer, err := libp2p.New(dialOpts...) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, dialer.Close()) }) + id := target.GetHost().ID() + port := cfg.ListenPort - addr := multiaddr.StringCast(fmt.Sprintf(tc.addr, cfg.ListenPort, target.GetHost().ID().String())) - require.NoError(t, dialer.Connect(ctx, peer.AddrInfo{ID: target.GetHost().ID(), Addrs: []multiaddr.Multiaddr{addr}})) + dial := func(t *testing.T, transport, addrFmt string, opts ...libp2p.Option) { + t.Helper() + dialer, err := libp2p.New(opts...) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, dialer.Close()) }) - require.Eventually(t, func() bool { - for _, conn := range target.GetHost().Network().ConnsToPeer(dialer.ID()) { - state := conn.ConnState() - if state.Transport == tc.transport && conn.Stat().Direction == network.DirInbound { - return true - } - } - return false - }, 5*time.Second, 25*time.Millisecond) - }) + addr := multiaddr.StringCast(fmt.Sprintf(addrFmt, port, id)) + require.NoError(t, dialer.Connect(ctx, peer.AddrInfo{ID: id, Addrs: []multiaddr.Multiaddr{addr}})) + + conns := target.GetHost().Network().ConnsToPeer(dialer.ID()) + require.NotEmpty(t, conns) + require.Equal(t, transport, conns[0].ConnState().Transport) + require.Equal(t, network.DirInbound, conns[0].Stat().Direction) } + + t.Run("tcp", func(t *testing.T) { + dial(t, "tcp", "/ip4/127.0.0.1/tcp/%d/p2p/%s", libp2p.Transport(tcp.NewTCPTransport)) + }) + t.Run("quic", func(t *testing.T) { + dial(t, "quic-v1", "/ip4/127.0.0.1/udp/%d/quic-v1/p2p/%s", libp2p.Transport(libp2pquic.NewTransport)) + }) } -func newTransportTestConfig(ctx context.Context, log commonlogger.AppLogger, clusterID string) *Config { - listener, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - panic(err) - } - port := listener.Addr().(*net.TCPAddr).Port - _ = listener.Close() +func newTransportTestConfig(ctx context.Context, t *testing.T, log commonlogger.AppLogger) *Config { + t.Helper() return &Config{ - ClusterID: clusterID, - ListenPort: port, + ClusterID: "transport-test", + ListenPort: freeTCPUDPPort(t), MaxMessageSize: cfgpkg.DefaultMaxMessageSize, Rotator: commonconfig.NewConfigRotator(ctx, log, &commonentities.OptimumConfig{ MaxMessageSize: cfgpkg.DefaultMaxMessageSize, - }, "hoodi", clusterID, func(*commonentities.DynamicConfig) {}), + }, "hoodi", "transport-test", func(*commonentities.DynamicConfig) {}), + } +} + +func freeTCPUDPPort(t *testing.T) int { + t.Helper() + for range 20 { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + port := ln.Addr().(*net.TCPAddr).Port + pc, err := net.ListenPacket("udp", fmt.Sprintf("127.0.0.1:%d", port)) + if err != nil { + _ = ln.Close() + continue + } + require.NoError(t, ln.Close()) + require.NoError(t, pc.Close()) + return port } + t.Fatal("could not allocate a port free on both TCP and UDP") + return 0 }