diff --git a/config/sample.app_conf.yml b/config/sample.app_conf.yml index ca93315..e25e835 100644 --- a/config/sample.app_conf.yml +++ b/config/sample.app_conf.yml @@ -56,6 +56,14 @@ telemetry_port: 48123 # chain: hoodi # OPT_DEV_CHAIN — overridden by JWT chain_id claim # gateway_id: dev-gateway # OPT_GATEWAY_ID — overridden by JWT sub claim +# ─── Announce IP (optional) ─────────────────────────────────────────────────── +# Override the public IPv4 address advertised to peers on both the CL-facing +# and mump2p hosts. Use when autodetection returns a private/pod-internal +# address peers can't route to (e.g. behind NAT or a CNI network) and running +# with a host-mode network namespace isn't an option. Equivalent to Prysm's +# --p2p-host-ip. +# announce_ip: 203.0.113.10 + # ─── Direct CL peers (optional) ────────────────────────────────────────────── # direct_cl_peers: # - /ip4/1.2.3.4/tcp/13000/p2p/16Uiu2... diff --git a/pkg/config/config.go b/pkg/config/config.go index 73409d5..2c7790f 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -42,14 +42,17 @@ type AppConfig struct { PProfAddr string `yaml:"pprof_addr" env:"OPT_PPROF_ADDR" default:"127.0.0.1:6060"` // mump2p trace-event categories consumed in-process for analysis (see handleMumP2PTrace). // All default false. TraceRPC is a high-frequency firehose — enable only for deep debugging. - TraceMesh bool `yaml:"trace_mesh" env:"OPT_TRACE_MESH" default:"false"` - TraceRPC bool `yaml:"trace_rpc" env:"OPT_TRACE_RPC" default:"false"` - TraceShard bool `yaml:"trace_shard" env:"OPT_TRACE_SHARD" default:"false"` - LogLevel string `yaml:"log_level" env:"OPT_LOG_LEVEL" default:"debug"` - IdentityLibP2PDir string `yaml:"identity_libp2p_dir" env:"OPT_IDENTITY_LIBP2P_DIR" default:"/tmp/libp2p"` - IdentityMumP2PDir string `yaml:"identity_mump2p_dir" env:"OPT_IDENTITY_MUMP2P_DIR" default:"/tmp/mump2p"` - AgentLibP2PPort int `yaml:"agent_lib_p2p_port" env:"OPT_AGENT_LIB_P2P_PORT" default:"33212"` - AgentMumP2PPort int `yaml:"agent_mump2p_port" env:"OPT_AGENT_MUMP2P_PORT" default:"33213"` + TraceMesh bool `yaml:"trace_mesh" env:"OPT_TRACE_MESH" default:"false"` + TraceRPC bool `yaml:"trace_rpc" env:"OPT_TRACE_RPC" default:"false"` + TraceShard bool `yaml:"trace_shard" env:"OPT_TRACE_SHARD" default:"false"` + LogLevel string `yaml:"log_level" env:"OPT_LOG_LEVEL" default:"debug"` + IdentityLibP2PDir string `yaml:"identity_libp2p_dir" env:"OPT_IDENTITY_LIBP2P_DIR" default:"/tmp/libp2p"` + IdentityMumP2PDir string `yaml:"identity_mump2p_dir" env:"OPT_IDENTITY_MUMP2P_DIR" default:"/tmp/mump2p"` + AgentLibP2PPort int `yaml:"agent_lib_p2p_port" env:"OPT_AGENT_LIB_P2P_PORT" default:"33212"` + AgentMumP2PPort int `yaml:"agent_mump2p_port" env:"OPT_AGENT_MUMP2P_PORT" default:"33213"` + // AnnounceIP, when set, always overrides the advertised public IPv4 (CL-facing + // and mump2p hosts); IPv6 autodetection still runs. Prysm's --p2p-host-ip equivalent. + AnnounceIP string `yaml:"announce_ip" env:"OPT_ANNOUNCE_IP"` DirectCLPeers []string `yaml:"direct_cl_peers" env:"OPT_DIRECT_CL_PEERS"` TelemetryEnable bool `yaml:"telemetry_enable" env:"OPT_ENABLE_TELEMETRY" default:"false"` TelemetryPort int `yaml:"telemetry_port" env:"OPT_TELEMETRY_PORT" default:"48123"` @@ -263,6 +266,16 @@ func (c *AppConfig) Validate() error { if c.GatewayClusterID == "" { return fmt.Errorf("OPT_GATEWAY_CLUSTER_ID is required") } + if c.AnnounceIP != "" { + ip := net.ParseIP(c.AnnounceIP) + v4 := ip.To4() + if ip == nil || v4 == nil { + return fmt.Errorf("OPT_ANNOUNCE_IP %q is not a valid IPv4 address", c.AnnounceIP) + } + // Normalize IPv4-mapped IPv6 text (e.g. "::ffff:1.2.3.4") to plain + // dotted-decimal; the raw form is invalid inside an /ip4/ multiaddr. + c.AnnounceIP = v4.String() + } if c.StreamEnable { if err := validateStreamListener("stream_addr", c.StreamAddr, c.StreamRequireAuth); err != nil { diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index d104879..15df9da 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -338,3 +338,106 @@ func TestStreamValidation(t *testing.T) { require.True(t, cfg.StreamOnly) }) } + +func TestValidate_AnnounceIP_Valid(t *testing.T) { + cfg := &config.AppConfig{ + IdentityLibP2PDir: testLibP2PDir, + IdentityMumP2PDir: testMumP2PDir, + AgentLibP2PPort: 33212, + AgentMumP2PPort: 33213, + TelemetryPort: 48123, + GatewayClusterID: "cluster", + AnnounceIP: "203.0.113.10", + } + require.NoError(t, cfg.Validate()) + require.Equal(t, "203.0.113.10", cfg.AnnounceIP) +} + +func TestValidate_AnnounceIP_Empty_NoOp(t *testing.T) { + // Not set at all - Validate should neither error nor touch the field. + cfg := &config.AppConfig{ + IdentityLibP2PDir: testLibP2PDir, + IdentityMumP2PDir: testMumP2PDir, + AgentLibP2PPort: 33212, + AgentMumP2PPort: 33213, + TelemetryPort: 48123, + GatewayClusterID: "cluster", + } + require.NoError(t, cfg.Validate()) + require.Empty(t, cfg.AnnounceIP) +} + +func TestValidate_AnnounceIP_Invalid(t *testing.T) { + cases := []string{ + "not-an-ip", + "999.999.999.999", + "", // handled separately by the empty-string no-op case, but a whitespace-only value is not empty and should still fail + "2001:db8::1", // a genuine IPv6 address, not IPv4 + "announce.example.com", // hostname, not an address + "203.0.113.10:8080", // host:port, not a bare address + } + for _, in := range cases { + if in == "" { + continue // covered by TestValidate_AnnounceIP_Empty_NoOp + } + cfg := &config.AppConfig{ + IdentityLibP2PDir: testLibP2PDir, + IdentityMumP2PDir: testMumP2PDir, + AgentLibP2PPort: 33212, + AgentMumP2PPort: 33213, + TelemetryPort: 48123, + GatewayClusterID: "cluster", + AnnounceIP: in, + } + err := cfg.Validate() + require.Errorf(t, err, "expected %q to be rejected as an invalid IPv4 address", in) + require.Contains(t, err.Error(), "OPT_ANNOUNCE_IP") + } +} + +// TestValidate_AnnounceIP_NormalizesIPv4MappedIPv6: To4() accepts IPv4-mapped +// IPv6 text too, so Validate must normalize it to dotted-decimal, not just check it. +func TestValidate_AnnounceIP_NormalizesIPv4MappedIPv6(t *testing.T) { + cfg := &config.AppConfig{ + IdentityLibP2PDir: testLibP2PDir, + IdentityMumP2PDir: testMumP2PDir, + AgentLibP2PPort: 33212, + AgentMumP2PPort: 33213, + TelemetryPort: 48123, + GatewayClusterID: "cluster", + AnnounceIP: "::ffff:203.0.113.10", + } + require.NoError(t, cfg.Validate()) + require.Equal(t, "203.0.113.10", cfg.AnnounceIP, + "AnnounceIP must be normalized to dotted-decimal, not left as IPv4-mapped IPv6 text") +} + +func TestLoadConfig_AnnounceIP_FromEnv(t *testing.T) { + dir := t.TempDir() + t.Setenv("OPT_IDENTITY_LIBP2P_DIR", filepath.Join(dir, "libid")) + t.Setenv("OPT_IDENTITY_MUMP2P_DIR", filepath.Join(dir, "mump2pid")) + t.Setenv("OPT_AGENT_LIB_P2P_PORT", "5000") + t.Setenv("OPT_AGENT_MUMP2P_PORT", "5001") + t.Setenv("OPT_GATEWAY_CLUSTER_ID", "gw-cluster") + t.Setenv("OPT_TELEMETRY_PORT", "8888") + t.Setenv("OPT_ANNOUNCE_IP", "198.51.100.7") + cfg, err := config.LoadConfig("") + require.NoError(t, err) + require.Equal(t, "198.51.100.7", cfg.AnnounceIP) +} + +func TestLoadConfig_AnnounceIP_FromYAML(t *testing.T) { + dir := t.TempDir() + path := writeTempConfig(t, ` +identity_libp2p_dir: `+filepath.Join(dir, "libid")+` +identity_mump2p_dir: `+filepath.Join(dir, "mump2pid")+` +agent_lib_p2p_port: 5000 +agent_mump2p_port: 5001 +gateway_cluster_id: gw-cluster +telemetry_port: 8888 +announce_ip: 203.0.113.55 +`) + cfg, err := config.LoadConfig(path) + require.NoError(t, err) + require.Equal(t, "203.0.113.55", cfg.AnnounceIP) +} diff --git a/pkg/service/bootstrapper/service.go b/pkg/service/bootstrapper/service.go index 5eb5eb4..88b3c2a 100644 --- a/pkg/service/bootstrapper/service.go +++ b/pkg/service/bootstrapper/service.go @@ -152,12 +152,33 @@ func (s *Service) RegisterAndGetMumP2PPeers() ([]string, error) { func (s *Service) predictMumP2PAddrInfo() (peerInfo peer.AddrInfo, publicIP string, err error) { publicIPV4, publicIPV6, err := commonnet.GetExternalIPs() if err != nil { - s.log.Error("failed to get public IP, falling back to interface IP", err) - publicIPV4, err = commonnet.ExternalIP() - if err != nil { - return peer.AddrInfo{}, "", fmt.Errorf("unable to get any IP address: %w", err) + if s.cfg.AnnounceIP == "" { + s.log.Error("failed to get public IP, falling back to interface IP", err) + publicIPV4, err = commonnet.ExternalIP() + if err != nil { + return peer.AddrInfo{}, "", fmt.Errorf("unable to get any IP address: %w", err) + } + } else { + // Autodetection (and its interface-inspection fallback) both + // only matter when we don't already have an operator-supplied + // address. Continue without IPv6 rather than failing startup. + s.log.Info("autodetection failed but announce_ip is configured, continuing without IPv6", + logger.WithString("autodetect_error", err.Error()), + ) + publicIPV6 = "" } } + if s.cfg.AnnounceIP != "" { + // Must match the override used when the mump2p host actually starts + // (setupMumP2PHost / mum_p2p.NewNode) - this prediction is registered + // with the bootstrap server ahead of time, and a mismatch here would + // have the gateway announce one address to bootstrap and then bind/ + // advertise a different one once the host is up. + s.log.Info("using configured announce_ip override for bootstrap prediction IPv4, autodetected IPv6 (if any) is kept", + logger.WithString("announce_ip", s.cfg.AnnounceIP), + ) + publicIPV4 = s.cfg.AnnounceIP + } s.log.Info("public IP address detected from prediction", logger.WithString("public_ip", publicIPV4), logger.WithString("public_ip_v6", publicIPV6), diff --git a/pkg/service/gossipsub-gateway/setup_libp2p_host.go b/pkg/service/gossipsub-gateway/setup_libp2p_host.go index 26f5cdd..ce91daf 100644 --- a/pkg/service/gossipsub-gateway/setup_libp2p_host.go +++ b/pkg/service/gossipsub-gateway/setup_libp2p_host.go @@ -91,9 +91,18 @@ func (s *Service) setupLibP2PHost() error { } listenAddrs := []multiaddr.Multiaddr{listenAddrIPv4, listenAddrIPv6} - publicIP, _, err := commonnet.GetExternalIPs() - if err != nil { - return fmt.Errorf("failed to get outbound IP address: %w", err) + var publicIP string + if s.cfg.AnnounceIP != "" { + s.log.Info("using configured announce_ip override for CL-facing host, skipping autodetection", + logger.WithString("announce_ip", s.cfg.AnnounceIP), + ) + publicIP = s.cfg.AnnounceIP + } else { + var err error + publicIP, _, err = commonnet.GetExternalIPs() + if err != nil { + return fmt.Errorf("failed to get outbound IP address: %w", err) + } } internalIPs, err := commonnet.GetPrivateIPs() diff --git a/pkg/service/gossipsub-gateway/setup_mump2p_host.go b/pkg/service/gossipsub-gateway/setup_mump2p_host.go index 77fe8dc..faea996 100644 --- a/pkg/service/gossipsub-gateway/setup_mump2p_host.go +++ b/pkg/service/gossipsub-gateway/setup_mump2p_host.go @@ -32,6 +32,7 @@ func (s *Service) setupMumP2PHost() error { MeshDegreeMax: int(config.DefaultMeshDegreeMax), BootstrapPeers: filtered, ClusterID: s.cfg.GatewayClusterID, + AnnounceIP: s.cfg.AnnounceIP, Rotator: s.cfg.GetDCRotator(), TraceMesh: s.cfg.TraceMesh, TraceRPC: s.cfg.TraceRPC, diff --git a/pkg/service/mum_p2p/config.go b/pkg/service/mum_p2p/config.go index b7059c2..3c1131a 100644 --- a/pkg/service/mum_p2p/config.go +++ b/pkg/service/mum_p2p/config.go @@ -17,6 +17,13 @@ type Config struct { ListenPort int `yaml:"listen_port"` MaxMessageSize int64 `yaml:"max_message_size_bytes"` + // AnnounceIP overrides the public IPv4 address advertised for this host, + // bypassing commonnet.GetExternalIPs() autodetection. Populated from + // AppConfig.AnnounceIP by the caller (see setupMumP2PHost); mirrors the + // same override used for the CL-facing libp2p host so both hosts agree + // on the advertised address. + AnnounceIP string `yaml:"announce_ip"` + // RLNC and message settings RandomMessageSize int64 `yaml:"random_message_size_bytes"` ShardFactor int `yaml:"rlnc_shard_factor"` diff --git a/pkg/service/mum_p2p/service.go b/pkg/service/mum_p2p/service.go index d2f4004..fc8f039 100644 --- a/pkg/service/mum_p2p/service.go +++ b/pkg/service/mum_p2p/service.go @@ -72,7 +72,22 @@ func NewNode( publicIPV4, publicIPV6, err := commonnet.GetExternalIPs() if err != nil { - return nil, fmt.Errorf("failed to get public IP address: %w", err) + if cfg.AnnounceIP == "" { + return nil, fmt.Errorf("failed to get public IP address: %w", err) + } + // Autodetection failed (no outbound internet, hermetic environment, + // etc.), but an explicit override is configured for IPv4 - proceed + // without IPv6 rather than failing startup entirely. + log.Info("autodetection failed but announce_ip is configured, continuing without IPv6", + logger.WithString("autodetect_error", err.Error()), + ) + publicIPV6 = "" + } + if cfg.AnnounceIP != "" { + log.Info("using configured announce_ip override for mump2p host IPv4, autodetected IPv6 (if any) is kept", + logger.WithString("announce_ip", cfg.AnnounceIP), + ) + publicIPV4 = cfg.AnnounceIP } log.Info("ip detected", logger.WithString("ipv4", publicIPV4), logger.WithString("ipv6", publicIPV6))