feat: update egress proxy secrets on running instances

cmd/api/api/instances.go

@@ -798,6 +798,57 @@ func (s *ApiService) StatInstancePath(ctx context.Context, request oapi.StatInst
 	return response, nil
 }
 
+// UpdateEgressSecrets updates egress proxy secret env values on a running instance
+// The id parameter can be an instance ID, name, or ID prefix
+// Note: Resolution is handled by ResolveResource middleware
+func (s *ApiService) UpdateEgressSecrets(ctx context.Context, request oapi.UpdateEgressSecretsRequestObject) (oapi.UpdateEgressSecretsResponseObject, error) {
+	inst := mw.GetResolvedInstance[instances.Instance](ctx)
+	if inst == nil {
+		return oapi.UpdateEgressSecrets500JSONResponse{
+			Code:    "internal_error",
+			Message: "resource not resolved",
+		}, nil
+	}
+	log := logger.FromContext(ctx)
+
+	if request.Body == nil {
+		return oapi.UpdateEgressSecrets400JSONResponse{
+			Code:    "invalid_request",
+			Message: "request body is required",
+		}, nil
+	}
+
+	env := make(map[string]string)
+	if request.Body.Env != nil {
+		env = request.Body.Env
+	}
+
+	result, err := s.InstanceManager.UpdateEgressSecrets(ctx, inst.Id, instances.UpdateEgressSecretsRequest{
+		Env: env,
+	})
+	if err != nil {
+		switch {
+		case errors.Is(err, instances.ErrInvalidState):
+			return oapi.UpdateEgressSecrets409JSONResponse{
+				Code:    "invalid_state",
+				Message: err.Error(),
+			}, nil
+		case errors.Is(err, instances.ErrInvalidRequest):
+			return oapi.UpdateEgressSecrets400JSONResponse{
+				Code:    "invalid_request",
+				Message: err.Error(),
+			}, nil
+		default:
+			log.ErrorContext(ctx, "failed to update egress secrets", "error", err)
+			return oapi.UpdateEgressSecrets500JSONResponse{
+				Code:    "internal_error",
+				Message: "failed to update egress secrets",
+			}, nil
+		}
+	}
+	return oapi.UpdateEgressSecrets200JSONResponse(instanceToOAPI(*result)), nil
+}
+
 // AttachVolume attaches a volume to an instance (not yet implemented)
 func (s *ApiService) AttachVolume(ctx context.Context, request oapi.AttachVolumeRequestObject) (oapi.AttachVolumeResponseObject, error) {
 	return oapi.AttachVolume500JSONResponse{

lib/builds/manager_test.go

@@ -163,6 +163,10 @@ func (m *mockInstanceManager) SetResourceValidator(v instances.ResourceValidator
 	// no-op for mock
 }
 
+func (m *mockInstanceManager) UpdateEgressSecrets(ctx context.Context, id string, req instances.UpdateEgressSecretsRequest) (*instances.Instance, error) {
+	return nil, nil
+}
+
 func (m *mockInstanceManager) GetVsockDialer(ctx context.Context, instanceID string) (hypervisor.VsockDialer, error) {
 	return nil, nil
 }

lib/egressproxy/service.go

@@ -217,6 +217,28 @@ func compileHeaderInjectRules(cfgRules []HeaderInjectRuleConfig) ([]headerInject
 	return out, nil
 }
 
+// UpdateHeaderInjectRules replaces the header injection rules for an already-registered
+// instance. Unlike RegisterInstance this does NOT re-apply iptables enforcement — it
+// only updates the in-memory policy used for MITM header rewriting. The instance must
+// have been previously registered via RegisterInstance.
+func (s *Service) UpdateHeaderInjectRules(instanceID string, rules []HeaderInjectRuleConfig) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	sourceIP, ok := s.sourceIPByInstance[instanceID]
+	if !ok {
+		return fmt.Errorf("instance %s not registered with egress proxy", instanceID)
+	}
+
+	compiled, err := compileHeaderInjectRules(rules)
+	if err != nil {
+		return err
+	}
+
+	s.policiesBySourceIP[sourceIP] = sourcePolicy{headerInjectRules: compiled}
+	return nil
+}
+
 func (s *Service) UnregisterInstance(_ context.Context, instanceID string) {
 	s.mu.Lock()
 	sourceIP, ok := s.sourceIPByInstance[instanceID]

lib/instances/egress_proxy_integration_test.go

@@ -128,6 +128,33 @@ func TestEgressProxyRewritesHTTPSHeaders(t *testing.T) {
 	require.Equal(t, 0, blockedExitCode, "curl output: %s", blockedOutput)
 	require.Equal(t, "", blockedOutput)
 
+	// --- Phase 2: Update egress proxy secrets (key rotation) ---
+	t.Log("Updating egress proxy secret to rotated key...")
+	updatedInst, err := manager.UpdateEgressSecrets(ctx, inst.Id, UpdateEgressSecretsRequest{
+		Env: map[string]string{
+			"OUTBOUND_OPENAI_KEY": "rotated-key-456",
+		},
+	})
+	require.NoError(t, err)
+	require.NotNil(t, updatedInst)
+
+	// Verify the guest still sees the mock value (unchanged)
+	envOutput2, envExitCode2, err := execCommand(ctx, inst, "sh", "-lc", "printf '%s' \"$OUTBOUND_OPENAI_KEY\"")
+	require.NoError(t, err)
+	require.Equal(t, 0, envExitCode2)
+	require.Equal(t, "mock-OUTBOUND_OPENAI_KEY", envOutput2, "guest should still see mock value after secret rotation")
+
+	// Verify the proxy now injects the rotated key
+	rotatedCmd := fmt.Sprintf(
+		"NO_PROXY= no_proxy= curl -k -sS https://%s:%s",
+		targetHost, targetPort,
+	)
+	rotatedOutput, rotatedExitCode, err := execCommand(ctx, inst, "sh", "-lc", rotatedCmd)
+	require.NoError(t, err)
+	require.Equal(t, 0, rotatedExitCode, "curl output: %s", rotatedOutput)
+	require.Contains(t, rotatedOutput, "Bearer rotated-key-456", "proxy should inject rotated key")
+	require.NotContains(t, rotatedOutput, "real-openai-key-123", "proxy should no longer inject old key")
+
 	require.NoError(t, manager.DeleteInstance(ctx, inst.Id))
 	deleted = true
 }

lib/instances/manager.go

@@ -41,6 +41,7 @@ type Manager interface {
 	StartInstance(ctx context.Context, id string, req StartInstanceRequest) (*Instance, error)
 	StreamInstanceLogs(ctx context.Context, id string, tail int, follow bool, source LogSource) (<-chan string, error)
 	RotateLogs(ctx context.Context, maxBytes int64, maxFiles int) error
+	UpdateEgressSecrets(ctx context.Context, id string, req UpdateEgressSecretsRequest) (*Instance, error)
 	AttachVolume(ctx context.Context, id string, volumeId string, req AttachVolumeRequest) (*Instance, error)
 	DetachVolume(ctx context.Context, id string, volumeId string) (*Instance, error)
 	// ListInstanceAllocations returns resource allocations for all instances.
@@ -441,6 +442,17 @@ func (m *manager) RotateLogs(ctx context.Context, maxBytes int64, maxFiles int)
 	return lastErr
 }
 
+// UpdateEgressSecrets updates the real credential values used by the egress proxy
+// for header injection on a running instance. This enables key rotation without
+// restarting the VM. The guest continues to see mock values — only the host-side
+// proxy rules are updated.
+func (m *manager) UpdateEgressSecrets(ctx context.Context, id string, req UpdateEgressSecretsRequest) (*Instance, error) {
+	lock := m.getInstanceLock(id)
+	lock.Lock()
+	defer lock.Unlock()
+	return m.updateEgressSecrets(ctx, id, req)
+}
+
 // AttachVolume attaches a volume to an instance (not yet implemented)
 func (m *manager) AttachVolume(ctx context.Context, id string, volumeId string, req AttachVolumeRequest) (*Instance, error) {
 	return nil, fmt.Errorf("attach volume not yet implemented")

lib/instances/types.go

@@ -267,6 +267,11 @@ type ForkSnapshotRequest struct {
 	TargetHypervisor hypervisor.Type // Optional, allowed only for Stopped snapshots
 }
 
+// UpdateEgressSecretsRequest is the domain request for updating egress proxy credential env values.
+type UpdateEgressSecretsRequest struct {
+	Env map[string]string // Map of env var names to new secret values
+}
+
 // AttachVolumeRequest is the domain request for attaching a volume (used for API compatibility)
 type AttachVolumeRequest struct {
 	MountPath string

lib/instances/update_egress_secrets.go

@@ -0,0 +1,89 @@
+package instances
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/kernel/hypeman/lib/logger"
+)
+
+// updateEgressSecrets performs the actual update of egress proxy secret env values.
+// Caller must hold the instance write lock.
+func (m *manager) updateEgressSecrets(ctx context.Context, id string, req UpdateEgressSecretsRequest) (*Instance, error) {
+	log := logger.FromContext(ctx)
+	log.InfoContext(ctx, "updating egress proxy secrets", "instance_id", id)
+
+	// 1. Load and validate instance state
+	meta, err := m.loadMetadata(id)
+	if err != nil {
+		return nil, err
+	}
+	inst := m.toInstance(ctx, meta)
+	stored := &meta.StoredMetadata
+
+	if inst.State != StateRunning && inst.State != StateInitializing {
+		return nil, fmt.Errorf("%w: instance must be running (current state: %s)", ErrInvalidState, inst.State)
+	}
+
+	// 2. Validate egress proxy is enabled
+	if stored.NetworkEgress == nil || !stored.NetworkEgress.Enabled {
+		return nil, fmt.Errorf("%w: egress proxy is not enabled on this instance", ErrInvalidState)
+	}
+	if len(stored.Credentials) == 0 {
+		return nil, fmt.Errorf("%w: no credential policies configured on this instance", ErrInvalidRequest)
+	}
+
+	// 3. Validate the request env vars
+	if len(req.Env) == 0 {
+		return nil, fmt.Errorf("%w: env must contain at least one entry", ErrInvalidRequest)
+	}
+
+	// Build set of env var names referenced by credential policies
+	referencedEnvVars := make(map[string]bool)
+	for _, policy := range stored.Credentials {
+		referencedEnvVars[policy.Source.Env] = true
+	}
+
+	for envName, envValue := range req.Env {
+		if !referencedEnvVars[envName] {
+			return nil, fmt.Errorf("%w: env var %q is not referenced by any credential policy", ErrInvalidRequest, envName)
+		}
+		if strings.TrimSpace(envValue) == "" {
+			return nil, fmt.Errorf("%w: env var %q must be non-empty", ErrInvalidRequest, envName)
+		}
+	}
+
+	// 4. Update the stored env values
+	for envName, envValue := range req.Env {
+		stored.Env[envName] = envValue
+	}
+
+	// 5. Rebuild and update the proxy inject rules
+	newRules := buildEgressProxyInjectRules(stored.NetworkEgress, stored.Credentials, stored.Env)
+
+	m.egressProxyMu.Lock()
+	svc := m.egressProxy
+	m.egressProxyMu.Unlock()
+
+	if svc == nil {
+		return nil, fmt.Errorf("egress proxy service is not running")
+	}
+
+	if err := svc.UpdateHeaderInjectRules(id, newRules); err != nil {
+		return nil, fmt.Errorf("update egress proxy rules: %w", err)
+	}
+
+	// 6. Persist updated metadata
+	meta = &metadata{StoredMetadata: *stored}
+	if err := m.saveMetadata(meta); err != nil {
+		log.ErrorContext(ctx, "failed to save metadata after egress secrets update", "instance_id", id, "error", err)
+		return nil, fmt.Errorf("save metadata: %w", err)
+	}
+
+	log.InfoContext(ctx, "egress proxy secrets updated", "instance_id", id, "updated_vars", len(req.Env))
+
+	// 7. Return current instance state
+	finalInst := m.toInstanceWithoutHydration(ctx, meta)
+	return &finalInst, nil
+}

lib/instances/update_egress_secrets_test.go

@@ -0,0 +1,65 @@
+package instances
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBuildEgressProxyInjectRules_AfterSecretUpdate(t *testing.T) {
+	t.Parallel()
+
+	egressPolicy := &NetworkEgressPolicy{Enabled: true}
+	credentials := map[string]CredentialPolicy{
+		"API_KEY": {
+			Source: CredentialSource{Env: "API_KEY"},
+			Inject: []CredentialInjectRule{
+				{
+					Hosts: []string{"api.example.com"},
+					As: CredentialInjectAs{
+						Header: "Authorization",
+						Format: "Bearer ${value}",
+					},
+				},
+			},
+		},
+	}
+
+	// Initial env
+	env := map[string]string{"API_KEY": "original-secret"}
+	rules := buildEgressProxyInjectRules(egressPolicy, credentials, env)
+	require.Len(t, rules, 1)
+	assert.Equal(t, "Bearer original-secret", rules[0].HeaderValue)
+
+	// After rotation
+	env["API_KEY"] = "rotated-secret"
+	rules = buildEgressProxyInjectRules(egressPolicy, credentials, env)
+	require.Len(t, rules, 1)
+	assert.Equal(t, "Bearer rotated-secret", rules[0].HeaderValue)
+}
+
+func TestValidateCredentialEnvBindings_AfterUpdate(t *testing.T) {
+	t.Parallel()
+
+	credentials := map[string]CredentialPolicy{
+		"API_KEY": {
+			Source: CredentialSource{Env: "API_KEY"},
+			Inject: []CredentialInjectRule{
+				{As: CredentialInjectAs{Header: "X-Key", Format: "${value}"}},
+			},
+		},
+	}
+
+	// Valid update
+	env := map[string]string{"API_KEY": "new-key"}
+	assert.NoError(t, validateCredentialEnvBindings(credentials, env))
+
+	// Empty value
+	env["API_KEY"] = "  "
+	assert.Error(t, validateCredentialEnvBindings(credentials, env))
+
+	// Missing key
+	delete(env, "API_KEY")
+	assert.Error(t, validateCredentialEnvBindings(credentials, env))
+}