diff --git a/cli/cmd/shell.go b/cli/cmd/shell.go index 468b5f0..3131fb6 100644 --- a/cli/cmd/shell.go +++ b/cli/cmd/shell.go @@ -45,6 +45,10 @@ func runShell(cmd *cobra.Command, args []string) error { if resp != nil && resp.StatusCode == http.StatusNotImplemented { return fmt.Errorf("interactive shells are not supported on this sandbox's isolation backend") } + return fmt.Errorf("failed to connect: %w", err) + } + if conn == nil { + return fmt.Errorf("websocket dial returned no connection and no error — this should not happen") } defer conn.CloseNow() diff --git a/go.mod b/go.mod index c1614fb..5552496 100644 --- a/go.mod +++ b/go.mod @@ -6,12 +6,14 @@ require ( github.com/alicebob/miniredis/v2 v2.38.0 github.com/coder/websocket v1.8.15 github.com/containerd/errdefs v1.0.0 + github.com/creack/pty v1.1.24 github.com/docker/docker v28.5.2+incompatible github.com/go-chi/chi/v5 v5.3.0 github.com/golang-migrate/migrate/v4 v4.19.1 github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.10.0 github.com/joho/godotenv v1.5.1 + github.com/mdlayher/vsock v1.3.0 github.com/opencontainers/image-spec v1.1.1 github.com/prometheus/client_golang v1.24.0 github.com/redis/go-redis/v9 v9.21.0 @@ -46,6 +48,7 @@ require ( github.com/lib/pq v1.10.9 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.10 // indirect + github.com/mdlayher/socket v0.6.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.2.0 // indirect github.com/moby/moby/api v1.54.2 // indirect @@ -80,7 +83,8 @@ require ( go.opentelemetry.io/otel/metric v1.44.0 // indirect go.opentelemetry.io/otel/trace v1.44.0 // indirect go.uber.org/atomic v1.11.0 // indirect - golang.org/x/crypto v0.51.0 // indirect + golang.org/x/crypto v0.53.0 // indirect + golang.org/x/net v0.56.0 // indirect golang.org/x/sync v0.21.0 // indirect golang.org/x/sys v0.47.0 // indirect golang.org/x/term v0.45.0 // indirect diff --git a/go.sum b/go.sum index ee619a5..f737610 100644 --- a/go.sum +++ b/go.sum @@ -97,6 +97,10 @@ github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8S github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI= github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o= +github.com/mdlayher/socket v0.6.0 h1:ScZPaAGyO1icQnbFrhPM8mnXyMu9qukC1K4ZoM2IQKU= +github.com/mdlayher/socket v0.6.0/go.mod h1:q7vozUAnxSqnjHc12Fik5yUKIzfZ8ITCfMkhOtE9z18= +github.com/mdlayher/vsock v1.3.0 h1:bqQfZ1OznI03y6YiXp2sze05RVdzLn/zsfjnjd4+ivI= +github.com/mdlayher/vsock v1.3.0/go.mod h1:WsuksavOvwCnV5UqGHUkvAvCy+Dqy81y4goKQTzxxNY= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= @@ -195,8 +199,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= -golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= -golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= diff --git a/guest-agent/guest-agent b/guest-agent/guest-agent new file mode 100644 index 0000000..65d066e Binary files /dev/null and b/guest-agent/guest-agent differ diff --git a/guest-agent/main.go b/guest-agent/main.go new file mode 100644 index 0000000..b43790a --- /dev/null +++ b/guest-agent/main.go @@ -0,0 +1,259 @@ +package main + +import ( + "bufio" + "bytes" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "log" + "os" + "os/exec" + + "github.com/creack/pty" + "github.com/mdlayher/vsock" +) + +const ( + agentPort = 52000 // JSON control protocol — exec/read_file/write_file + shellPort = 52001 // raw PTY streaming — interactive shell sessions + + frameData byte = 0x01 + frameResize byte = 0x02 +) + +// --- JSON control protocol (exec, read_file, write_file) --- + +type Request struct { + Type string `json:"type"` + Cmd []string `json:"cmd,omitempty"` + Path string `json:"path,omitempty"` + Content string `json:"content,omitempty"` +} + +type Response struct { + Stdout string `json:"stdout,omitempty"` + Stderr string `json:"stderr,omitempty"` + ExitCode int `json:"exit_code,omitempty"` + Content string `json:"content,omitempty"` + Error string `json:"error,omitempty"` +} + +func main() { + go startShellListener() + + l, err := vsock.Listen(agentPort, nil) + if err != nil { + log.Fatalf("failed to listen on vsock port %d: %v", agentPort, err) + } + log.Printf("guest-agent listening on vsock port %d", agentPort) + + for { + conn, err := l.Accept() + if err != nil { + log.Printf("accept error: %v", err) + continue + } + go handleConn(conn) + } +} + +func handleConn(conn io.ReadWriteCloser) { + defer func() { + _ = conn.Close() + }() + + scanner := bufio.NewScanner(conn) + scanner.Buffer(make([]byte, 1024*1024), 10*1024*1024) + + for scanner.Scan() { + var req Request + if err := json.Unmarshal(scanner.Bytes(), &req); err != nil { + writeResp(conn, Response{Error: "invalid request: " + err.Error()}) + continue + } + writeResp(conn, handle(req)) + } +} + +func handle(req Request) Response { + switch req.Type { + case "exec": + return doExec(req.Cmd) + case "write_file": + return doWriteFile(req.Path, req.Content) + case "read_file": + return doReadFile(req.Path) + default: + return Response{Error: "unknown request type: " + req.Type} + } +} + +func doExec(cmdArgs []string) Response { + if len(cmdArgs) == 0 { + return Response{Error: "cmd is required"} + } + cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + exitCode := 0 + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + exitCode = exitErr.ExitCode() + } else { + return Response{Error: err.Error()} + } + } + return Response{Stdout: stdout.String(), Stderr: stderr.String(), ExitCode: exitCode} +} + +func doWriteFile(path, content string) Response { + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + return Response{Error: err.Error()} + } + return Response{} +} + +func doReadFile(path string) Response { + data, err := os.ReadFile(path) + if err != nil { + return Response{Error: err.Error()} + } + return Response{Content: string(data)} +} + +func writeResp(w io.Writer, resp Response) { + b, _ := json.Marshal(resp) + if _, err := fmt.Fprintf(w, "%s\n", b); err != nil { + log.Printf("failed to write response: %v", err) + } +} + +// --- Raw PTY shell protocol --- + +// startShellListener accepts persistent shell connections on a port +// separate from the JSON control protocol — a PTY session is a raw, +// continuous byte stream, not a request/response exchange, so it needs +// its own framing rather than being shoehorned into the JSON protocol. +func startShellListener() { + l, err := vsock.Listen(shellPort, nil) + if err != nil { + log.Fatalf("failed to listen on vsock shell port %d: %v", shellPort, err) + } + log.Printf("guest-agent shell listener on vsock port %d", shellPort) + + for { + conn, err := l.Accept() + if err != nil { + log.Printf("shell accept error: %v", err) + continue + } + go handleShellConn(conn) + } +} + +func handleShellConn(conn io.ReadWriteCloser) { + defer func() { + _ = conn.Close() + }() + + shellPath := "/bin/bash" + if _, err := os.Stat(shellPath); err != nil { + shellPath = "/bin/sh" + } + + cmd := exec.Command(shellPath) + cmd.Env = append(os.Environ(), "TERM=xterm-256color") + ptmx, err := pty.Start(cmd) + if err != nil { + log.Printf("failed to start pty: %v", err) + return + } + defer func() { + _ = ptmx.Close() + if cmd.Process != nil { + _ = cmd.Process.Kill() + _, _ = cmd.Process.Wait() + } + }() + + done := make(chan struct{}) + + // ptmx -> conn: PTY output streamed back to the host as data frames. + go func() { + defer close(done) + buf := make([]byte, 4096) + for { + n, err := ptmx.Read(buf) + if n > 0 { + if werr := writeFrame(conn, frameData, buf[:n]); werr != nil { + return + } + } + if err != nil { + return + } + } + }() + + // conn -> ptmx: host keystrokes (data frames) and terminal resize + // events (resize frames) arriving on the same stream. + for { + typ, payload, err := readFrame(conn) + if err != nil { + break + } + switch typ { + case frameData: + _, _ = ptmx.Write(payload) + case frameResize: + if len(payload) == 4 { + cols := binary.BigEndian.Uint16(payload[0:2]) + rows := binary.BigEndian.Uint16(payload[2:4]) + _ = pty.Setsize(ptmx, &pty.Winsize{Cols: cols, Rows: rows}) + } + } + } + + <-done +} + +// writeFrame/readFrame implement a minimal length-prefixed framing: +// [1 byte type][4 byte big-endian length][payload]. Needed because raw +// vsock is just an undifferentiated byte stream — without framing, there +// would be no way to distinguish "this is PTY output" from "this is a +// resize command" on the wire. +func writeFrame(w io.Writer, typ byte, payload []byte) error { + header := make([]byte, 5) + header[0] = typ + binary.BigEndian.PutUint32(header[1:], uint32(len(payload))) + if _, err := w.Write(header); err != nil { + return err + } + if len(payload) > 0 { + if _, err := w.Write(payload); err != nil { + return err + } + } + return nil +} + +func readFrame(r io.Reader) (byte, []byte, error) { + header := make([]byte, 5) + if _, err := io.ReadFull(r, header); err != nil { + return 0, nil, err + } + typ := header[0] + length := binary.BigEndian.Uint32(header[1:]) + payload := make([]byte, length) + if length > 0 { + if _, err := io.ReadFull(r, payload); err != nil { + return 0, nil, err + } + } + return typ, payload, nil +} diff --git a/internal/api/shell.go b/internal/api/shell.go index 8d25adc..7be3552 100644 --- a/internal/api/shell.go +++ b/internal/api/shell.go @@ -41,6 +41,7 @@ func (a *API) OpenShellSession(w http.ResponseWriter, r *http.Request) { sb, err := a.store.Get(r.Context(), id) if err != nil { + slog.Error("shell: failed to get sandbox", "sandbox_id", id, "error", err) http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -55,6 +56,7 @@ func (a *API) OpenShellSession(w http.ResponseWriter, r *http.Request) { shell, err := interactive.OpenShell(r.Context(), sb.ID) if err != nil { + slog.Error("shell: failed to open backend shell", "sandbox_id", sb.ID, "error", err) http.Error(w, err.Error(), http.StatusInternalServerError) return } diff --git a/internal/firecracker/interfaces.go b/internal/firecracker/interfaces.go index 0d5e453..57fdaa9 100644 --- a/internal/firecracker/interfaces.go +++ b/internal/firecracker/interfaces.go @@ -3,6 +3,8 @@ package firecracker import ( "context" "time" + + "github.com/harshalvk/cage/internal/backend" ) // fcAPI is the subset of apiClient's methods FirecrackerManager depends on @@ -33,3 +35,8 @@ type processHandle interface { Kill() error Wait() error } + +// shellDialerFunc abstracts opening a persistent shell connection - +// injectable the same way fcAPI/fcVsock are, so tests can fake it without +// real vsock/KVM +type shellDialerFunc func(udsPath string) (backend.Shell, error) diff --git a/internal/firecracker/manager.go b/internal/firecracker/manager.go index 5402982..fdec3f6 100644 --- a/internal/firecracker/manager.go +++ b/internal/firecracker/manager.go @@ -10,6 +10,8 @@ import ( "path/filepath" "sync" "time" + + "github.com/harshalvk/cage/internal/backend" ) // pauseManifest is persisted as json alongside a snapshot's files, so @@ -51,9 +53,10 @@ type FirecrackerManager struct { // Injectable dependencies - default to real implementations in // NewFirecrackerManager, overridden by tests via the unexporeted // constructor NewFirecracekrManagerForTest - spawnProcess func(ctx context.Context, bin, apiSocket string) (processHandle, error) + spawnProcess func(bin, apiSocket string) (processHandle, error) newAPI func(socketPath string) fcAPI newVsock func(udsPath string) fcVsock + newShell shellDialerFunc mu sync.Mutex running map[string]*vmInstance // sandboxID -> instance @@ -78,7 +81,14 @@ func NewFirecrackerManager(cfg Config) (*FirecrackerManager, error) { spawnProcess: realSpawnProcess, newAPI: func(socketPath string) fcAPI { return newAPIClient(socketPath) }, newVsock: func(udsPath string) fcVsock { return newVsockClient(udsPath) }, - running: make(map[string]*vmInstance), + newShell: func(udsPath string) (backend.Shell, error) { + conn, err := dialShellConn(udsPath) + if err != nil { + return nil, err + } + return &FirecrackerShell{conn: conn}, nil + }, + running: make(map[string]*vmInstance), }, nil } @@ -88,9 +98,10 @@ func NewFirecrackerManager(cfg Config) (*FirecrackerManager, error) { func NewFirecrackerManagerForTest( cfg Config, rootfs *RootfsManager, - spawnProcess func(ctx context.Context, bin, apiSocket string) (processHandle, error), + spawnProcess func(bin, apiSocket string) (processHandle, error), newAPI func(socketPath string) fcAPI, newVsock func(udsPath string) fcVsock, + newShell shellDialerFunc, ) *FirecrackerManager { return &FirecrackerManager{ cfg: cfg, @@ -98,12 +109,18 @@ func NewFirecrackerManagerForTest( spawnProcess: spawnProcess, newAPI: newAPI, newVsock: newVsock, + newShell: newShell, running: make(map[string]*vmInstance), } } -func realSpawnProcess(ctx context.Context, bin, apiSocket string) (processHandle, error) { - cmd := exec.CommandContext(ctx, bin, "--api-sock", apiSocket) +func realSpawnProcess(bin, apiSocket string) (processHandle, error) { + cmd := exec.Command(bin, "--api-sock", apiSocket) + logPath := apiSocket + ".log" + if logFile, err := os.Create(logPath); err == nil { + cmd.Stdout = logFile + cmd.Stderr = logFile + } if err := cmd.Start(); err != nil { return nil, err } @@ -145,7 +162,7 @@ func (m *FirecrackerManager) CreateSandbox(ctx context.Context, sandboxID, templ removeFileIfExists(apiSocket) removeFileIfExists(vsockUDS) - process, err := m.spawnProcess(ctx, m.cfg.FirecrackerBin, apiSocket) // ← was: exec.CommandContext(...) + cmd.Start() + process, err := m.spawnProcess(m.cfg.FirecrackerBin, apiSocket) // ← was: exec.CommandContext(...) + cmd.Start() if err != nil { if cerr := m.rootfs.Cleanup(sandboxID); cerr != nil { slog.Warn("firecracker: rootfs cleanup failed after spawn failure", "error", cerr) @@ -203,9 +220,38 @@ func (m *FirecrackerManager) CreateSandbox(ctx context.Context, sandboxID, templ } m.mu.Unlock() + go m.watchProcessExit(sandboxID, process) + return nil } +// watchProcessExit blocks until the VM's process exits (whether from a +// clean KillSandbox call, a crash, or a kernel panic) and reaps it, then +// removes any stale entry from m.running. Without this, an unexpectedly +// dead process becomes a zombie at the OS level AND a phantom "running" +// sandbox at the application level — exactly the failure this fixes. +func (m *FirecrackerManager) watchProcessExit(sandboxID string, process processHandle) { + err := process.Wait() + + m.mu.Lock() + inst, stillTracked := m.running[sandboxID] + if stillTracked && inst.process == process { + delete(m.running, sandboxID) + } + m.mu.Unlock() + + if stillTracked { + if err != nil { + slog.Warn("firecracker: vm process exited unexpectedly", "sandbox_id", sandboxID, "error", err) + } else { + slog.Warn("firecracker: vm process exited unexpectedly (no error)", "sandbox_id", sandboxID) + } + } + // If !stillTracked, this exit was expected (KillSandbox or PauseSandbox + // already removed the entry and will/did call Wait() themselves) — + // nothing to log or clean up here. +} + func (m *FirecrackerManager) KillSandbox(ctx context.Context, sandboxID string) error { m.mu.Lock() inst, ok := m.running[sandboxID] @@ -222,9 +268,9 @@ func (m *FirecrackerManager) KillSandbox(ctx context.Context, sandboxID string) if err := inst.process.Kill(); err != nil { slog.Warn("firecracker: failed to kill process", "sandbox_id", sandboxID, "error", err) } - if err := inst.process.Wait(); err != nil { - slog.Warn("firecracker: failed to reap process", "sandbox_id", sandboxID, "error", err) - } + // if err := inst.process.Wait(); err != nil { + // slog.Warn("firecracker: failed to reap process", "sandbox_id", sandboxID, "error", err) + // } } removeFileIfExists(inst.apiSocket) removeFileIfExists(inst.vsockUDS) @@ -321,7 +367,7 @@ func (m *FirecrackerManager) PauseSandbox(ctx context.Context, sandboxID string) api := m.newAPI(inst.apiSocket) if err := api.pauseVM(ctx); err != nil { - if rerr := os.RemoveAll(pauseDir); err != nil { + if rerr := os.RemoveAll(pauseDir); rerr != nil { slog.Warn("firecracker: failed to clean up pause dir after error", "error", rerr) } return "", fmt.Errorf("failed to pause vm: %w", err) @@ -355,9 +401,9 @@ func (m *FirecrackerManager) PauseSandbox(ctx context.Context, sandboxID string) if err := inst.process.Kill(); err != nil { slog.Warn("firecracker: failed to kill process after snapshot", "sandbox_id", sandboxID, "error", err) } - if err := inst.process.Wait(); err != nil { - slog.Warn("firecracker: failed to reap process after snapshot", "sandbox_id", sandboxID, "error", err) - } + // if err := inst.process.Wait(); err != nil { + // slog.Warn("firecracker: failed to reap process after snapshot", "sandbox_id", sandboxID, "error", err) + // } removeFileIfExists(inst.apiSocket) removeFileIfExists(inst.vsockUDS) @@ -390,7 +436,7 @@ func (m *FirecrackerManager) ResumeSandbox(ctx context.Context, sandboxID, pause removeFileIfExists(apiSocket) removeFileIfExists(vsockUDS) - process, err := m.spawnProcess(ctx, m.cfg.FirecrackerBin, apiSocket) + process, err := m.spawnProcess(m.cfg.FirecrackerBin, apiSocket) if err != nil { return fmt.Errorf("failed to start firecracker process for resume: %w", err) } @@ -500,3 +546,11 @@ func (m *FirecrackerManager) AdoptWarmResource(ctx context.Context, sandboxID, p m.running[sandboxID] = inst return nil } + +func (m *FirecrackerManager) OpenShell(ctx context.Context, sandboxID string) (backend.Shell, error) { + inst, err := m.get(sandboxID) + if err != nil { + return nil, err + } + return m.newShell(inst.vsockUDS) +} diff --git a/internal/firecracker/manager_test.go b/internal/firecracker/manager_test.go index a0e6f1f..1efbbf1 100644 --- a/internal/firecracker/manager_test.go +++ b/internal/firecracker/manager_test.go @@ -3,11 +3,13 @@ package firecracker import ( "context" "errors" + "io" "os" "path/filepath" "testing" "time" + "github.com/harshalvk/cage/internal/backend" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -26,6 +28,30 @@ type fakeAPI struct { failAt string // which method should return an error, "" for none } +type fakeShell struct { + written [][]byte + resized [2]uint16 + closed bool +} + +func (f *fakeShell) Read(p []byte) (int, error) { + msg := []byte("fake shell output") + n := copy(p, msg) + return n, io.EOF // one read, then EOF — enough to prove the plumbing works +} +func (f *fakeShell) Write(p []byte) (int, error) { + f.written = append(f.written, append([]byte(nil), p...)) + return len(p), nil +} +func (f *fakeShell) Resize(cols, rows uint16) error { + f.resized = [2]uint16{cols, rows} + return nil +} +func (f *fakeShell) Close() error { + f.closed = true + return nil +} + func (f *fakeAPI) setMachineConfig(ctx context.Context, vcpu, mem int64) error { if f.failAt == "machine-config" { return errors.New("boom") @@ -110,7 +136,7 @@ func newTestManager(t *testing.T, apiFailAt string, vsockErr error) *Firecracker rootfs, err := NewRootfsManager(baseDir, filepath.Join(runDir, "rootfs")) require.NoError(t, err) - spawnProcess := func(ctx context.Context, bin, apiSocket string) (processHandle, error) { + spawnProcess := func(bin, apiSocket string) (processHandle, error) { // Simulate the api socket "appearing" the way a real firecracker // process would, since waitForSocket polls for the file on disk. require.NoError(t, os.WriteFile(apiSocket, []byte{}, 0644)) @@ -123,6 +149,7 @@ func newTestManager(t *testing.T, apiFailAt string, vsockErr error) *Firecracker spawnProcess, func(socketPath string) fcAPI { return &fakeAPI{failAt: apiFailAt} }, func(udsPath string) fcVsock { return &fakeVsock{readyErr: vsockErr} }, + func(udsPath string) (backend.Shell, error) { return &fakeShell{}, nil }, ) } @@ -346,3 +373,31 @@ func TestRemoveImage_NonexistentRefIsNotAnError(t *testing.T) { err := mgr.RemoveImage(context.Background(), filepath.Join(mgr.cfg.RunDir, "pauses", "never-existed")) assert.NoError(t, err, "removing an already-gone pause ref should be a no-op, not an error") } + +func TestOpenShell_Success(t *testing.T) { + mgr := newTestManager(t, "", nil) + require.NoError(t, mgr.CreateSandbox(context.Background(), "sb-1", "base")) + + shell, err := mgr.OpenShell(context.Background(), "sb-1") + require.NoError(t, err) + require.NotNil(t, shell) + + n, err := shell.Write([]byte("echo hi\n")) + require.NoError(t, err) + assert.Equal(t, 8, n) + + require.NoError(t, shell.Resize(80, 24)) + + buf := make([]byte, 64) + n, _ = shell.Read(buf) + assert.Contains(t, string(buf[:n]), "fake shell output") + + require.NoError(t, shell.Close()) +} + +func TestOpenShell_UnknownSandbox(t *testing.T) { + mgr := newTestManager(t, "", nil) + + _, err := mgr.OpenShell(context.Background(), "does-not-exist") + assert.Error(t, err) +} diff --git a/internal/firecracker/shell.go b/internal/firecracker/shell.go new file mode 100644 index 0000000..202a38b --- /dev/null +++ b/internal/firecracker/shell.go @@ -0,0 +1,86 @@ +package firecracker + +import ( + "encoding/binary" + "fmt" + "io" + "net" +) + +// FirecrackerShell wraps a persistent vsock connection to the guest +// agent's shell listener, applying the same length-prefixed framing the +// guest speaks +type FirecrackerShell struct { + conn net.Conn +} + +// Read blocks until a data frame arrives, skipping any unexpected +// non-data frames rather than erroring the whole session over one +// malformed frame +func (s *FirecrackerShell) Read(p []byte) (int, error) { + for { + typ, payload, err := readFrame(s.conn) + if err != nil { + return 0, err + } + if typ == frameData { + return copy(p, payload), nil + } + } +} + +func (s *FirecrackerShell) Write(p []byte) (int, error) { + if err := writeFrame(s.conn, frameData, p); err != nil { + return 0, err + } + return len(p), nil +} + +func (s *FirecrackerShell) Resize(cols, rows uint16) error { + payload := make([]byte, 4) + binary.BigEndian.PutUint16(payload[0:2], cols) + binary.BigEndian.PutUint16(payload[2:4], rows) + return writeFrame(s.conn, frameResize, payload) +} + +func (s *FirecrackerShell) Close() error { + return s.conn.Close() +} + +// frame type constants and writeFrame/readFrame mirror guest-agent's +// protocol exactly -- keep both sides in sync if this ever changes +const ( + frameData byte = 0x01 + frameResize byte = 0x02 +) + +func writeFrame(w io.Writer, typ byte, payload []byte) error { + header := make([]byte, 5) + header[0] = typ + binary.BigEndian.PutUint32(header[1:], uint32(len(payload))) + if _, err := w.Write(header); err != nil { + return fmt.Errorf("failed to write frame header: %w", err) + } + if len(payload) > 0 { + if _, err := w.Write(payload); err != nil { + return fmt.Errorf("failed to write frame payload: %w", err) + } + } + return nil +} + +func readFrame(r io.Reader) (byte, []byte, error) { + header := make([]byte, 5) + if _, err := io.ReadFull(r, header); err != nil { + return 0, nil, err + } + typ := header[0] + length := binary.BigEndian.Uint32(header[1:]) + payload := make([]byte, length) + if length > 0 { + if _, err := io.ReadFull(r, payload); err != nil { + return 0, nil, err + } + } + return typ, payload, nil +} diff --git a/internal/firecracker/vsock.go b/internal/firecracker/vsock.go index b4c4b2d..531cd36 100644 --- a/internal/firecracker/vsock.go +++ b/internal/firecracker/vsock.go @@ -4,12 +4,14 @@ import ( "bufio" "encoding/json" "fmt" + "log/slog" "net" "strings" "time" ) const guestAgentPort = 52000 +const shellVsockPort = 52001 // must match guest-agent's shellPort // agentRequest/agentResponse mirror the guest agent's protocol exactly — // keep these in sync with guest-agent/main.go's Request/Response types. @@ -109,3 +111,50 @@ func (c *vsockClient) waitReady(timeout time.Duration) error { } return fmt.Errorf("guest agent did not become ready within %s: %w", timeout, lastErr) } + +// dialShellConn performs the vsock CONNECT handshake against the guest +// agent's shell port and returns the raw connection for continued, +// persistent, full-duplex streaming - unlike vsockClient.send, which +// dials, does one round trip, and closes +func dialShellConn(udsPath string) (net.Conn, error) { + conn, err := net.DialTimeout("unix", udsPath, 5*time.Second) + if err != nil { + return nil, fmt.Errorf("failed to dial vsock uds: %w", err) + } + + if _, err := fmt.Fprintf(conn, "CONNECT %d\n", shellVsockPort); err != nil { + if cerr := conn.Close(); cerr != nil { + slog.Warn("dialShellConn: failed to close connection after write error", "error", cerr) + } + return nil, fmt.Errorf("failed to send vsock connect: %w", err) + } + + reader := bufio.NewReader(conn) + handshake, err := reader.ReadString('\n') + if err != nil { + if cerr := conn.Close(); cerr != nil { + slog.Warn("dialShellConn: failed to close connection after handshake read error", "error", cerr) + } + return nil, fmt.Errorf("failed to read vsock handshake: %w", err) + } + if !strings.HasPrefix(handshake, "OK") { + if cerr := conn.Close(); cerr != nil { + slog.Warn("dialShellConn: failed to close connection after failed handshake", "error", cerr) + } + return nil, fmt.Errorf("vsock handshake failed: %s", strings.TrimSpace(handshake)) + } + + return &bufferedConn{Conn: conn, r: reader}, nil +} + +// bufferedConn ensures reads after the handshake correctly return any +// data bufio.Reader already pulled of the wire, before failing through +// to further raw reads from the underlying connection +type bufferedConn struct { + net.Conn + r *bufio.Reader +} + +func (b *bufferedConn) Read(p []byte) (int, error) { + return b.r.Read(p) +}