feat: add Firecracker networking (bridge/NAT/DNS) and interactive PTY shell sessions

cli/cmd/shell.go

@@ -45,6 +45,10 @@ func runShell(cmd *cobra.Command, args []string) error {
 		if resp != nil && resp.StatusCode == http.StatusNotImplemented {
 			return fmt.Errorf("interactive shells are not supported on this sandbox's isolation backend")
 		}
+		return fmt.Errorf("failed to connect: %w", err)
+	}
+	if conn == nil {
+		return fmt.Errorf("websocket dial returned no connection and no error — this should not happen")
 	}
 	defer conn.CloseNow()
 

go.mod

@@ -6,12 +6,14 @@ require (
 	github.com/alicebob/miniredis/v2 v2.38.0
 	github.com/coder/websocket v1.8.15
 	github.com/containerd/errdefs v1.0.0
+	github.com/creack/pty v1.1.24
 	github.com/docker/docker v28.5.2+incompatible
 	github.com/go-chi/chi/v5 v5.3.0
 	github.com/golang-migrate/migrate/v4 v4.19.1
 	github.com/google/uuid v1.6.0
 	github.com/jackc/pgx/v5 v5.10.0
 	github.com/joho/godotenv v1.5.1
+	github.com/mdlayher/vsock v1.3.0
 	github.com/opencontainers/image-spec v1.1.1
 	github.com/prometheus/client_golang v1.24.0
 	github.com/redis/go-redis/v9 v9.21.0
@@ -46,6 +48,7 @@ require (
 	github.com/lib/pq v1.10.9 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/magiconair/properties v1.8.10 // indirect
+	github.com/mdlayher/socket v0.6.0 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
 	github.com/moby/go-archive v0.2.0 // indirect
 	github.com/moby/moby/api v1.54.2 // indirect
@@ -80,7 +83,8 @@ require (
 	go.opentelemetry.io/otel/metric v1.44.0 // indirect
 	go.opentelemetry.io/otel/trace v1.44.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
-	golang.org/x/crypto v0.51.0 // indirect
+	golang.org/x/crypto v0.53.0 // indirect
+	golang.org/x/net v0.56.0 // indirect
 	golang.org/x/sync v0.21.0 // indirect
 	golang.org/x/sys v0.47.0 // indirect
 	golang.org/x/term v0.45.0 // indirect

go.sum

@@ -97,6 +97,10 @@ github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8S
 github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
 github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI=
 github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o=
+github.com/mdlayher/socket v0.6.0 h1:ScZPaAGyO1icQnbFrhPM8mnXyMu9qukC1K4ZoM2IQKU=
+github.com/mdlayher/socket v0.6.0/go.mod h1:q7vozUAnxSqnjHc12Fik5yUKIzfZ8ITCfMkhOtE9z18=
+github.com/mdlayher/vsock v1.3.0 h1:bqQfZ1OznI03y6YiXp2sze05RVdzLn/zsfjnjd4+ivI=
+github.com/mdlayher/vsock v1.3.0/go.mod h1:WsuksavOvwCnV5UqGHUkvAvCy+Dqy81y4goKQTzxxNY=
 github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
 github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
 github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8=
@@ -195,8 +199,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
 go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
-golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI=
-golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
+golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
+golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
 golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
 golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
 golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=

guest-agent/main.go

@@ -0,0 +1,259 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+
+	"github.com/creack/pty"
+	"github.com/mdlayher/vsock"
+)
+
+const (
+	agentPort = 52000 // JSON control protocol — exec/read_file/write_file
+	shellPort = 52001 // raw PTY streaming — interactive shell sessions
+
+	frameData   byte = 0x01
+	frameResize byte = 0x02
+)
+
+// --- JSON control protocol (exec, read_file, write_file) ---
+
+type Request struct {
+	Type    string   `json:"type"`
+	Cmd     []string `json:"cmd,omitempty"`
+	Path    string   `json:"path,omitempty"`
+	Content string   `json:"content,omitempty"`
+}
+
+type Response struct {
+	Stdout   string `json:"stdout,omitempty"`
+	Stderr   string `json:"stderr,omitempty"`
+	ExitCode int    `json:"exit_code,omitempty"`
+	Content  string `json:"content,omitempty"`
+	Error    string `json:"error,omitempty"`
+}
+
+func main() {
+	go startShellListener()
+
+	l, err := vsock.Listen(agentPort, nil)
+	if err != nil {
+		log.Fatalf("failed to listen on vsock port %d: %v", agentPort, err)
+	}
+	log.Printf("guest-agent listening on vsock port %d", agentPort)
+
+	for {
+		conn, err := l.Accept()
+		if err != nil {
+			log.Printf("accept error: %v", err)
+			continue
+		}
+		go handleConn(conn)
+	}
+}
+
+func handleConn(conn io.ReadWriteCloser) {
+	defer func() {
+		_ = conn.Close()
+	}()
+
+	scanner := bufio.NewScanner(conn)
+	scanner.Buffer(make([]byte, 1024*1024), 10*1024*1024)
+
+	for scanner.Scan() {
+		var req Request
+		if err := json.Unmarshal(scanner.Bytes(), &req); err != nil {
+			writeResp(conn, Response{Error: "invalid request: " + err.Error()})
+			continue
+		}
+		writeResp(conn, handle(req))
+	}
+}
+
+func handle(req Request) Response {
+	switch req.Type {
+	case "exec":
+		return doExec(req.Cmd)
+	case "write_file":
+		return doWriteFile(req.Path, req.Content)
+	case "read_file":
+		return doReadFile(req.Path)
+	default:
+		return Response{Error: "unknown request type: " + req.Type}
+	}
+}
+
+func doExec(cmdArgs []string) Response {
+	if len(cmdArgs) == 0 {
+		return Response{Error: "cmd is required"}
+	}
+	cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...)
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+
+	err := cmd.Run()
+	exitCode := 0
+	if err != nil {
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			exitCode = exitErr.ExitCode()
+		} else {
+			return Response{Error: err.Error()}
+		}
+	}
+	return Response{Stdout: stdout.String(), Stderr: stderr.String(), ExitCode: exitCode}
+}
+
+func doWriteFile(path, content string) Response {
+	if err := os.WriteFile(path, []byte(content), 0644); err != nil {
+		return Response{Error: err.Error()}
+	}
+	return Response{}
+}
+
+func doReadFile(path string) Response {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return Response{Error: err.Error()}
+	}
+	return Response{Content: string(data)}
+}
+
+func writeResp(w io.Writer, resp Response) {
+	b, _ := json.Marshal(resp)
+	if _, err := fmt.Fprintf(w, "%s\n", b); err != nil {
+		log.Printf("failed to write response: %v", err)
+	}
+}
+
+// --- Raw PTY shell protocol ---
+
+// startShellListener accepts persistent shell connections on a port
+// separate from the JSON control protocol — a PTY session is a raw,
+// continuous byte stream, not a request/response exchange, so it needs
+// its own framing rather than being shoehorned into the JSON protocol.
+func startShellListener() {
+	l, err := vsock.Listen(shellPort, nil)
+	if err != nil {
+		log.Fatalf("failed to listen on vsock shell port %d: %v", shellPort, err)
+	}
+	log.Printf("guest-agent shell listener on vsock port %d", shellPort)
+
+	for {
+		conn, err := l.Accept()
+		if err != nil {
+			log.Printf("shell accept error: %v", err)
+			continue
+		}
+		go handleShellConn(conn)
+	}
+}
+
+func handleShellConn(conn io.ReadWriteCloser) {
+	defer func() {
+		_ = conn.Close()
+	}()
+
+	shellPath := "/bin/bash"
+	if _, err := os.Stat(shellPath); err != nil {
+		shellPath = "/bin/sh"
+	}
+
+	cmd := exec.Command(shellPath)
+	cmd.Env = append(os.Environ(), "TERM=xterm-256color")
+	ptmx, err := pty.Start(cmd)
+	if err != nil {
+		log.Printf("failed to start pty: %v", err)
+		return
+	}
+	defer func() {
+		_ = ptmx.Close()
+		if cmd.Process != nil {
+			_ = cmd.Process.Kill()
+			_, _ = cmd.Process.Wait()
+		}
+	}()
+
+	done := make(chan struct{})
+
+	// ptmx -> conn: PTY output streamed back to the host as data frames.
+	go func() {
+		defer close(done)
+		buf := make([]byte, 4096)
+		for {
+			n, err := ptmx.Read(buf)
+			if n > 0 {
+				if werr := writeFrame(conn, frameData, buf[:n]); werr != nil {
+					return
+				}
+			}
+			if err != nil {
+				return
+			}
+		}
+	}()
+
+	// conn -> ptmx: host keystrokes (data frames) and terminal resize
+	// events (resize frames) arriving on the same stream.
+	for {
+		typ, payload, err := readFrame(conn)
+		if err != nil {
+			break
+		}
+		switch typ {
+		case frameData:
+			_, _ = ptmx.Write(payload)
+		case frameResize:
+			if len(payload) == 4 {
+				cols := binary.BigEndian.Uint16(payload[0:2])
+				rows := binary.BigEndian.Uint16(payload[2:4])
+				_ = pty.Setsize(ptmx, &pty.Winsize{Cols: cols, Rows: rows})
+			}
+		}
+	}
+
+	<-done
+}
+
+// writeFrame/readFrame implement a minimal length-prefixed framing:
+// [1 byte type][4 byte big-endian length][payload]. Needed because raw
+// vsock is just an undifferentiated byte stream — without framing, there
+// would be no way to distinguish "this is PTY output" from "this is a
+// resize command" on the wire.
+func writeFrame(w io.Writer, typ byte, payload []byte) error {
+	header := make([]byte, 5)
+	header[0] = typ
+	binary.BigEndian.PutUint32(header[1:], uint32(len(payload)))
+	if _, err := w.Write(header); err != nil {
+		return err
+	}
+	if len(payload) > 0 {
+		if _, err := w.Write(payload); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func readFrame(r io.Reader) (byte, []byte, error) {
+	header := make([]byte, 5)
+	if _, err := io.ReadFull(r, header); err != nil {
+		return 0, nil, err
+	}
+	typ := header[0]
+	length := binary.BigEndian.Uint32(header[1:])
+	payload := make([]byte, length)
+	if length > 0 {
+		if _, err := io.ReadFull(r, payload); err != nil {
+			return 0, nil, err
+		}
+	}
+	return typ, payload, nil
+}

internal/api/shell.go

@@ -41,6 +41,7 @@ func (a *API) OpenShellSession(w http.ResponseWriter, r *http.Request) {
 
 	sb, err := a.store.Get(r.Context(), id)
 	if err != nil {
+		slog.Error("shell: failed to get sandbox", "sandbox_id", id, "error", err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
@@ -55,6 +56,7 @@ func (a *API) OpenShellSession(w http.ResponseWriter, r *http.Request) {
 
 	shell, err := interactive.OpenShell(r.Context(), sb.ID)
 	if err != nil {
+		slog.Error("shell: failed to open backend shell", "sandbox_id", sb.ID, "error", err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}

internal/firecracker/interfaces.go

@@ -3,6 +3,8 @@ package firecracker
 import (
 	"context"
 	"time"
+
+	"github.com/harshalvk/cage/internal/backend"
 )
 
 // fcAPI is the subset of apiClient's methods FirecrackerManager depends on
@@ -33,3 +35,8 @@ type processHandle interface {
 	Kill() error
 	Wait() error
 }
+
+// shellDialerFunc abstracts opening a persistent shell connection -
+// injectable the same way fcAPI/fcVsock are, so tests can fake it without
+// real vsock/KVM
+type shellDialerFunc func(udsPath string) (backend.Shell, error)