diff --git a/.github/workflows/firecracker-integration.yml b/.github/workflows/firecracker-integration.yml new file mode 100644 index 0000000..f08cf10 --- /dev/null +++ b/.github/workflows/firecracker-integration.yml @@ -0,0 +1,36 @@ +name: Firecracker Integration (Real KVM) + +on: + push: + branches: [stagging, master] + paths: + - "internal/firecracker/**" + - "gues-agent/**" + workflow_dispatch: {} + +jobs: + kvm-integration: + runs-on: [self-hosted, kvm] + timeout-minutes: 15 + env: + FIRECRACKER_BIN: /usr/local/bin/firecracker + FIRECRACKER_KERNEL: /opt/cage-ci/kernel/vmlinux.bin + FIRECRACKER_ROOTFS_DIR: /opt/cage-ci/rootfs-base + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: "1.25" + cache: true + + - name: Verify KVM access + run: | + if [ ! -e /dev/kvm ]; then + echo "::error::/dev/kvm not available on this runner" + exit 1 + fi + + - name: Run real-kvm integration tests + run: go test -tags kvm ./internal/firecracker/... -v -timeout 10m diff --git a/Makefile b/Makefile index 005b3bb..ec94587 100644 --- a/Makefile +++ b/Makefile @@ -46,4 +46,10 @@ clean: rm -rf tmp genkey: - go run ./cmd/genkey -name=$(name) \ No newline at end of file + go run ./cmd/genkey -name=$(name) + +firecracker-build-rootfs: + ./scripts/build-firecracker-rootfs.sh $(slug) + +test-firecracker-integration: + go test -tags kvm ./internal/firecracker/... -v -timeout 5m \ No newline at end of file diff --git a/cmd/cage/main.go b/cmd/cage/main.go index 02416e0..b6b25e7 100644 --- a/cmd/cage/main.go +++ b/cmd/cage/main.go @@ -81,15 +81,27 @@ func run() error { switch cfg.IsolationBackend { case "firecracker": - fcMgr, err := firecracker.NewFirecrackerManager(firecracker.Config{ + netMgr, netErr := firecracker.NewNetworkManager(firecracker.NetworkConfig{ + Enabled: cfg.FirecrackerNetworkEnabled, + BridgeName: "cage0", + BridgeCIDR: cfg.FirecrackerBridgeCIDR, + SubnetCIDR: cfg.FirecrackerSubnetCIDR, + DNSServer: cfg.FirecrackerDNSServer, + }) + if netErr != nil { + return nil + } + logger.Info("firecracker network manager initialized", "enabled", cfg.FirecrackerNetworkEnabled, "bridge_cidr", cfg.FirecrackerBridgeCIDR) + fcMgr, fcErr := firecracker.NewFirecrackerManager(firecracker.Config{ FirecrackerBin: cfg.FirecrackerBin, KernelPath: cfg.FirecrackerKernel, RootfsBaseDir: cfg.FirecrackerRootfsDir, RunDir: cfg.FirecrackerRunDir, VCPUCount: 1, MemSizeMiB: 128, + Network: netMgr, }) - if err != nil { + if fcErr != nil { return err } sandboxBackend = fcMgr diff --git a/docs/adr/0012-defer-firecracker.md b/docs/adr/0012-defer-firecracker.md index 07975b5..216c433 100644 --- a/docs/adr/0012-defer-firecracker.md +++ b/docs/adr/0012-defer-firecracker.md @@ -43,4 +43,8 @@ the firecracker_rootfs_slug migration) to avoid the Docker-image/rootfs-slug ambiguity. Setup is documented in docs/firecracker-setup.md. Remaining open items: no networking (deliberately deprioritized, vsock covers exec/files), no CI coverage for real KVM behavior (mocked tests only), not yet merged -from the `firecracker` branch to `master`. \ No newline at end of file +from the `firecracker` branch to `master`. + +## Update — 2026-08-19 + +Networking implemented: host-side bridge (cage0), per-VM tap devices, NAT/MASQUERADE, kernel-level static IP via boot args. Opt-in via FIRECRACKER_NETWORK_ENABLED. Base rootfs rebuilt via debootstrap (jammy) after discovering the original quickstart demo image lacks a working dpkg — see docs/firecracker-setup.md. All originally-flagged Firecracker gaps (isolation, pause/resume, warm pool, networking) are now closed. Remaining known gaps: no CI coverage against real KVM (mocked tests only), no automated rootfs-build tooling (fully manual debootstrap process). \ No newline at end of file diff --git a/docs/firecracker-setup.md b/docs/firecracker-setup.md index bb971af..5de6a92 100644 --- a/docs/firecracker-setup.md +++ b/docs/firecracker-setup.md @@ -75,6 +75,15 @@ template's rootfs is a manual process. Automating it (a `make firecracker-build-template slug=X` target) is a known gap, not yet built. +## Building the base rootfs + +\`\`\`bash +export FIRECRACKER_ROOTFS_DIR=/path/to/your/rootfs-dir +make firecracker-build-rootfs slug=base +\`\`\` + +This script handles debootstrap, the devpts/ptmx fix, root autologin, guest agent injection, and packaging — with verification at each step. See `scripts/build-firecracker-rootfs.sh` if you need to customize the base distro/version. + ## Registering a template for Firecracker Templates need `firecracker_rootfs_slug` set in Postgres (see migration @@ -95,8 +104,7 @@ UPDATE templates SET firecracker_rootfs_slug = 'my-slug' WHERE slug = 'my-templa - **Both backends cannot run simultaneously.** `ISOLATION_BACKEND` is a single global switch, not a per-sandbox choice. - **Snapshot/restore is tied to your Firecracker binary version.** Don't - upgrade the `firecracker` binary while sandboxes are paused — see ADR - 0015. + upgrade the `firecracker` binary while sandboxes are paused — see ADR 0015. - **CI does not test this backend.** GitHub-hosted runners do not expose `/dev/kvm`; only the mocked unit tests in `internal/firecracker` run in - CI. Real validation requires a local run against real KVM. \ No newline at end of file + CI. Real validation requires a local run against real KVM. diff --git a/guest-agent/guest-agent b/guest-agent/guest-agent index 65d066e..c907235 100644 Binary files a/guest-agent/guest-agent and b/guest-agent/guest-agent differ diff --git a/internal/config/config.go b/internal/config/config.go index 556a0df..cff7251 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -12,20 +12,24 @@ import ( ) type Config struct { - Port string - DatabaseURL string - RedisURL string - ReaperInterval time.Duration - SandboxTTL time.Duration - PausedTTL time.Duration - WarmPoolSize int - LogLevel string - MetricsToken string - IsolationBackend string // docker or firecracker - FirecrackerBin string - FirecrackerKernel string - FirecrackerRootfsDir string - FirecrackerRunDir string + Port string + DatabaseURL string + RedisURL string + ReaperInterval time.Duration + SandboxTTL time.Duration + PausedTTL time.Duration + WarmPoolSize int + LogLevel string + MetricsToken string + IsolationBackend string // docker or firecracker + FirecrackerBin string + FirecrackerKernel string + FirecrackerRootfsDir string + FirecrackerRunDir string + FirecrackerNetworkEnabled bool + FirecrackerBridgeCIDR string + FirecrackerSubnetCIDR string + FirecrackerDNSServer string } func LoadConfig() (*Config, error) { @@ -45,6 +49,10 @@ func LoadConfig() (*Config, error) { cfg.FirecrackerKernel = getEnv("FIRECRACKER_KERNEL", "/var/lib/cage/vmlinux.bin") cfg.FirecrackerRootfsDir = getEnv("FIRECRACKER_ROOTFS_DIR", "/var/lib/cage/rootfs-base") cfg.FirecrackerRunDir = getEnv("FIRECRACKER_RUN_DIR", "/var/lib/cage/fc-run") + cfg.FirecrackerNetworkEnabled = getEnv("FIRECRACKER_NETWORK_ENABLED", "false") == "true" + cfg.FirecrackerBridgeCIDR = getEnv("FIRECRACKER_BRIDGE_CIDR", "172.16.0.1/24") + cfg.FirecrackerSubnetCIDR = getEnv("FIRECRACKER_SUBNET_CIDR", "172.16.0.0/24") + cfg.FirecrackerDNSServer = getEnv("FIRECRACKER_DNS_SERVER", "8.8.8.8") cfg.IsolationBackend = getEnv("ISOLATION_BACKEND", "docker") if cfg.IsolationBackend != "docker" && cfg.IsolationBackend != "firecracker" { diff --git a/internal/firecracker/client.go b/internal/firecracker/client.go index ae39121..121f7c8 100644 --- a/internal/firecracker/client.go +++ b/internal/firecracker/client.go @@ -150,3 +150,12 @@ func (c *apiClient) loadSnapshot(ctx context.Context, snapshotPath, memFilePath func (c *apiClient) startInstance(ctx context.Context) error { return c.put(ctx, "/actions", map[string]string{"action_type": "InstanceStart"}) } + +// setNetworkInterface attaches a host tap device to the VM as its eth0. +// Must be called before startInstance. +func (c *apiClient) setNetworkInterface(ctx context.Context, tapName string) error { + return c.put(ctx, "/network-interfaces/eth0", map[string]string{ + "iface_id": "eth0", + "host_dev_name": tapName, + }) +} diff --git a/internal/firecracker/integration_test.go b/internal/firecracker/integration_test.go new file mode 100644 index 0000000..8406063 --- /dev/null +++ b/internal/firecracker/integration_test.go @@ -0,0 +1,154 @@ +//go:build kvm + +// This file only compiles/runs with `go test -tags kvm ./...` — it needs +// real KVM access and a built Firecracker environment (binary, kernel, +// base rootfs), and must never run as part of the normal test suite in +// standard CI, which has neither. +package firecracker_test + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/harshalvk/cage/internal/firecracker" +) + +func integrationConfig(t *testing.T) firecracker.Config { + t.Helper() + + bin := os.Getenv("FIRECRACKER_BIN") + kernel := os.Getenv("FIRECRACKER_KERNEL") + rootfsDir := os.Getenv("FIRECRACKER_ROOTFS_DIR") + runDir := t.TempDir() + + if bin == "" || kernel == "" || rootfsDir == "" { + t.Fatal("FIRECRACKER_BIN, FIRECRACKER_KERNEL, and FIRECRACKER_ROOTFS_DIR must be set for integration tests") + } + if _, err := os.Stat(filepath.Join(rootfsDir, "base.ext4")); err != nil { + t.Fatalf("base.ext4 not found in FIRECRACKER_ROOTFS_DIR: %v", err) + } + + return firecracker.Config{ + FirecrackerBin: bin, + KernelPath: kernel, + RootfsBaseDir: rootfsDir, + RunDir: runDir, + VCPUCount: 1, + MemSizeMiB: 128, + BootTimeout: 15 * time.Second, + } +} + +func TestIntegration_FullLifecycle(t *testing.T) { + mgr, err := firecracker.NewFirecrackerManager(integrationConfig(t)) + require.NoError(t, err) + + ctx := context.Background() + sandboxID := "it-" + t.Name() + + t.Log("creating real sandbox against KVM...") + require.NoError(t, mgr.CreateSandbox(ctx, sandboxID, "base")) + defer mgr.KillSandbox(ctx, sandboxID) + + running, err := mgr.IsRunning(ctx, sandboxID) + require.NoError(t, err) + assert.True(t, running) + + t.Log("running a real command...") + stdout, _, exitCode, err := mgr.ExecCommand(ctx, sandboxID, []string{"echo", "integration-test-ok"}) + require.NoError(t, err) + assert.Equal(t, 0, exitCode) + assert.Contains(t, stdout, "integration-test-ok") + + t.Log("writing and reading a real file...") + require.NoError(t, mgr.WriteFile(ctx, sandboxID, "/tmp/it.txt", "hello from integration test")) + content, err := mgr.ReadFile(ctx, sandboxID, "/tmp/it.txt") + require.NoError(t, err) + assert.Equal(t, "hello from integration test", content) +} + +func TestIntegration_PauseResumeSnapshot(t *testing.T) { + mgr, err := firecracker.NewFirecrackerManager(integrationConfig(t)) + require.NoError(t, err) + + ctx := context.Background() + sandboxID := "it-" + t.Name() + + require.NoError(t, mgr.CreateSandbox(ctx, sandboxID, "base")) + + require.NoError(t, mgr.WriteFile(ctx, sandboxID, "/tmp/before-pause.txt", "survived a real snapshot")) + + t.Log("pausing (real snapshot + memory capture)...") + pauseRef, err := mgr.PauseSandbox(ctx, sandboxID) + require.NoError(t, err) + defer mgr.RemoveImage(ctx, pauseRef) + + running, err := mgr.IsRunning(ctx, sandboxID) + require.NoError(t, err) + assert.False(t, running, "sandbox should not be tracked as running while paused") + + t.Log("resuming from snapshot...") + require.NoError(t, mgr.ResumeSandbox(ctx, sandboxID, pauseRef)) + defer mgr.KillSandbox(ctx, sandboxID) + + running, err = mgr.IsRunning(ctx, sandboxID) + require.NoError(t, err) + assert.True(t, running) + + content, err := mgr.ReadFile(ctx, sandboxID, "/tmp/before-pause.txt") + require.NoError(t, err) + assert.Equal(t, "survived a real snapshot", content, "file content must survive a real pause/resume cycle") +} + +func TestIntegration_OpenShell(t *testing.T) { + mgr, err := firecracker.NewFirecrackerManager(integrationConfig(t)) + require.NoError(t, err) + + ctx := context.Background() + sandboxID := "it-" + t.Name() + + require.NoError(t, mgr.CreateSandbox(ctx, sandboxID, "base")) + defer mgr.KillSandbox(ctx, sandboxID) + + t.Log("opening a real interactive shell over vsock...") + shell, err := mgr.OpenShell(ctx, sandboxID) + require.NoError(t, err) + defer shell.Close() + + _, err = shell.Write([]byte("echo shell-test-ok\n")) + require.NoError(t, err) + + buf := make([]byte, 1024) + deadline := time.Now().Add(5 * time.Second) + var output string + for time.Now().Before(deadline) { + n, rerr := shell.Read(buf) + if n > 0 { + output += string(buf[:n]) + if assert.ObjectsAreEqual(true, contains(output, "shell-test-ok")) { + break + } + } + if rerr != nil { + break + } + } + assert.Contains(t, output, "shell-test-ok") +} + +func contains(s, substr string) bool { + return len(s) >= len(substr) && (func() bool { + for i := 0; i+len(substr) <= len(s); i++ { + if s[i:i+len(substr)] == substr { + return true + } + } + return false + })() +} diff --git a/internal/firecracker/interfaces.go b/internal/firecracker/interfaces.go index 57fdaa9..829ca28 100644 --- a/internal/firecracker/interfaces.go +++ b/internal/firecracker/interfaces.go @@ -16,6 +16,7 @@ type fcAPI interface { setBootSource(ctx context.Context, kernelPath, bootArgs string) error setRootDrive(ctx context.Context, rootfsPath string) error setVsock(ctx context.Context, guestCID uint32, udsPath string) error + setNetworkInterface(ctx context.Context, tapName string) error startInstance(ctx context.Context) error pauseVM(ctx context.Context) error createSnapshot(ctx context.Context, snapshotPath, memFilePath string) error diff --git a/internal/firecracker/manager.go b/internal/firecracker/manager.go index fdec3f6..506063c 100644 --- a/internal/firecracker/manager.go +++ b/internal/firecracker/manager.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "log/slog" + "net" "os" "os/exec" "path/filepath" @@ -33,6 +34,7 @@ type vmInstance struct { vsockUDS string rootfsPath string vsock fcVsock + guestIP net.IP } // Config holds the paths and defaults FirecrackerManager needs at startup. @@ -44,6 +46,7 @@ type Config struct { VCPUCount int64 MemSizeMiB int64 BootTimeout time.Duration + Network *NetworkManager } type FirecrackerManager struct { @@ -162,6 +165,18 @@ func (m *FirecrackerManager) CreateSandbox(ctx context.Context, sandboxID, templ removeFileIfExists(apiSocket) removeFileIfExists(vsockUDS) + var tapName string + var guestIP net.IP + if m.cfg.Network != nil && m.cfg.Network.cfg.Enabled { + tapName, guestIP, err = m.cfg.Network.AllocateTap(sandboxID) + if err != nil { + if cerr := m.rootfs.Cleanup(sandboxID); cerr != nil { + slog.Warn("firecracker: rootfs cleanup failed after network allocation failure", "error", cerr) + } + return fmt.Errorf("failed to allocate network: %w", err) + } + } + process, err := m.spawnProcess(m.cfg.FirecrackerBin, apiSocket) // ← was: exec.CommandContext(...) + cmd.Start() if err != nil { if cerr := m.rootfs.Cleanup(sandboxID); cerr != nil { @@ -185,18 +200,32 @@ func (m *FirecrackerManager) CreateSandbox(ctx context.Context, sandboxID, templ if cerr := m.rootfs.Cleanup(sandboxID); cerr != nil { slog.Warn("firecracker: rootfs cleanup failed during rollback", "stage", stage, "error", cerr) } + if guestIP != nil { + m.cfg.Network.ReleaseTap(sandboxID, guestIP) + } return err } + bootArgs := "console=ttyS0 reboot=k panic=1 pci=off" + if guestIP != nil { + gateway, _, _ := net.ParseCIDR(m.cfg.Network.cfg.BridgeCIDR) + bootArgs += fmt.Sprintf(" ip=%s::%s:255.255.255.0::eth0:off", guestIP, gateway) + } + if err := api.setMachineConfig(ctx, m.cfg.VCPUCount, m.cfg.MemSizeMiB); err != nil { return rollback("machine-config", err) } - if err := api.setBootSource(ctx, m.cfg.KernelPath, "console=ttyS0 reboot=k panic=1 pci=off"); err != nil { + if err := api.setBootSource(ctx, m.cfg.KernelPath, bootArgs); err != nil { return rollback("boot-source", err) } if err := api.setRootDrive(ctx, rootfsPath); err != nil { return rollback("root-drive", err) } + if guestIP != nil { + if err := api.setNetworkInterface(ctx, tapName); err != nil { + return rollback("network-interface", err) + } + } if err := api.setVsock(ctx, 3, vsockUDS); err != nil { return rollback("vsock", err) } @@ -208,6 +237,12 @@ func (m *FirecrackerManager) CreateSandbox(ctx context.Context, sandboxID, templ if err := vsock.waitReady(m.cfg.BootTimeout); err != nil { return rollback("guest-agent-ready", fmt.Errorf("guest agent never became ready: %w", err)) } + if guestIP != nil && m.cfg.Network.cfg.DNSServer != "" { + resolvConf := fmt.Sprintf("nameserver %s\n", m.cfg.Network.cfg.DNSServer) + if _, err := vsock.send(agentRequest{Type: "write_file", Path: "/etc/resolv.conf", Content: resolvConf}); err != nil { + slog.Warn("firecracker: failed to write resolve.conf, DNS will not work", "sandbox_id", sandboxID, "error", err) + } + } m.mu.Lock() m.running[sandboxID] = &vmInstance{ @@ -217,6 +252,7 @@ func (m *FirecrackerManager) CreateSandbox(ctx context.Context, sandboxID, templ vsockUDS: vsockUDS, rootfsPath: rootfsPath, vsock: vsock, + guestIP: guestIP, } m.mu.Unlock() @@ -275,6 +311,10 @@ func (m *FirecrackerManager) KillSandbox(ctx context.Context, sandboxID string) removeFileIfExists(inst.apiSocket) removeFileIfExists(inst.vsockUDS) + if inst.guestIP != nil && m.cfg.Network != nil { + m.cfg.Network.ReleaseTap(sandboxID, inst.guestIP) + } + return m.rootfs.Cleanup(sandboxID) } diff --git a/internal/firecracker/manager_test.go b/internal/firecracker/manager_test.go index 1efbbf1..69570f5 100644 --- a/internal/firecracker/manager_test.go +++ b/internal/firecracker/manager_test.go @@ -76,6 +76,12 @@ func (f *fakeAPI) setVsock(ctx context.Context, cid uint32, path string) error { } return nil } +func (f *fakeAPI) setNetworkInterface(ctx context.Context, tapName string) error { + if f.failAt == "network-interface" { + return errors.New("boom") + } + return nil +} func (f *fakeAPI) startInstance(ctx context.Context) error { if f.failAt == "start-instance" { return errors.New("boom") diff --git a/internal/firecracker/network.go b/internal/firecracker/network.go new file mode 100644 index 0000000..e6a5409 --- /dev/null +++ b/internal/firecracker/network.go @@ -0,0 +1,212 @@ +package firecracker + +import ( + "fmt" + "net" + "os" + "os/exec" + "sync" +) + +// NetworkConfig describes the host-side network cage sets up for +// Firecracker sandboxes: one shared linux bridge, NATed to the internet +// with each vm getting its own tap device attached to it +type NetworkConfig struct { + Enabled bool + BridgeName string + BridgeCIDR string + SubnetCIDR string + DNSServer string +} + +// NetworkManager owns the shared bridge and hands out per-vm tap devices +// and IP addresses. one instance is shared across every sandbox +type NetworkManager struct { + cfg NetworkConfig + + mu sync.Mutex + nextIP uint32 + baseIP net.IP + released []uint32 +} + +// NewNetworkManager sets up the shared bridge and NAT rules once +// Safe to call on every server startup - every step is written to be idempotent, +// so re-running against an already-configured bridge is a no-op rather +// than an error +func NewNetworkManager(cfg NetworkConfig) (*NetworkManager, error) { + if !cfg.Enabled { + return &NetworkManager{cfg: cfg}, nil + } + + if os.Getuid() != 0 { + return nil, fmt.Errorf("firecracker networking requires root privilages (CAP_NET_ADMIN) - run the server as root, or set FIRECRACKER_NETWORK_ENABLED=false") + } + + ip, _, err := net.ParseCIDR(cfg.BridgeCIDR) + if err != nil { + return nil, fmt.Errorf("invalid bridge CIDR: %w", err) + } + + nm := &NetworkManager{cfg: cfg, baseIP: ip.Mask(ip.DefaultMask()), nextIP: 2} + + if err := nm.setupBridge(); err != nil { + return nil, fmt.Errorf("failed to set up bridge: %w", err) + } + if err := nm.setupNAT(); err != nil { + return nil, fmt.Errorf("failed to set up NAT: %w", err) + } + + return nm, nil +} + +// func (nm *NetworkManager) setupBridge() error { +// // "ip link add" errors if the bridge already exists -- check first so +// // restarting the server doesn't fail on an already-configured bridge +// if err := runQuiet("ip", "link", "show", nm.cfg.BridgeName); err == nil { +// return nil +// } + +// if err := run("ip", "link", "add", "name", nm.cfg.BridgeName, "type", "bridge"); err != nil { +// return err +// } +// if err := run("ip", "addr", "add", nm.cfg.BridgeCIDR, "dev", nm.cfg.BridgeName); err == nil { +// return err +// } +// if err := run("ip", "link", "set", nm.cfg.BridgeName, "up"); err != nil { +// return err +// } +// return nil +// } + +func (nm *NetworkManager) setupBridge() error { + if err := runQuiet("ip", "link", "show", nm.cfg.BridgeName); err != nil { + // Bridge doesn't exist yet — create and address it. + if err := run("ip", "link", "add", "name", nm.cfg.BridgeName, "type", "bridge"); err != nil { + return err + } + if err := run("ip", "addr", "add", nm.cfg.BridgeCIDR, "dev", nm.cfg.BridgeName); err != nil { + return err + } + } + + // Always ensure it's up, whether newly created or pre-existing — + // a bridge left DOWN from a previous run must be corrected here, + // not silently skipped by the idempotency check above. + return run("ip", "link", "set", nm.cfg.BridgeName, "up") +} + +func (nm *NetworkManager) setupNAT() error { + if err := run("sh", "-c", "echo 1 > /proc/sys/net/ipv4/ip_forward"); err != nil { + return fmt.Errorf("failed to enable ip_forward: %w", err) + } + + // -C checks whether the rule already exists; only add it if -C fails + // so re-running this doesn't stack duplicate NAT rule on every restart + checkArgs := []string{"-t", "nat", "-C", "POSTROUTING", "-s", nm.cfg.SubnetCIDR, "-j", "MASQUERADE"} + if err := runQuiet("iptables", checkArgs...); err == nil { + return nil + } + + addArgs := []string{"-t", "nat", "-A", "POSTROUTING", "-s", nm.cfg.SubnetCIDR, "-j", "MASQUERADE"} + return run("iptables", addArgs...) +} + +// AllocateTap creates a new tap device for one sandbox, attaches it to the +// shared bridge, and assigns it a guest IP from the pool. Returns the tap +// device name and the guest's assigned IP +func (nm *NetworkManager) AllocateTap(sandboxID string) (tapName string, guestIP net.IP, err error) { + if !nm.cfg.Enabled { + return "", nil, fmt.Errorf("networking is not enabled") + } + + tapName = tapDeviceName(sandboxID) + + if err := run("ip", "tuntap", "add", "dev", tapName, "mode", "tap"); err != nil { + return "", nil, fmt.Errorf("failed to create tap device: %w", err) + } + if err := run("ip", "link", "set", tapName, "master", nm.cfg.BridgeName); err != nil { + nm.releaseTapDevice(tapName) + return "", nil, fmt.Errorf("failed to attach tap to bridge: %w", err) + } + if err := run("ip", "link", "set", tapName, "up"); err != nil { + nm.releaseTapDevice(tapName) + return "", nil, fmt.Errorf("failed to bring tap device up: %w", err) + } + + guestIP = nm.allocateIP() + return tapName, guestIP, nil +} + +// ReleaseTap tears down a sandbox's tap device and returns its ip to the pool +func (nm *NetworkManager) ReleaseTap(sandboxID string, guestIP net.IP) { + if !nm.cfg.Enabled { + return + } + nm.releaseTapDevice(tapDeviceName(sandboxID)) + nm.releaseIP(guestIP) +} + +func (nm *NetworkManager) releaseTapDevice(tapName string) { + // Best-effort - a failed tap deletion leaves an orphaned host device, + // which is a minor leak worth monitoring but shouldn't block sandbox + // cleanup from proceeding + _ = run("ip", "link", "delete", tapName) +} + +func (nm *NetworkManager) allocateIP() net.IP { + nm.mu.Lock() + defer nm.mu.Unlock() + + var offset uint32 + if len(nm.released) > 0 { + offset, nm.released = nm.released[len(nm.released)-1], nm.released[:len(nm.released)-1] + } else { + offset = nm.nextIP + nm.nextIP++ + } + return offsetIP(nm.baseIP, offset) +} + +func (nm *NetworkManager) releaseIP(ip net.IP) { + nm.mu.Lock() + defer nm.mu.Unlock() + offset := ipOffset(nm.baseIP, ip) + nm.released = append(nm.released, offset) +} + +func tapDeviceName(sandboxID string) string { + // linux interface names are capped at 15 chars - use a short perfix + // plus the first 10 chars of the sandbox ID, which is unique enough + // in practice for concurrently-running sandboxes + if len(sandboxID) > 10 { + sandboxID = sandboxID[:10] + } + return "cg-" + sandboxID +} + +func offsetIP(base net.IP, offset uint32) net.IP { + ip := make(net.IP, len(base.To4())) + copy(ip, base.To4()) + ip[3] += byte(offset) + return ip +} + +func ipOffset(base net.IP, ip net.IP) uint32 { + b := base.To4() + i := ip.To4() + return uint32(i[3] - b[3]) +} + +func run(name string, args ...string) error { + cmd := exec.Command(name, args...) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("%s %v failed: %w (output: %s)", name, args, err, out) + } + return nil +} + +func runQuiet(name string, args ...string) error { + return exec.Command(name, args...).Run() +} diff --git a/internal/store/store.go b/internal/store/store.go index 9779bf4..5ff4322 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -23,7 +23,7 @@ type Sandbox struct { Status SandboxStatus `json:"status"` CreatedAt time.Time `json:"created_at"` ExpiresAt time.Time `json:"expires_at"` - TemplateSlug string `json:"template"` + TemplateSlug string `json:"template_slug"` PausedImageID *string `json:"-"` } diff --git a/scripts/build-firecracker-rootfs.sh b/scripts/build-firecracker-rootfs.sh new file mode 100644 index 0000000..56e18c9 --- /dev/null +++ b/scripts/build-firecracker-rootfs.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Builds a Firecracker-compatible rootfs image via debootstrap, with the +# Cage guest agent injected and every gotcha discovered during initial +# development already accounted for (devpts permissions, root autologin, +# a properly sized filesystem). +# +# Usage: ./scripts/build-firecracker-rootfs.sh [size-mb] +# Example: ./scripts/build-firecracker-rootfs.sh base 2048 + +TEMPLATE_SLUG="${1:?Usage: $0 [size-mb]}" +SIZE_MB="${2:-2048}" + +: "${FIRECRACKER_ROOTFS_DIR:?Set FIRECRACKER_ROOTFS_DIR before running this script}" + +WORKDIR="$(mktemp -d)" +BUILD_TREE="$WORKDIR/rootfs-build" +IMAGE_PATH="$WORKDIR/rootfs.ext4" +MOUNT_POINT="$WORKDIR/mount" +GUEST_AGENT_SRC="$(cd "$(dirname "${BASH_SOURCE[0]}")/../guest-agent" && pwd)" + +cleanup() { + if mountpoint -q "$MOUNT_POINT" 2>/dev/null; then + echo "→ Cleaning up: unmounting $MOUNT_POINT" + sudo umount "$MOUNT_POINT" + fi + sudo rm -rf "$WORKDIR" +} +trap cleanup EXIT + +echo "=== Building rootfs for template '$TEMPLATE_SLUG' (${SIZE_MB}MB) ===" + +echo "→ Step 0/8: Verifying sudo access..." +if ! sudo -v; then + echo "ERROR: this script requires sudo access" >&2 + exit 1 +fi + +echo "→ Step 1/8: Checking for debootstrap..." +if ! command -v debootstrap >/dev/null 2>&1; then + echo " debootstrap not found, installing..." + sudo apt update && sudo apt install -y debootstrap +fi + +echo "→ Step 2/8: Bootstrapping base Ubuntu jammy system (this takes a few minutes)..." +sudo debootstrap --arch=amd64 jammy "$BUILD_TREE" http://archive.ubuntu.com/ubuntu + +echo "→ Step 3/8: Setting hostname..." +echo "cage-sandbox" | sudo tee "$BUILD_TREE/etc/hostname" > /dev/null + +echo "→ Step 4/8: Fixing devpts permissions (ptmxmode=666 — without this, PTY sessions fail silently)..." +echo "devpts /dev/pts devpts gid=5,mode=620,ptmxmode=666 0 0" | sudo tee -a "$BUILD_TREE/etc/fstab" > /dev/null + +echo "→ Step 5/8: Enabling root autologin on serial console..." +sudo mkdir -p "$BUILD_TREE/etc/systemd/system/serial-getty@ttyS0.service.d" +sudo tee "$BUILD_TREE/etc/systemd/system/serial-getty@ttyS0.service.d/autologin.conf" > /dev/null <<'EOF' +[Service] +ExecStart= +ExecStart=-/sbin/agetty --autologin root --noclear %I $TERM +EOF + +echo "→ Step 6/8: Building and injecting the Cage guest agent..." +(cd "$GUEST_AGENT_SRC" && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o "$WORKDIR/guest-agent" .) +BUILT_HASH=$(md5sum "$WORKDIR/guest-agent" | awk '{print $1}') +echo " built guest-agent, md5=$BUILT_HASH" + +sudo mkdir -p "$BUILD_TREE/usr/local/bin" +sudo cp "$WORKDIR/guest-agent" "$BUILD_TREE/usr/local/bin/guest-agent" +sudo chmod +x "$BUILD_TREE/usr/local/bin/guest-agent" + +INJECTED_HASH=$(md5sum "$BUILD_TREE/usr/local/bin/guest-agent" | awk '{print $1}') +if [ "$BUILT_HASH" != "$INJECTED_HASH" ]; then + echo " ERROR: injected guest-agent hash ($INJECTED_HASH) does not match built hash ($BUILT_HASH)" >&2 + exit 1 +fi +echo " verified: injected binary matches freshly built binary" + +sudo mkdir -p "$BUILD_TREE/etc/systemd/system" +sudo tee "$BUILD_TREE/etc/systemd/system/guest-agent.service" > /dev/null <<'EOF' +[Unit] +Description=Cage guest agent +After=network.target + +[Service] +ExecStart=/usr/local/bin/guest-agent +Restart=always +User=root + +[Install] +WantedBy=multi-user.target +EOF +sudo mkdir -p "$BUILD_TREE/etc/systemd/system/multi-user.target.wants" +sudo ln -sf /etc/systemd/system/guest-agent.service \ + "$BUILD_TREE/etc/systemd/system/multi-user.target.wants/guest-agent.service" + +echo "→ Step 7/8: Verifying dpkg is functional before packaging..." +if [ ! -f "$BUILD_TREE/var/lib/dpkg/status" ]; then + echo " ERROR: /var/lib/dpkg/status missing — debootstrap did not produce a working dpkg system" >&2 + exit 1 +fi +echo " verified: dpkg/status present" + +echo "→ Step 8/8: Packaging into a ${SIZE_MB}MB ext4 image..." +dd if=/dev/zero of="$IMAGE_PATH" bs=1M count="$SIZE_MB" status=none +mkfs.ext4 -q "$IMAGE_PATH" + +mkdir -p "$MOUNT_POINT" +sudo mount -o loop "$IMAGE_PATH" "$MOUNT_POINT" +sudo cp -a "$BUILD_TREE/." "$MOUNT_POINT/" + +# Final verification INSIDE the mount, before unmounting — this exact +# check would have caught nearly every stale-copy bug hit during +# development of this feature. +FINAL_HASH=$(md5sum "$MOUNT_POINT/usr/local/bin/guest-agent" | awk '{print $1}') +if [ "$FINAL_HASH" != "$BUILT_HASH" ]; then + echo " ERROR: final packaged guest-agent hash mismatch after copy into image" >&2 + exit 1 +fi +if ! grep -q "ptmxmode=666" "$MOUNT_POINT/etc/fstab"; then + echo " ERROR: devpts fix missing from packaged image's fstab" >&2 + exit 1 +fi +echo " verified: guest-agent and fstab fix both present in final image" + +sudo umount "$MOUNT_POINT" + +DEST="$FIRECRACKER_ROOTFS_DIR/${TEMPLATE_SLUG}.ext4" +mkdir -p "$FIRECRACKER_ROOTFS_DIR" +cp "$IMAGE_PATH" "$DEST" + +echo "" +echo "=== Done ===" +echo "Rootfs for template '$TEMPLATE_SLUG' written to: $DEST" +echo "Remember to set firecracker_rootfs_slug = '$TEMPLATE_SLUG' for this template in Postgres if it's a new one." \ No newline at end of file diff --git a/scripts/setup-kvm-runner.sh b/scripts/setup-kvm-runner.sh new file mode 100644 index 0000000..e66b155 --- /dev/null +++ b/scripts/setup-kvm-runner.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail + +# One-time setup for a self-hosted GitHub Actions runner with KVM access, +# for running Firecracker integration tests. Run this on a Linux host (or +# WSL2 with KVM passthrough confirmed working) BEFORE registering it as a +# runner via GitHub's own runner registration script. + +echo "→ Checking for KVM access..." +if [ ! -e /dev/kvm ]; then + echo "ERROR: /dev/kvm does not exist. This host cannot run Firecracker." >&2 + echo "See docs/firecracker-setup.md for KVM prerequisites." >&2 + exit 1 +fi +echo " OK" + +echo "→ Installing Firecracker binary..." +FC_VERSION=$(curl -s https://api.github.com/repos/firecracker-microvm/firecracker/releases/latest | grep '"tag_name"' | cut -d'"' -f4) +FC_URL=$(curl -s "https://api.github.com/repos/firecracker-microvm/firecracker/releases/latest" \ + | grep browser_download_url | grep x86_64 | grep -v debug | cut -d '"' -f4) +curl -L -o /tmp/firecracker.tgz "$FC_URL" +tar xzf /tmp/firecracker.tgz -C /tmp +sudo mv /tmp/release-*/firecracker-*-x86_64 /usr/local/bin/firecracker +sudo chmod +x /usr/local/bin/firecracker +echo " installed $(firecracker --version)" + +echo "→ Downloading kernel image..." +sudo mkdir -p /opt/cage-ci/kernel +sudo curl -fsSL -o /opt/cage-ci/kernel/vmlinux.bin \ + https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/x86_64/kernels/vmlinux.bin + +echo "→ Building base rootfs via debootstrap..." +export FIRECRACKER_ROOTFS_DIR=/opt/cage-ci/rootfs-base +sudo mkdir -p "$FIRECRACKER_ROOTFS_DIR" +"$(dirname "${BASH_SOURCE[0]}")/build-firecracker-rootfs.sh" base + +echo "" +echo "=== KVM runner prerequisites installed ===" +echo "Set these as environment variables for the GitHub Actions runner service:" +echo " FIRECRACKER_BIN=/usr/local/bin/firecracker" +echo " FIRECRACKER_KERNEL=/opt/cage-ci/kernel/vmlinux.bin" +echo " FIRECRACKER_ROOTFS_DIR=/opt/cage-ci/rootfs-base" +echo "" +echo "Next: register this host as a GitHub Actions self-hosted runner" +echo " (Settings -> Actions -> Runners -> New self-hosted runner in your repo)," +echo " using the label 'kvm' when configuring it." \ No newline at end of file