Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 25 additions & 2 deletions cmd/warmbox/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,7 @@ Usage:
warmbox volume list
warmbox volume clone <name> <new>
warmbox volume rm <name>
warmbox volume gc [--dry-run]

warmbox snapshot create <volume> [--name <id>]
warmbox snapshot list
Expand Down Expand Up @@ -1290,7 +1291,11 @@ func newVolumeStore(ctx context.Context, cfg *desktop.Config) (*volume.Store, st
return nil, "disabled"
}
fmt.Fprintf(os.Stderr, "warmbox: volumes backed by local dir %s (no cloud remote configured; run `warmbox cloud set`)\n", local)
return volume.NewStore(f, cfg.VolumePrefix, cfg.VolumeDir, cfg.VolumeBase, chunk), "local"
st := volume.NewStore(f, cfg.VolumePrefix, cfg.VolumeDir, cfg.VolumeBase, chunk)
// The remote is a dir on this disk: read chunks straight through to it
// instead of copying every byte into the local cache first.
st.SetLocalRemoteDir(local)
return st, "local"
}

// openCatalog opens the SQLite catalog at <workdir>/warmbox.db.
Expand Down Expand Up @@ -1883,6 +1888,12 @@ const defaultAPITimeout = 30 * time.Second
// readiness before giving up, so anything shorter would kill healthy creates.
const createTimeout = 180 * time.Second

// destroyTimeout covers a slow volume commit: destroy commits the volume
// before answering, and a multi-GB commit outlasts the default API timeout —
// which used to make the CLI report failure for an operation that then
// succeeded server-side (and left the lease looking stale).
const destroyTimeout = 150 * time.Second

// cleanupTimeout bounds best-effort teardown (connect's session DELETE):
// long enough for a slow daemon, short enough that Ctrl-C always exits.
const cleanupTimeout = 5 * time.Second
Expand Down Expand Up @@ -2676,7 +2687,7 @@ func cmdDestroy(args []string) {
fatal("Usage: warmbox destroy <id>")
}
req, _ := http.NewRequest(http.MethodDelete, withToken(apiBase(addr)+"/api/desktops/"+rest[0], token), nil)
resp, err := doAuthed(req)
resp, err := doAuthedTimeout(req, destroyTimeout)
if err != nil {
fatal("Error: %v", err)
}
Expand Down Expand Up @@ -3560,6 +3571,7 @@ Usage:
warmbox volume list
warmbox volume clone <name> <new>
warmbox volume rm <name>
warmbox volume gc [--dry-run] delete chunks no volume or snapshot uses
`)
}

Expand Down Expand Up @@ -3771,6 +3783,17 @@ func cmdVolume(args []string) {
resp, err := doAuthed(req)
doVolume(resp, err)

case "gc":
fs := flag.NewFlagSet("volume gc", flag.ExitOnError)
fs.StringVar(&addr, "addr", addr, "daemon API address")
fs.StringVar(&token, "token", token, "user API token (or `warmbox login` first)")
dry := fs.Bool("dry-run", false, "list orphaned chunks without deleting them")
_ = fs.Parse(args[1:])
payload, _ := json.Marshal(map[string]bool{"dry_run": *dry})
resp, err := postAuthed(withToken(apiBase(addr)+"/api/volumes/gc", token),
"application/json", strings.NewReader(string(payload)))
doVolume(resp, err)

default:
volumeUsage()
os.Exit(1)
Expand Down
15 changes: 15 additions & 0 deletions docs/volumes.md
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ warmbox volume list
warmbox volume clone <name> <new>
warmbox volume snapshot <name> [--as <snap>]
warmbox volume rm <name>
warmbox volume gc [--dry-run] delete remote chunks no volume or snapshot uses
```

Daemon flags for volumes:
Expand Down Expand Up @@ -118,6 +119,20 @@ changing size means growing the disk file and then telling ext4 to stretch
- `Create` stays O(1): it just extends the local image; the filesystem catches up
on first boot. The extra space is holes until written.

## Storage layout and garbage collection

Each volume byte lives in two places, not three: the live `disk.img` and one
copy in the chunk store. When the store is a local dir (no cloud remote), the
daemon reads chunks straight through to it instead of keeping a second copy in
`volumes/chunks/`, and any leftover cache from before is removed. With a real
remote, the local chunk cache is bounded (4 GiB, oldest-first).

`warmbox volume gc [--dry-run]` deletes remote chunks no volume manifest and no
snapshot references — the orphans failed commits and prepares leave behind.
Chunks newer than an hour are spared, so an in-flight commit's not-yet-linked
uploads survive. It only deletes unreferenced chunks, so it is safe to run
while a desktop is attached.

## REST API

```
Expand Down
73 changes: 73 additions & 0 deletions internal/api/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,7 @@ func (s *Server) Handler() http.Handler {
mux.HandleFunc("GET /api/volumes/{name}", s.volumeGet)
mux.HandleFunc("DELETE /api/volumes/{name}", s.volumeDelete)
mux.HandleFunc("POST /api/volumes/{name}/clone", s.volumeClone)
mux.HandleFunc("POST /api/volumes/gc", s.volumeGC)

// Snapshots: frozen volume manifests.
mux.HandleFunc("POST /api/snapshots", s.snapshotCreate)
Expand Down Expand Up @@ -1718,13 +1719,63 @@ func (s *Server) ReapStaleLeases() int {

// lockVolume takes the single-writer lock for a volume (a catalog lease when
// available, otherwise the in-process store lock).
//
// A destroy that timed out client-side leaves its lease behind while the
// server already tore the desktop down — without reconciliation every later
// create fails with "already leased" until a daemon restart. So a conflict
// against an owner with no live desktop is treated as stale: it is released
// and the acquire retried once, and the theft is logged.
func (s *Server) lockVolume(name, owner string) error {
if err := s.acquireLease(name, owner); err == nil {
return nil
} else if !isLeaseConflict(err) {
return err
}
if s.volumeOwnerLive(name) {
return fmt.Errorf("volume %q is already leased", name)
}
stale := s.volumeOwner(name)
if s.log != nil {
fmt.Fprintf(s.log, "warmbox: reclaimed stale volume lease %s from dead desktop %s\n", name, stale)
}
s.unlockVolume(name, stale)
s.volumes.Release(name, stale)
return s.acquireLease(name, owner)
}

func (s *Server) acquireLease(name, owner string) error {
if s.catalog != nil {
return s.catalog.AcquireLease(name, owner)
}
return s.volumes.Attach(name, owner)
}

// volumeOwnerLive reports whether a volume's current owner is a live desktop.
// A nil manager means fail-closed: the owner counts as live, so a lease is
// never stolen without something to check liveness against.
func (s *Server) volumeOwnerLive(name string) bool {
owner := s.volumeOwner(name)
if owner == "" || s.mgr == nil {
return owner != ""
}
for _, info := range s.mgr.List() {
if info.ID == owner {
return true
}
}
return false
}

// isLeaseConflict reports whether err is a "held by someone else" lock
// failure, as opposed to a storage error.
func isLeaseConflict(err error) bool {
if err == nil {
return false
}
return strings.Contains(err.Error(), "already attached") ||
strings.Contains(err.Error(), "already leased")
}

func (s *Server) unlockVolume(name, owner string) {
if s.catalog != nil {
_ = s.catalog.ReleaseLease(name, owner)
Expand Down Expand Up @@ -1899,6 +1950,28 @@ func (s *Server) volumeDelete(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]string{"status": "deleted", "name": name})
}

// volumeGC deletes remote chunks no volume or snapshot manifest references.
// Failed commits and prepares leave exactly such orphans behind, and nothing
// else ever cleans them. It only deletes unreferenced chunks, so running it
// while a desktop is attached is safe: the attached volume's manifest keeps
// its chunks referenced. Chunks newer than the grace period are spared, so an
// in-flight commit's not-yet-linked uploads survive.
func (s *Server) volumeGC(w http.ResponseWriter, r *http.Request) {
if !s.volumesEnabled(w, r) {
return
}
var req struct {
DryRun bool `json:"dry_run"`
}
_ = json.NewDecoder(r.Body).Decode(&req)
rep, err := s.volumes.GC(r.Context(), req.DryRun)
if err != nil {
s.fail(w, r, http.StatusBadGateway, "volume_gc_failed", "garbage collection failed", err)
return
}
writeJSON(w, http.StatusOK, rep)
}

func (s *Server) volumeClone(w http.ResponseWriter, r *http.Request) {
if !s.volumesEnabled(w, r) {
return
Expand Down
Loading
Loading