diff --git a/cmd/warmbox/main.go b/cmd/warmbox/main.go index d7e203f..cf62629 100644 --- a/cmd/warmbox/main.go +++ b/cmd/warmbox/main.go @@ -241,6 +241,7 @@ Usage: warmbox volume list warmbox volume clone warmbox volume rm + warmbox volume gc [--dry-run] warmbox snapshot create [--name ] warmbox snapshot list @@ -1290,7 +1291,11 @@ func newVolumeStore(ctx context.Context, cfg *desktop.Config) (*volume.Store, st return nil, "disabled" } fmt.Fprintf(os.Stderr, "warmbox: volumes backed by local dir %s (no cloud remote configured; run `warmbox cloud set`)\n", local) - return volume.NewStore(f, cfg.VolumePrefix, cfg.VolumeDir, cfg.VolumeBase, chunk), "local" + st := volume.NewStore(f, cfg.VolumePrefix, cfg.VolumeDir, cfg.VolumeBase, chunk) + // The remote is a dir on this disk: read chunks straight through to it + // instead of copying every byte into the local cache first. + st.SetLocalRemoteDir(local) + return st, "local" } // openCatalog opens the SQLite catalog at /warmbox.db. @@ -1883,6 +1888,12 @@ const defaultAPITimeout = 30 * time.Second // readiness before giving up, so anything shorter would kill healthy creates. const createTimeout = 180 * time.Second +// destroyTimeout covers a slow volume commit: destroy commits the volume +// before answering, and a multi-GB commit outlasts the default API timeout — +// which used to make the CLI report failure for an operation that then +// succeeded server-side (and left the lease looking stale). +const destroyTimeout = 150 * time.Second + // cleanupTimeout bounds best-effort teardown (connect's session DELETE): // long enough for a slow daemon, short enough that Ctrl-C always exits. const cleanupTimeout = 5 * time.Second @@ -2676,7 +2687,7 @@ func cmdDestroy(args []string) { fatal("Usage: warmbox destroy ") } req, _ := http.NewRequest(http.MethodDelete, withToken(apiBase(addr)+"/api/desktops/"+rest[0], token), nil) - resp, err := doAuthed(req) + resp, err := doAuthedTimeout(req, destroyTimeout) if err != nil { fatal("Error: %v", err) } @@ -3560,6 +3571,7 @@ Usage: warmbox volume list warmbox volume clone warmbox volume rm + warmbox volume gc [--dry-run] delete chunks no volume or snapshot uses `) } @@ -3771,6 +3783,17 @@ func cmdVolume(args []string) { resp, err := doAuthed(req) doVolume(resp, err) + case "gc": + fs := flag.NewFlagSet("volume gc", flag.ExitOnError) + fs.StringVar(&addr, "addr", addr, "daemon API address") + fs.StringVar(&token, "token", token, "user API token (or `warmbox login` first)") + dry := fs.Bool("dry-run", false, "list orphaned chunks without deleting them") + _ = fs.Parse(args[1:]) + payload, _ := json.Marshal(map[string]bool{"dry_run": *dry}) + resp, err := postAuthed(withToken(apiBase(addr)+"/api/volumes/gc", token), + "application/json", strings.NewReader(string(payload))) + doVolume(resp, err) + default: volumeUsage() os.Exit(1) diff --git a/docs/volumes.md b/docs/volumes.md index 3825d5c..a65f74c 100644 --- a/docs/volumes.md +++ b/docs/volumes.md @@ -77,6 +77,7 @@ warmbox volume list warmbox volume clone warmbox volume snapshot [--as ] warmbox volume rm +warmbox volume gc [--dry-run] delete remote chunks no volume or snapshot uses ``` Daemon flags for volumes: @@ -118,6 +119,20 @@ changing size means growing the disk file and then telling ext4 to stretch - `Create` stays O(1): it just extends the local image; the filesystem catches up on first boot. The extra space is holes until written. +## Storage layout and garbage collection + +Each volume byte lives in two places, not three: the live `disk.img` and one +copy in the chunk store. When the store is a local dir (no cloud remote), the +daemon reads chunks straight through to it instead of keeping a second copy in +`volumes/chunks/`, and any leftover cache from before is removed. With a real +remote, the local chunk cache is bounded (4 GiB, oldest-first). + +`warmbox volume gc [--dry-run]` deletes remote chunks no volume manifest and no +snapshot references — the orphans failed commits and prepares leave behind. +Chunks newer than an hour are spared, so an in-flight commit's not-yet-linked +uploads survive. It only deletes unreferenced chunks, so it is safe to run +while a desktop is attached. + ## REST API ``` diff --git a/internal/api/server.go b/internal/api/server.go index 6774ff8..8c9054d 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -198,6 +198,7 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("GET /api/volumes/{name}", s.volumeGet) mux.HandleFunc("DELETE /api/volumes/{name}", s.volumeDelete) mux.HandleFunc("POST /api/volumes/{name}/clone", s.volumeClone) + mux.HandleFunc("POST /api/volumes/gc", s.volumeGC) // Snapshots: frozen volume manifests. mux.HandleFunc("POST /api/snapshots", s.snapshotCreate) @@ -1718,13 +1719,63 @@ func (s *Server) ReapStaleLeases() int { // lockVolume takes the single-writer lock for a volume (a catalog lease when // available, otherwise the in-process store lock). +// +// A destroy that timed out client-side leaves its lease behind while the +// server already tore the desktop down — without reconciliation every later +// create fails with "already leased" until a daemon restart. So a conflict +// against an owner with no live desktop is treated as stale: it is released +// and the acquire retried once, and the theft is logged. func (s *Server) lockVolume(name, owner string) error { + if err := s.acquireLease(name, owner); err == nil { + return nil + } else if !isLeaseConflict(err) { + return err + } + if s.volumeOwnerLive(name) { + return fmt.Errorf("volume %q is already leased", name) + } + stale := s.volumeOwner(name) + if s.log != nil { + fmt.Fprintf(s.log, "warmbox: reclaimed stale volume lease %s from dead desktop %s\n", name, stale) + } + s.unlockVolume(name, stale) + s.volumes.Release(name, stale) + return s.acquireLease(name, owner) +} + +func (s *Server) acquireLease(name, owner string) error { if s.catalog != nil { return s.catalog.AcquireLease(name, owner) } return s.volumes.Attach(name, owner) } +// volumeOwnerLive reports whether a volume's current owner is a live desktop. +// A nil manager means fail-closed: the owner counts as live, so a lease is +// never stolen without something to check liveness against. +func (s *Server) volumeOwnerLive(name string) bool { + owner := s.volumeOwner(name) + if owner == "" || s.mgr == nil { + return owner != "" + } + for _, info := range s.mgr.List() { + if info.ID == owner { + return true + } + } + return false +} + +// isLeaseConflict reports whether err is a "held by someone else" lock +// failure, as opposed to a storage error. +func isLeaseConflict(err error) bool { + if err == nil { + return false + } + return strings.Contains(err.Error(), "already attached") || + strings.Contains(err.Error(), "already leased") +} + func (s *Server) unlockVolume(name, owner string) { if s.catalog != nil { _ = s.catalog.ReleaseLease(name, owner) @@ -1899,6 +1950,28 @@ func (s *Server) volumeDelete(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, map[string]string{"status": "deleted", "name": name}) } +// volumeGC deletes remote chunks no volume or snapshot manifest references. +// Failed commits and prepares leave exactly such orphans behind, and nothing +// else ever cleans them. It only deletes unreferenced chunks, so running it +// while a desktop is attached is safe: the attached volume's manifest keeps +// its chunks referenced. Chunks newer than the grace period are spared, so an +// in-flight commit's not-yet-linked uploads survive. +func (s *Server) volumeGC(w http.ResponseWriter, r *http.Request) { + if !s.volumesEnabled(w, r) { + return + } + var req struct { + DryRun bool `json:"dry_run"` + } + _ = json.NewDecoder(r.Body).Decode(&req) + rep, err := s.volumes.GC(r.Context(), req.DryRun) + if err != nil { + s.fail(w, r, http.StatusBadGateway, "volume_gc_failed", "garbage collection failed", err) + return + } + writeJSON(w, http.StatusOK, rep) +} + func (s *Server) volumeClone(w http.ResponseWriter, r *http.Request) { if !s.volumesEnabled(w, r) { return diff --git a/internal/cloudstore/cloudstore.go b/internal/cloudstore/cloudstore.go index c220a11..8885e94 100644 --- a/internal/cloudstore/cloudstore.go +++ b/internal/cloudstore/cloudstore.go @@ -19,6 +19,7 @@ import ( "io" "os" "path/filepath" + "sort" "strconv" "strings" "time" @@ -31,6 +32,11 @@ import ( // DefaultChunkSize is the transfer/allocation granularity. const DefaultChunkSize = 16 << 20 +// DefaultMaxCacheBytes bounds the local chunk cache for real (non-local) +// remotes: 4 GiB of 16 MiB chunks. A cache bigger than this is a second copy +// of the data, not a cache. +const DefaultMaxCacheBytes = 4 << 30 + // Manifest maps chunk index -> content hash (and the geometry). type Manifest struct { ChunkSize int64 `json:"chunk_size"` @@ -43,6 +49,17 @@ type Store struct { fs rfs.Fs chunkRoot string // e.g. "volumes/chunks" chunkSize int64 + + // localChunks, when non-empty, is a directory on this host's disk holding + // exactly the objects under chunkRoot: the "remote" is a local dir, so + // GetChunk reads straight through to it instead of copying every byte + // into cacheDir first. Without this a local backend stores every volume + // byte three times (live image + cacheDir copy + remote copy). + localChunks string + + // maxCacheBytes bounds cacheDir for real remotes (LRU by mtime). + // <= 0 means DefaultMaxCacheBytes. + maxCacheBytes int64 } // New builds a Store. chunkSize <= 0 uses DefaultChunkSize. @@ -50,12 +67,56 @@ func New(f rfs.Fs, chunkRoot string, chunkSize int64) *Store { if chunkSize <= 0 { chunkSize = DefaultChunkSize } - return &Store{fs: f, chunkRoot: chunkRoot, chunkSize: chunkSize} + return &Store{fs: f, chunkRoot: chunkRoot, chunkSize: chunkSize, maxCacheBytes: DefaultMaxCacheBytes} +} + +// SetLocalChunksDir tells the store its remote chunk root lives at dir on +// this host's disk, so reads go straight to it and the cache stays empty. +// dir must hold exactly the objects under chunkRoot. +func (s *Store) SetLocalChunksDir(dir string) { s.localChunks = dir } + +// SetMaxCacheBytes bounds the local chunk cache for real remotes. n <= 0 +// restores the default. +func (s *Store) SetMaxCacheBytes(n int64) { + if n <= 0 { + n = DefaultMaxCacheBytes + } + s.maxCacheBytes = n } // ChunkSize returns the configured chunk size. func (s *Store) ChunkSize() int64 { return s.chunkSize } +// ChunkRef names a stored chunk and its size. +type ChunkRef struct { + Hash string + Size int64 + ModTime time.Time +} + +// DeleteChunk removes one stored chunk. +func (s *Store) DeleteChunk(ctx context.Context, hash string) error { + return s.Delete(ctx, s.chunkPath(hash)) +} + +// ListChunks lists the chunks in the remote store. Used by garbage +// collection to find hashes no manifest references anymore. +func (s *Store) ListChunks(ctx context.Context) ([]ChunkRef, error) { + entries, err := s.fs.List(ctx, s.chunkRoot) + if err != nil { + return nil, err + } + var out []ChunkRef + for _, e := range entries { + obj, ok := e.(rfs.Object) + if !ok { + continue + } + out = append(out, ChunkRef{Hash: pathBase(obj.Remote()), Size: obj.Size(), ModTime: obj.ModTime(ctx)}) + } + return out, nil +} + // Fs returns the underlying rclone filesystem. func (s *Store) Fs() rfs.Fs { return s.fs } @@ -161,23 +222,152 @@ func (s *Store) PutChunk(ctx context.Context, data []byte) (string, error) { } // GetChunk returns a chunk by content hash, using cacheDir as a local cache. +// +// A cache hit is hash-verified: a partial file left by a crashed or killed +// run is deleted and re-fetched, never served as good data. Cache entries are +// written atomically (temp file + rename), so a crash cannot leave a torn +// file behind in the first place. +// +// When the remote is a local dir (see SetLocalChunksDir) the bytes are read +// straight through to it and cacheDir is untouched: copying them would store +// every byte twice on one disk for no benefit. func (s *Store) GetChunk(ctx context.Context, hash, cacheDir string) ([]byte, error) { + if s.localChunks != "" { + b, err := os.ReadFile(filepath.Join(s.localChunks, hash)) + if err != nil { + return nil, err + } + return b, nil + } if cacheDir != "" { if b, err := os.ReadFile(filepath.Join(cacheDir, hash)); err == nil { - return b, nil + if Hash(b) == hash { + return b, nil + } + // Corrupt entry: drop it so the fetch below replaces it. + _ = os.Remove(filepath.Join(cacheDir, hash)) } } b, err := s.GetBytes(ctx, s.chunkPath(hash)) if err != nil { return nil, err } + if Hash(b) != hash { + return nil, fmt.Errorf("chunk %s: remote bytes do not match their hash", hash) + } if cacheDir != "" { _ = os.MkdirAll(cacheDir, 0o755) - _ = os.WriteFile(filepath.Join(cacheDir, hash), b, 0o644) + _ = writeFileAtomic(filepath.Join(cacheDir, hash), b, 0o644) + _, _ = s.pruneCache(cacheDir) } return b, nil } +// writeFileAtomic writes data to path via a temp file in the same directory +// plus rename, so a crash mid-write leaves the old file (or nothing), never +// a partial one. +func writeFileAtomic(path string, data []byte, perm os.FileMode) error { + tmp, err := os.CreateTemp(filepath.Dir(path), ".tmp-*") + if err != nil { + return err + } + tmpName := tmp.Name() + if _, err := tmp.Write(data); err != nil { + tmp.Close() + os.Remove(tmpName) + return err + } + if err := tmp.Close(); err != nil { + os.Remove(tmpName) + return err + } + if err := os.Chmod(tmpName, perm); err != nil { + os.Remove(tmpName) + return err + } + if err := os.Rename(tmpName, path); err != nil { + os.Remove(tmpName) + return err + } + return nil +} + +// PruneCache deletes the oldest cache entries (by mtime) until cacheDir fits +// in maxBytes, and returns the bytes reclaimed. maxBytes <= 0 uses the +// store's bound. It never touches a cache the local read-through made +// obsolete: with no cache writes there is nothing to bound. +func (s *Store) PruneCache(cacheDir string, maxBytes int64) (int64, error) { + if s.localChunks != "" { + return 0, nil + } + return s.pruneCacheTo(cacheDir, maxBytes) +} + +func (s *Store) pruneCache(cacheDir string) (int64, error) { + return s.pruneCacheTo(cacheDir, 0) +} + +func (s *Store) pruneCacheTo(cacheDir string, maxBytes int64) (int64, error) { + max := maxBytes + if max <= 0 { + max = s.maxCacheBytes + } + if max <= 0 { + max = DefaultMaxCacheBytes + } + if cacheDir == "" { + return 0, nil + } + entries, err := os.ReadDir(cacheDir) + if err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + type fi struct { + name string + size int64 + mod time.Time + } + var total int64 + files := make([]fi, 0, len(entries)) + for _, e := range entries { + if e.IsDir() { + continue + } + if strings.HasPrefix(e.Name(), ".tmp-") { + // A torn write from a crashed run: never a valid entry. Only + // remove settled ones so an in-flight write is never disturbed. + if info, err := e.Info(); err == nil && time.Since(info.ModTime()) > time.Hour { + _ = os.Remove(filepath.Join(cacheDir, e.Name())) + } + continue + } + info, err := e.Info() + if err != nil { + continue + } + total += info.Size() + files = append(files, fi{e.Name(), info.Size(), info.ModTime()}) + } + if total <= max { + return 0, nil + } + sort.Slice(files, func(i, j int) bool { return files[i].mod.Before(files[j].mod) }) + var reclaimed int64 + for _, f := range files { + if total <= max { + break + } + if err := os.Remove(filepath.Join(cacheDir, f.name)); err == nil { + total -= f.size + reclaimed += f.size + } + } + return reclaimed, nil +} + // Hash returns the content hash of a chunk. func Hash(data []byte) string { sum := sha256.Sum256(data) @@ -225,23 +415,75 @@ func (s *Store) BuildManifest(ctx context.Context, img *os.File, size, chunkSize return m, nil } +// ApplyStats reports what ApplyManifest did. Wrote is the number of chunks +// actually written; Skipped is the number already identical on disk. +type ApplyStats struct { + Wrote int + Skipped int + BytesWritten int64 + BytesSkipped int64 +} + // ApplyManifest materializes a manifest into a local image. Chunks already in // cacheDir are reused (so shared chunks are downloaded once per host). -func (s *Store) ApplyManifest(ctx context.Context, m *Manifest, img *os.File, cacheDir string) error { +// +// Every manifest chunk is compared against the bytes already on disk first, +// and identical ranges are left untouched: a create on an up-to-date image +// performs zero chunk writes instead of rewriting the whole disk. Ranges the +// manifest does not list are never touched, so a crash mid-apply can only +// leave an old-or-new chunk behind — never a torn one — and the next apply +// repairs it by the same comparison. Fetched bytes are always hash-verified +// before they reach the image, so a corrupt cache entry (see GetChunk) can +// never be written into a disk. +func (s *Store) ApplyManifest(ctx context.Context, m *Manifest, img *os.File, cacheDir string) (ApplyStats, error) { + var st ApplyStats + if m.ChunkSize <= 0 { + return st, fmt.Errorf("manifest has no chunk size") + } + buf := make([]byte, m.ChunkSize) for idxStr, hash := range m.Chunks { idx, err := strconv.Atoi(idxStr) if err != nil { continue } + off := int64(idx) * m.ChunkSize + ln := m.ChunkSize + if rem := m.Size - off; rem < ln { + ln = rem + } + if ln <= 0 { + continue + } + b := buf[:ln] + n, err := img.ReadAt(b, off) + if err != nil && err != io.EOF { + return st, err + } + for i := n; i < len(b); i++ { + b[i] = 0 + } + if Hash(b) == hash { + st.Skipped++ + st.BytesSkipped += ln + continue + } data, err := s.GetChunk(ctx, hash, cacheDir) if err != nil { - return fmt.Errorf("chunk %d: %w", idx, err) + return st, fmt.Errorf("chunk %d: %w", idx, err) + } + if Hash(data) != hash { + return st, fmt.Errorf("chunk %d: fetched bytes do not match their hash", idx) } - if _, err := img.WriteAt(data, int64(idx)*m.ChunkSize); err != nil { - return err + if _, err := img.WriteAt(data, off); err != nil { + return st, err } + st.Wrote++ + st.BytesWritten += int64(len(data)) } - return nil + if cacheDir != "" { + _, _ = s.pruneCache(cacheDir) + } + return st, nil } // AllocatedChunks returns the chunk indices that contain allocated (non-hole) diff --git a/internal/cloudstore/cloudstore_test.go b/internal/cloudstore/cloudstore_test.go index 603b67e..ba27a5f 100644 --- a/internal/cloudstore/cloudstore_test.go +++ b/internal/cloudstore/cloudstore_test.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "testing" + "time" _ "github.com/rclone/rclone/backend/local" rfs "github.com/rclone/rclone/fs" @@ -75,7 +76,7 @@ func TestManifestRoundTrip(t *testing.T) { if err := g.Truncate(size); err != nil { t.Fatal(err) } - if err := s.ApplyManifest(ctx, m, g, filepath.Join(root, "cache")); err != nil { + if _, err := s.ApplyManifest(ctx, m, g, filepath.Join(root, "cache")); err != nil { t.Fatal(err) } for _, off := range []int64{3 * chunk, 200 * chunk} { @@ -129,3 +130,236 @@ func keys(m *Manifest) []string { } return out } + +// A cache hit must be hash-verified: garbage in the cache file (a torn write +// from a killed run) is re-fetched from the remote, never served. +func TestGetChunkVerifiesCacheHit(t *testing.T) { + ctx := context.Background() + s, root := testStore(t) + cache := filepath.Join(root, "cache") + + data := bytes.Repeat([]byte{0xAB}, 64) + h, err := s.PutChunk(ctx, data) + if err != nil { + t.Fatalf("put: %v", err) + } + if err := os.MkdirAll(cache, 0o755); err != nil { + t.Fatal(err) + } + // Poison the cache entry the way a crash mid-write would leave it. + if err := os.WriteFile(filepath.Join(cache, h), []byte("torn garbage"), 0o644); err != nil { + t.Fatal(err) + } + got, err := s.GetChunk(ctx, h, cache) + if err != nil { + t.Fatalf("get: %v", err) + } + if !bytes.Equal(got, data) { + t.Fatalf("served corrupt cache data: %q", got) + } + // The poisoned entry is repaired, so the next hit is clean. + got2, err := s.GetChunk(ctx, h, cache) + if err != nil || !bytes.Equal(got2, data) { + t.Fatalf("cache not repaired: %v %q", err, got2) + } +} + +// Corrupt bytes fetched from the remote must never be cached or returned. +func TestGetChunkRejectsCorruptRemote(t *testing.T) { + ctx := context.Background() + s, root := testStore(t) + cache := filepath.Join(root, "cache") + + if err := s.PutBytes(ctx, s.chunkPath("deadbeef"), []byte("not the hash")); err != nil { + t.Fatal(err) + } + if _, err := s.GetChunk(ctx, "deadbeef", cache); err == nil { + t.Fatal("expected an error for hash-mismatched remote bytes") + } + if _, err := os.Stat(filepath.Join(cache, "deadbeef")); !os.IsNotExist(err) { + t.Fatal("corrupt bytes were cached") + } +} + +// The second apply of the same manifest must write nothing. +func TestApplyManifestSkipsIdentical(t *testing.T) { + ctx := context.Background() + s, root := testStore(t) + + img := filepath.Join(root, "disk.img") + f, err := os.OpenFile(img, os.O_CREATE|os.O_RDWR, 0o644) + if err != nil { + t.Fatal(err) + } + defer f.Close() + const size = 4 << 20 + const chunk = 1 << 20 + if err := f.Truncate(size); err != nil { + t.Fatal(err) + } + if _, err := f.WriteAt(bytes.Repeat([]byte{7}, chunk), 0); err != nil { + t.Fatal(err) + } + if _, err := f.WriteAt(bytes.Repeat([]byte{9}, chunk), 2*chunk); err != nil { + t.Fatal(err) + } + m, err := s.BuildManifest(ctx, f, size, chunk) + if err != nil { + t.Fatal(err) + } + f.Close() + + // Materialize into a fresh image: first apply writes, second skips. + out := filepath.Join(root, "out.img") + g, err := os.OpenFile(out, os.O_CREATE|os.O_RDWR, 0o644) + if err != nil { + t.Fatal(err) + } + defer g.Close() + if err := g.Truncate(size); err != nil { + t.Fatal(err) + } + cache := filepath.Join(root, "cache") + st, err := s.ApplyManifest(ctx, m, g, cache) + if err != nil { + t.Fatal(err) + } + if st.Wrote != 2 { + t.Fatalf("first apply wrote %d chunks, want 2", st.Wrote) + } + st, err = s.ApplyManifest(ctx, m, g, cache) + if err != nil { + t.Fatal(err) + } + if st.Wrote != 0 { + t.Fatalf("second apply wrote %d chunks, want 0", st.Wrote) + } + if st.Skipped != 2 { + t.Fatalf("second apply skipped %d chunks, want 2", st.Skipped) + } +} + +// Kill -9 mid-apply leaves old-or-new chunks, never torn ones: a poisoned +// cache entry plus a corrupted disk range must heal on the next apply. +func TestApplyManifestHealsCorruption(t *testing.T) { + ctx := context.Background() + s, root := testStore(t) + + img := filepath.Join(root, "disk.img") + f, err := os.OpenFile(img, os.O_CREATE|os.O_RDWR, 0o644) + if err != nil { + t.Fatal(err) + } + defer f.Close() + const size = 2 << 20 + const chunk = 1 << 20 + if err := f.Truncate(size); err != nil { + t.Fatal(err) + } + want := bytes.Repeat([]byte{0x5A}, chunk) + if _, err := f.WriteAt(want, 0); err != nil { + t.Fatal(err) + } + m, err := s.BuildManifest(ctx, f, size, chunk) + if err != nil { + t.Fatal(err) + } + cache := filepath.Join(root, "cache") + if _, err := s.ApplyManifest(ctx, m, f, cache); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(cache, 0o755); err != nil { + t.Fatal(err) + } + + // Simulate the crash: garbage on disk and a torn cache file. + if _, err := f.WriteAt(bytes.Repeat([]byte{0xFF}, chunk), 0); err != nil { + t.Fatal(err) + } + var hash string + for _, h := range m.Chunks { + hash = h + } + if err := os.WriteFile(filepath.Join(cache, hash), []byte("partial"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := s.ApplyManifest(ctx, m, f, cache); err != nil { + t.Fatalf("apply after corruption: %v", err) + } + got := make([]byte, chunk) + if _, err := f.ReadAt(got, 0); err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Fatal("disk does not match the manifest after healing") + } +} + +// The cache is bounded: entries over the cap are reclaimed oldest-first. +func TestPruneCacheBounds(t *testing.T) { + s, root := testStore(t) + cache := filepath.Join(root, "cache") + if err := os.MkdirAll(cache, 0o755); err != nil { + t.Fatal(err) + } + writeSized := func(name string, n int, old bool) { + p := filepath.Join(cache, name) + if err := os.WriteFile(p, make([]byte, n), 0o644); err != nil { + t.Fatal(err) + } + if old { + past := time.Now().Add(-2 * time.Hour) + if err := os.Chtimes(p, past, past); err != nil { + t.Fatal(err) + } + } + } + writeSized("old-a", 100, true) + writeSized("old-b", 100, true) + writeSized("new", 100, false) + + reclaimed, err := s.PruneCache(cache, 150) + if err != nil { + t.Fatal(err) + } + if reclaimed < 100 { + t.Fatalf("reclaimed %d bytes, want >= 100", reclaimed) + } + if _, err := os.Stat(filepath.Join(cache, "new")); err != nil { + t.Fatal("newest entry was pruned before older ones") + } + var total int64 + entries, _ := os.ReadDir(cache) + for _, e := range entries { + if info, err := e.Info(); err == nil { + total += info.Size() + } + } + if total > 150 { + t.Fatalf("cache still %d bytes over a 150-byte cap", total) + } +} + +// With a local chunk dir, reads go straight through and no cache is created. +func TestLocalChunksReadThrough(t *testing.T) { + ctx := context.Background() + s, root := testStore(t) + s.SetLocalChunksDir(filepath.Join(root, "remote", "chunks")) + + data := bytes.Repeat([]byte{0xCD}, 32) + h, err := s.PutChunk(ctx, data) + if err != nil { + t.Fatalf("put: %v", err) + } + cache := filepath.Join(root, "cache-that-should-never-exist") + got, err := s.GetChunk(ctx, h, cache) + if err != nil { + t.Fatalf("get: %v", err) + } + if !bytes.Equal(got, data) { + t.Fatalf("read-through mismatch: %q", got) + } + if _, err := os.Stat(cache); !os.IsNotExist(err) { + t.Fatal("read-through created a cache copy") + } +} diff --git a/internal/volume/volume.go b/internal/volume/volume.go index ba25357..ffba322 100644 --- a/internal/volume/volume.go +++ b/internal/volume/volume.go @@ -9,7 +9,9 @@ package volume import ( "context" "crypto/rand" + "crypto/sha256" "encoding/hex" + "encoding/json" "errors" "fmt" "io" @@ -17,6 +19,7 @@ import ( "os/exec" "path/filepath" "sort" + "strconv" "strings" "sync" "time" @@ -47,10 +50,30 @@ type Store struct { baseImage string chunkSize int64 + // localRemote is set when the remote store is a directory on this host: + // chunk reads go straight to it and the cacheDir copy is skipped. + localRemote string + mu sync.Mutex attached map[string]string // volume name -> owner (VM id) + + applyMu sync.Mutex + lastApply cloudstore.ApplyStats + + // gcGrace skips chunks newer than this during garbage collection, so an + // in-flight commit's not-yet-referenced uploads are never deleted. + // Zero uses DefaultGCGrace. + gcGrace time.Duration + + cleanedCache sync.Once } +// DefaultGCGrace is how young a chunk may be and still be safe from the +// garbage collector. A commit uploads chunks before writing the manifest that +// references them; without a grace window GC could delete an upload that is +// seconds away from being referenced. +const DefaultGCGrace = time.Hour + // NewStore builds a Store. prefix defaults to "volumes"; chunkSize <= 0 uses // DefaultChunkSize. func NewStore(f rfs.Fs, prefix, localDir, baseImage string, chunkSize int64) *Store { @@ -71,6 +94,36 @@ func NewStore(f rfs.Fs, prefix, localDir, baseImage string, chunkSize int64) *St } } +// SetLocalRemoteDir tells the store its remote lives at dir on this host's +// disk, so chunk reads go straight through to it instead of being copied into +// the local cache first. dir is the remote root (the "volume-remote" dir), +// not the chunks dir itself. +func (s *Store) SetLocalRemoteDir(dir string) { + s.localRemote = dir + s.cs.SetLocalChunksDir(filepath.Join(dir, s.prefix, "chunks")) +} + +// SetGCGrace overrides how young a chunk may be and still be safe from GC. +// A negative grace disables the grace window (tests). +func (s *Store) SetGCGrace(d time.Duration) { s.gcGrace = d } + +func (s *Store) gcGracePeriod() time.Duration { + if s.gcGrace < 0 { + return 0 // tests: no grace + } + if s.gcGrace == 0 { + return DefaultGCGrace + } + return s.gcGrace +} + +// LastApply reports what the most recent EnsureLocal wrote vs skipped. +func (s *Store) LastApply() cloudstore.ApplyStats { + s.applyMu.Lock() + defer s.applyMu.Unlock() + return s.lastApply +} + // --- paths --- func (s *Store) metaRemote(name string) string { return s.prefix + "/" + name + "/meta.json" } @@ -129,7 +182,9 @@ func (s *Store) Create(ctx context.Context, name string, size int64, from string return nil, fmt.Errorf("source manifest: %w", err) } if srcImg := s.ImagePath(from); fileExists(srcImg) { - _ = cloneFile(srcImg, s.ImagePath(name)) + if err := cloneFile(srcImg, s.ImagePath(name)); err != nil { + return nil, err + } } meta.Size, meta.ChunkSize, meta.From = src.Size, src.ChunkSize, from man = m @@ -138,7 +193,9 @@ func (s *Store) Create(ctx context.Context, name string, size int64, from string if err != nil { return nil, err } - _ = cloneFile(s.baseImage, s.ImagePath(name)) + if err := cloneFile(s.baseImage, s.ImagePath(name)); err != nil { + return nil, err + } meta.Size, meta.ChunkSize, meta.From = bm.Size, bm.ChunkSize, "_base" man = bm default: @@ -181,6 +238,14 @@ func (s *Store) Create(ctx context.Context, name string, size int64, from string if err := s.cs.PutJSON(ctx, s.manifestRemote(name), man); err != nil { return nil, err } + // The local image provably matches the manifest in every path above: a + // clone copies the source bytes, and a fresh sparse image matches the + // empty manifest. (CreateFromSnapshot is the exception — it leaves a + // sparse image behind for EnsureLocal — and records no sidecar.) + // The grow path below only extends with holes, so it stays matched too. + if fileExists(s.ImagePath(name)) { + s.recordApplied(name, man) + } return meta, nil } @@ -399,11 +464,25 @@ func (s *Store) CreateFromSnapshot(ctx context.Context, name, snapID string) (*M // EnsureLocal materializes the volume's image locally from the manifest using // the shared chunk cache, and returns its path. +// +// Ranges already identical to the manifest are left untouched (per-chunk +// hash compare), so a create on an up-to-date image performs zero chunk +// writes. A sidecar records the last materialized manifest plus the image's +// size/mtime, so the fully-up-to-date case skips even the comparison reads. +// When the remote is a local dir, chunk reads go straight through to it and +// the legacy cache copy is removed. func (s *Store) EnsureLocal(ctx context.Context, name string) (string, error) { meta, err := s.Get(ctx, name) if err != nil { return "", err } + // Local backend: the cache is pure waste (every byte would live twice on + // one disk), so drop it once instead of feeding it. Concurrent applies + // already hold full chunk bytes in memory, so deleting files out from + // under them only turns into a re-fetch, never corruption. + if s.localRemote != "" { + s.cleanedCache.Do(func() { _ = os.RemoveAll(s.cacheDir()) }) + } man, err := s.Manifest(ctx, name) if err != nil { return "", err @@ -411,6 +490,10 @@ func (s *Store) EnsureLocal(ctx context.Context, name string) (string, error) { if err := os.MkdirAll(s.dirPath(name), 0o755); err != nil { return "", err } + if s.appliedCurrent(name, man) { + s.setLastApply(cloudstore.ApplyStats{}) + return s.ImagePath(name), nil + } f, err := os.OpenFile(s.ImagePath(name), os.O_CREATE|os.O_RDWR, 0o644) if err != nil { return "", err @@ -419,12 +502,92 @@ func (s *Store) EnsureLocal(ctx context.Context, name string) (string, error) { if err := f.Truncate(meta.Size); err != nil { return "", err } - if err := s.cs.ApplyManifest(ctx, man, f, s.cacheDir()); err != nil { + st, err := s.cs.ApplyManifest(ctx, man, f, s.cacheDir()) + if err != nil { return "", err } + s.setLastApply(st) + if err := f.Sync(); err != nil { + return "", err + } + s.recordApplied(name, man) return s.ImagePath(name), nil } +func (s *Store) setLastApply(st cloudstore.ApplyStats) { + s.applyMu.Lock() + defer s.applyMu.Unlock() + s.lastApply = st +} + +// appliedSidecar is the sidecar recording what disk.img currently holds. +type appliedSidecar struct { + Manifest string `json:"manifest"` // digest of the materialized manifest + Size int64 `json:"size"` + Mtime int64 `json:"mtime_ns"` +} + +func (s *Store) sidecarPath(name string) string { + return filepath.Join(s.dirPath(name), ".applied.json") +} + +// manifestDigest is the content hash of a manifest's canonical form. +func manifestDigest(m *cloudstore.Manifest) string { + keys := make([]string, 0, len(m.Chunks)) + for k := range m.Chunks { + keys = append(keys, k) + } + sort.Strings(keys) + var sb strings.Builder + sb.WriteString(strconv.FormatInt(m.ChunkSize, 10)) + sb.WriteByte('/') + sb.WriteString(strconv.FormatInt(m.Size, 10)) + for _, k := range keys { + sb.WriteByte('/') + sb.WriteString(k) + sb.WriteByte('=') + sb.WriteString(m.Chunks[k]) + } + sum := sha256.Sum256([]byte(sb.String())) + return hex.EncodeToString(sum[:]) +} + +// appliedCurrent reports whether disk.img provably matches man: the sidecar +// names the same manifest digest and the image is byte-identical in size and +// mtime to when the sidecar was written. Any guest run or external write +// changes mtime and forces the full per-chunk comparison. +func (s *Store) appliedCurrent(name string, man *cloudstore.Manifest) bool { + raw, err := os.ReadFile(s.sidecarPath(name)) + if err != nil { + return false + } + var sc appliedSidecar + if err := json.Unmarshal(raw, &sc); err != nil { + return false + } + if sc.Manifest == "" || sc.Manifest != manifestDigest(man) { + return false + } + fi, err := os.Stat(s.ImagePath(name)) + if err != nil || fi.Size() != sc.Size { + return false + } + return fi.ModTime().UnixNano() == sc.Mtime +} + +func (s *Store) recordApplied(name string, man *cloudstore.Manifest) { + fi, err := os.Stat(s.ImagePath(name)) + if err != nil { + return + } + sc := appliedSidecar{Manifest: manifestDigest(man), Size: fi.Size(), Mtime: fi.ModTime().UnixNano()} + raw, err := json.Marshal(sc) + if err != nil { + return + } + _ = os.WriteFile(s.sidecarPath(name), raw, 0o644) +} + // Commit uploads the local image's allocated chunks and refreshes the manifest. func (s *Store) Commit(ctx context.Context, name string) error { meta, err := s.Get(ctx, name) @@ -440,7 +603,107 @@ func (s *Store) Commit(ctx context.Context, name string) error { if err != nil { return err } - return s.cs.PutJSON(ctx, s.manifestRemote(name), m) + if err := s.cs.PutJSON(ctx, s.manifestRemote(name), m); err != nil { + return err + } + // The manifest was just built from this exact disk, so they match. + s.recordApplied(name, m) + return nil +} + +// --- garbage collection --- + +// GCReport describes what a garbage-collection pass found and did. +type GCReport struct { + Volumes int `json:"volumes"` + Chunks int `json:"chunks"` + Orphans []string `json:"orphans"` + Reclaimed int64 `json:"reclaimed_bytes"` + Deleted int `json:"deleted"` + DryRun bool `json:"dry_run"` +} + +// ReferencedHashes collects the chunk hashes named by every volume manifest, +// every snapshot manifest, and the base-image manifest. Anything stored but +// not in this set is unreachable and safe to delete — including while a +// desktop is attached, because the attached volume's own manifest keeps its +// chunks referenced. +func (s *Store) ReferencedHashes(ctx context.Context) (map[string]bool, error) { + ref := map[string]bool{} + vols, err := s.List(ctx) + if err != nil { + return nil, err + } + for _, v := range vols { + man, err := s.Manifest(ctx, v.Name) + if err != nil { + continue + } + for _, h := range man.Chunks { + ref[h] = true + } + } + snaps, err := s.ListSnapshots(ctx) + if err != nil { + return nil, err + } + for _, sn := range snaps { + man := &cloudstore.Manifest{Chunks: map[string]string{}} + if err := s.cs.GetJSON(ctx, s.snapshotManifestRemote(sn.ID), man); err != nil { + continue + } + for _, h := range man.Chunks { + ref[h] = true + } + } + base := &cloudstore.Manifest{Chunks: map[string]string{}} + if err := s.cs.GetJSON(ctx, s.baseManifestRemote(), base); err == nil { + for _, h := range base.Chunks { + ref[h] = true + } + } + return ref, nil +} + +// GC deletes remote chunks no manifest or snapshot references. Chunks newer +// than the grace period are spared, so an in-flight commit's not-yet-linked +// uploads survive. Dry-run lists without deleting. +func (s *Store) GC(ctx context.Context, dryRun bool) (GCReport, error) { + var rep GCReport + rep.DryRun = dryRun + ref, err := s.ReferencedHashes(ctx) + if err != nil { + return rep, err + } + chunks, err := s.cs.ListChunks(ctx) + if err != nil { + return rep, err + } + rep.Chunks = len(chunks) + grace := s.gcGracePeriod() + cutoff := time.Now().Add(-grace) + for _, c := range chunks { + if ref[c.Hash] { + continue + } + if c.ModTime.After(cutoff) { + continue + } + rep.Orphans = append(rep.Orphans, c.Hash) + rep.Reclaimed += c.Size + } + sort.Strings(rep.Orphans) + vols, _ := s.List(ctx) + rep.Volumes = len(vols) + if dryRun { + return rep, nil + } + for _, h := range rep.Orphans { + if err := s.cs.DeleteChunk(ctx, h); err == nil { + rep.Deleted++ + } + } + return rep, nil } // --- single-attach lock (in-process) --- diff --git a/internal/volume/volume_test.go b/internal/volume/volume_test.go index 5e0cea6..bde6521 100644 --- a/internal/volume/volume_test.go +++ b/internal/volume/volume_test.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "testing" + "time" _ "github.com/rclone/rclone/backend/local" rfs "github.com/rclone/rclone/fs" @@ -191,3 +192,240 @@ func TestSparseManifest(t *testing.T) { t.Fatalf("expected exactly 1 chunk, got %d", len(m.Chunks)) } } + +// The second EnsureLocal on an unchanged image must write zero chunks: the +// first call records a sidecar of what disk.img holds, the second takes the +// fast path without even comparing. +func TestEnsureLocalSkipsIdentical(t *testing.T) { + ctx := context.Background() + f, root := testRemote(t) + st := NewStore(f, "volumes", filepath.Join(root, "local"), "", 16) + + if _, err := st.Create(ctx, "v", 64, ""); err != nil { + t.Fatalf("create: %v", err) + } + mustWriteAt(t, st.ImagePath("v"), 0, []byte("hello-world-1234")) + mustWriteAt(t, st.ImagePath("v"), 32, []byte("second-chunk-data")) + if err := st.Commit(ctx, "v"); err != nil { + t.Fatalf("commit: %v", err) + } + + // Fresh store, empty local dir: materialize from the remote. + st2 := NewStore(f, "volumes", filepath.Join(root, "local2"), "", 16) + img, err := st2.EnsureLocal(ctx, "v") + if err != nil { + t.Fatalf("ensure: %v", err) + } + want, _ := os.ReadFile(st.ImagePath("v")) + got, _ := os.ReadFile(img) + if !bytes.Equal(want, got) { + t.Fatal("materialized image differs") + } + if st2.LastApply().Wrote == 0 { + t.Fatal("first EnsureLocal should have written chunks") + } + + // Delete the sidecar to force the full per-chunk comparison: still zero + // writes, every chunk skipped. + if err := os.Remove(filepath.Join(root, "local2", "v", ".applied.json")); err != nil { + t.Fatal(err) + } + if _, err := st2.EnsureLocal(ctx, "v"); err != nil { + t.Fatalf("ensure again: %v", err) + } + applied := st2.LastApply() + if applied.Wrote != 0 { + t.Fatalf("second EnsureLocal wrote %d chunks, want 0", applied.Wrote) + } + if applied.Skipped == 0 { + t.Fatal("second EnsureLocal compared nothing") + } + + // Untouched since: the sidecar fast path skips even the comparison reads. + if _, err := st2.EnsureLocal(ctx, "v"); err != nil { + t.Fatalf("ensure third: %v", err) + } + if st := st2.LastApply(); st.Wrote != 0 || st.Skipped != 0 { + t.Fatalf("fast path did work: %+v", st) + } +} + +// Kill -9 mid-EnsureLocal leaves old-or-new chunks plus possibly a torn cache +// file. The next EnsureLocal must heal both: poison the cache and corrupt the +// disk, then verify the image matches the manifest. +func TestEnsureLocalHealsCrash(t *testing.T) { + ctx := context.Background() + f, root := testRemote(t) + st := NewStore(f, "volumes", filepath.Join(root, "local"), "", 16) + + if _, err := st.Create(ctx, "v", 48, ""); err != nil { + t.Fatalf("create: %v", err) + } + want := []byte("0123456789abcdef") + mustWriteAt(t, st.ImagePath("v"), 0, want) + if err := st.Commit(ctx, "v"); err != nil { + t.Fatalf("commit: %v", err) + } + man, err := st.Manifest(ctx, "v") + if err != nil { + t.Fatal(err) + } + var hash string + for _, h := range man.Chunks { + hash = h + } + + // The crash: garbage straight into the image and a torn cache file. + mustWriteAt(t, st.ImagePath("v"), 0, bytes.Repeat([]byte{0xFF}, 16)) + cache := filepath.Join(root, "local", "chunks") + if err := os.MkdirAll(cache, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(cache, hash), []byte("partial"), 0o644); err != nil { + t.Fatal(err) + } + + if _, err := st.EnsureLocal(ctx, "v"); err != nil { + t.Fatalf("ensure after crash: %v", err) + } + got := make([]byte, 16) + rf, err := os.Open(st.ImagePath("v")) + if err != nil { + t.Fatal(err) + } + defer rf.Close() + if _, err := rf.ReadAt(got, 0); err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Fatalf("disk not healed: %q", got) + } + if st.LastApply().Wrote == 0 { + t.Fatal("healing apply should have rewritten the corrupt chunk") + } +} + +// GC must list exactly the orphaned hashes on dry-run and reclaim them for +// real, while chunks referenced by any live manifest or snapshot survive. +func TestGCReclaimsOrphans(t *testing.T) { + ctx := context.Background() + f, root := testRemote(t) + st := NewStore(f, "volumes", filepath.Join(root, "local"), "", 16) + st.SetGCGrace(-1) // no grace window in tests + + if _, err := st.Create(ctx, "v", 64, ""); err != nil { + t.Fatalf("create: %v", err) + } + mustWriteAt(t, st.ImagePath("v"), 0, []byte("chunk-zero-data!")) + mustWriteAt(t, st.ImagePath("v"), 32, []byte("chunk-two-data!!")) + if err := st.Commit(ctx, "v"); err != nil { + t.Fatalf("commit: %v", err) + } + man, err := st.Manifest(ctx, "v") + if err != nil { + t.Fatal(err) + } + chunkTwo := man.Chunks["2"] + + // A snapshot pins chunkTwo even after the volume stops referencing it. + if _, err := st.Snapshot(ctx, "v", "snap1"); err != nil { + t.Fatalf("snapshot: %v", err) + } + mustWriteAt(t, st.ImagePath("v"), 32, bytes.Repeat([]byte{0}, 16)) + if err := st.Commit(ctx, "v"); err != nil { + t.Fatalf("commit 2: %v", err) + } + + // An orphan from a failed commit: uploaded, never referenced. + orphan, err := st.cs.PutChunk(ctx, []byte("orphan-payload!")) + if err != nil { + t.Fatalf("orphan put: %v", err) + } + + rep, err := st.GC(ctx, true) + if err != nil { + t.Fatalf("gc dry-run: %v", err) + } + if len(rep.Orphans) != 1 || rep.Orphans[0] != orphan { + t.Fatalf("dry-run orphans = %v, want exactly [%s]", rep.Orphans, orphan) + } + + // Drop the snapshot: chunkTwo joins the orphan set. + if err := st.DeleteSnapshot(ctx, "snap1"); err != nil { + t.Fatal(err) + } + rep, err = st.GC(ctx, true) + if err != nil { + t.Fatalf("gc dry-run 2: %v", err) + } + if len(rep.Orphans) != 2 { + t.Fatalf("dry-run orphans = %v, want 2", rep.Orphans) + } + + rep, err = st.GC(ctx, false) + if err != nil { + t.Fatalf("gc: %v", err) + } + if rep.Deleted != 2 { + t.Fatalf("deleted %d chunks, want 2", rep.Deleted) + } + // Referenced chunks survive; orphans are gone. + if !st.cs.Exists(ctx, "volumes/chunks/"+man.Chunks["0"]) { + t.Fatal("referenced chunk was deleted") + } + if st.cs.Exists(ctx, "volumes/chunks/"+orphan) { + t.Fatal("orphan survived gc") + } + if st.cs.Exists(ctx, "volumes/chunks/"+chunkTwo) { + t.Fatal("snapshot-released chunk survived gc") + } + + // Young chunks are spared by the grace window (in-flight commits). + st.SetGCGrace(time.Hour) + fresh, err := st.cs.PutChunk(ctx, []byte("fresh-payload!!")) + if err != nil { + t.Fatal(err) + } + rep, err = st.GC(ctx, true) + if err != nil { + t.Fatal(err) + } + for _, h := range rep.Orphans { + if h == fresh { + t.Fatal("grace window did not protect a fresh chunk") + } + } +} + +// With a local remote, EnsureLocal must never populate the cache dir: every +// byte would otherwise live twice on one disk. +func TestLocalRemoteSkipsCache(t *testing.T) { + ctx := context.Background() + f, root := testRemote(t) + remote := filepath.Join(root, "remote") + + st := NewStore(f, "volumes", filepath.Join(root, "local1"), "", 16) + st.SetLocalRemoteDir(remote) + if _, err := st.Create(ctx, "v", 48, ""); err != nil { + t.Fatalf("create: %v", err) + } + mustWriteAt(t, st.ImagePath("v"), 0, []byte("local-data-12345")) + if err := st.Commit(ctx, "v"); err != nil { + t.Fatalf("commit: %v", err) + } + + st2 := NewStore(f, "volumes", filepath.Join(root, "local2"), "", 16) + st2.SetLocalRemoteDir(remote) + img, err := st2.EnsureLocal(ctx, "v") + if err != nil { + t.Fatalf("ensure: %v", err) + } + want, _ := os.ReadFile(st.ImagePath("v")) + got, _ := os.ReadFile(img) + if !bytes.Equal(want, got) { + t.Fatal("read-through image differs") + } + if _, err := os.Stat(filepath.Join(root, "local2", "chunks")); !os.IsNotExist(err) { + t.Fatal("local backend populated the cache dir") + } +}