From 722f087b1a580f72c8b0c69fcf99c6a069ed34dd Mon Sep 17 00:00:00 2001 From: coolstackcoder Date: Mon, 26 Oct 2026 20:59:51 +0530 Subject: [PATCH] raft: test followers killed during InstallSnapshot Kill a lagging follower at three points of a snapshot install (torn temp file, after the snapshot is saved, after the log is rewritten), let the leader compact past that snapshot, then restart the follower from disk. Recovery holds at all three points. Also fsync the directory after the atomic rename in writeAtomic, so a power loss cannot keep a compacted log without its snapshot. --- internal/raft/raft_test.go | 51 ++++--- internal/raft/snapshot_crash_test.go | 220 +++++++++++++++++++++++++++ internal/raft/storage.go | 23 ++- 3 files changed, 276 insertions(+), 18 deletions(-) create mode 100644 internal/raft/snapshot_crash_test.go diff --git a/internal/raft/raft_test.go b/internal/raft/raft_test.go index 068d59f..5e5201e 100644 --- a/internal/raft/raft_test.go +++ b/internal/raft/raft_test.go @@ -61,7 +61,12 @@ func (t memTransport) InstallSnapshot(_ context.Context, to string, r *pb.Snapsh if err != nil { return nil, err } - return n.HandleInstallSnapshot(r), nil + resp := n.HandleInstallSnapshot(r) + // A node that died while handling the RPC never gets to reply. + if _, err := t.nw.target(t.from, to); err != nil { + return nil, err + } + return resp, nil } // listFSM records every applied command so tests can compare histories. @@ -96,29 +101,39 @@ func (f *listFSM) history() []string { } type cluster struct { - t *testing.T - nw *network - ids []string - nodes map[string]*Node - fsms map[string]*listFSM - storages map[string]*MemoryStorage - snapAt uint64 + t *testing.T + nw *network + ids []string + nodes map[string]*Node + fsms map[string]*listFSM + snapAt uint64 + // open returns the storage a (re)started node recovers from. + open func(id string) Storage } func newCluster(t *testing.T, size int, snapshotThreshold uint64) *cluster { + mem := map[string]*MemoryStorage{} + return newClusterWith(t, size, snapshotThreshold, func(id string) Storage { + if mem[id] == nil { + mem[id] = NewMemoryStorage() + } + return mem[id] + }) +} + +func newClusterWith(t *testing.T, size int, snapshotThreshold uint64, open func(id string) Storage) *cluster { c := &cluster{ - t: t, - nw: &network{nodes: map[string]*Node{}, down: map[string]bool{}}, - nodes: map[string]*Node{}, - fsms: map[string]*listFSM{}, - storages: map[string]*MemoryStorage{}, - snapAt: snapshotThreshold, + t: t, + nw: &network{nodes: map[string]*Node{}, down: map[string]bool{}}, + nodes: map[string]*Node{}, + fsms: map[string]*listFSM{}, + snapAt: snapshotThreshold, + open: open, } for i := 1; i <= size; i++ { c.ids = append(c.ids, fmt.Sprintf("n%d", i)) } for _, id := range c.ids { - c.storages[id] = NewMemoryStorage() c.start(id) } t.Cleanup(func() { @@ -129,7 +144,9 @@ func newCluster(t *testing.T, size int, snapshotThreshold uint64) *cluster { return c } -func (c *cluster) start(id string) { +func (c *cluster) start(id string) { c.startWith(id, c.open(id)) } + +func (c *cluster) startWith(id string, storage Storage) { peers := map[string]string{} for _, p := range c.ids { peers[p] = p // the in-memory transport addresses nodes by id @@ -142,7 +159,7 @@ func (c *cluster) start(id string) { ElectionTimeoutMin: 120 * time.Millisecond, ElectionTimeoutMax: 240 * time.Millisecond, SnapshotThreshold: c.snapAt, - Storage: c.storages[id], + Storage: storage, Transport: memTransport{nw: c.nw, from: id}, FSM: fsm, Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), diff --git a/internal/raft/snapshot_crash_test.go b/internal/raft/snapshot_crash_test.go new file mode 100644 index 0000000..73f91a6 --- /dev/null +++ b/internal/raft/snapshot_crash_test.go @@ -0,0 +1,220 @@ +package raft + +import ( + "fmt" + "os" + "path/filepath" + "sync" + "testing" + "time" + + pb "github.com/coolstack-coder/nebulaos/gen/nebula/v1" +) + +// crashPoint is where in InstallSnapshot a follower process gets killed. +type crashPoint int + +const ( + // The snapshot temp file is half written; the old snapshot is untouched. + crashMidSnapshotWrite crashPoint = iota + // The new snapshot is on disk but the log has not been rewritten yet. + crashAfterSnapshotSave + // Snapshot and log are both on disk; the FSM has not been restored yet. + crashAfterLogRewrite +) + +func (p crashPoint) String() string { + switch p { + case crashMidSnapshotWrite: + return "mid_snapshot_write" + case crashAfterSnapshotSave: + return "after_snapshot_save" + default: + return "after_log_rewrite" + } +} + +// crashingStorage wraps a FileStorage and simulates a SIGKILL at a chosen +// point of a snapshot install. Writes up to the crash point reach the disk, +// every write after it is dropped as if the process had died, and onCrash +// cuts the node off so its in-flight reply is never delivered. +type crashingStorage struct { + Storage + dir string + at crashPoint + onCrash func(snapIndex uint64) + + mu sync.Mutex + dead bool + snapIdx uint64 +} + +func (s *crashingStorage) die() { + s.dead = true + s.onCrash(s.snapIdx) +} + +func (s *crashingStorage) SaveSnapshot(index, term uint64, data []byte) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.dead { + return nil + } + s.snapIdx = index + switch s.at { + case crashMidSnapshotWrite: + torn := data[:len(data)/2] + if err := os.WriteFile(filepath.Join(s.dir, "snapshot.bin.tmp"), torn, 0o644); err != nil { + return err + } + s.die() + return nil + case crashAfterSnapshotSave: + if err := s.Storage.SaveSnapshot(index, term, data); err != nil { + return err + } + s.die() + return nil + } + return s.Storage.SaveSnapshot(index, term, data) +} + +func (s *crashingStorage) RewriteLog(entries []*pb.LogEntry) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.dead { + return nil + } + if err := s.Storage.RewriteLog(entries); err != nil { + return err + } + if s.at == crashAfterLogRewrite && s.snapIdx != 0 { + s.die() + } + return nil +} + +func (s *crashingStorage) SaveState(term uint64, votedFor string) error { + if s.isDead() { + return nil + } + return s.Storage.SaveState(term, votedFor) +} + +func (s *crashingStorage) AppendLog(entries []*pb.LogEntry) error { + if s.isDead() { + return nil + } + return s.Storage.AppendLog(entries) +} + +func (s *crashingStorage) AppendLogNoSync(entries []*pb.LogEntry) error { + if s.isDead() { + return nil + } + return s.Storage.AppendLogNoSync(entries) +} + +func (s *crashingStorage) isDead() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.dead +} + +// newFileCluster runs every node on a real FileStorage under root. Writes +// are not fsynced: a SIGKILL leaves the page cache intact, so fsync does not +// change what a killed process leaves behind. +func newFileCluster(t *testing.T, size int, snapshotThreshold uint64, root string) *cluster { + return newClusterWith(t, size, snapshotThreshold, func(id string) Storage { + s, err := NewFileStorage(filepath.Join(root, id), false) + if err != nil { + t.Fatal(err) + } + return s + }) +} + +// TestFollowerKilledDuringSnapshotInstall kills a lagging follower at each +// point of InstallSnapshot, lets the leader compact past the snapshot it was +// installing, then restarts it from whatever reached the disk. The follower +// must come back and converge on exactly the leader's state. +func TestFollowerKilledDuringSnapshotInstall(t *testing.T) { + for _, at := range []crashPoint{crashMidSnapshotWrite, crashAfterSnapshotSave, crashAfterLogRewrite} { + t.Run(at.String(), func(t *testing.T) { testKilledDuringInstall(t, at) }) + } +} + +func testKilledDuringInstall(t *testing.T, at crashPoint) { + root := t.TempDir() + c := newFileCluster(t, 3, 20, root) + l := c.leader() + var victim string + for _, id := range c.ids { + if id != l { + victim = id + break + } + } + write := func(from, to int) { + for i := from; i < to; i++ { + if err := c.apply(l, fmt.Sprintf("v%d", i)); err != nil { + t.Fatal(err) + } + } + } + + // Fall far enough behind that the leader has to send a snapshot. + c.setDown(victim, true) + write(0, 60) + + // Bring the follower back on storage that dies partway through the + // install. + c.crash(victim) + crashed := make(chan uint64, 1) + c.startWith(victim, &crashingStorage{ + Storage: c.open(victim), + dir: filepath.Join(root, victim), + at: at, + onCrash: func(idx uint64) { + c.setDown(victim, true) + crashed <- idx + }, + }) + c.setDown(victim, false) + var installing uint64 + select { + case installing = <-crashed: + case <-time.After(5 * time.Second): + t.Fatal("follower never started installing a snapshot") + } + c.crash(victim) + + // While it is down the leader moves on and compacts past the snapshot + // the follower was installing, so the restart cannot just resume it. + write(60, 120) + if s := c.nodes[l].Status(); s.SnapshotIndex <= installing { + t.Fatalf("leader did not compact past index %d (snapshot at %d)", installing, s.SnapshotIndex) + } + + // Restart from disk while still cut off and check what survived: a + // snapshot that was fully saved must be kept, a torn one ignored. + c.start(victim) + recovered := c.nodes[victim].Status().SnapshotIndex + if at == crashMidSnapshotWrite && recovered >= installing { + t.Fatalf("torn snapshot %d was loaded (recovered %d)", installing, recovered) + } + if at != crashMidSnapshotWrite && recovered != installing { + t.Fatalf("saved snapshot %d was lost on restart (recovered %d)", installing, recovered) + } + c.checkPrefixes() + + c.setDown(victim, false) + c.waitApplied(120, c.ids...) + c.checkPrefixes() + if c.nodes[victim].Status().Stats.SnapshotsLoaded == 0 { + t.Fatal("expected the restarted follower to catch up via a snapshot") + } + if got, want := c.fsms[victim].history(), c.fsms[l].history(); len(got) != len(want) { + t.Fatalf("follower applied %d commands, leader %d", len(got), len(want)) + } +} diff --git a/internal/raft/storage.go b/internal/raft/storage.go index ef173b4..3bd6da4 100644 --- a/internal/raft/storage.go +++ b/internal/raft/storage.go @@ -254,7 +254,28 @@ func (s *FileStorage) writeAtomic(name string, b []byte) error { if err := f.Close(); err != nil { return err } - return os.Rename(tmp, s.path(name)) + if err := os.Rename(tmp, s.path(name)); err != nil { + return err + } + if !s.sync { + return nil + } + // The rename is only durable once the directory is synced. Without this + // a power loss after compaction could keep the rewritten log but lose + // the snapshot it starts from. + return syncDir(s.dir) +} + +func syncDir(dir string) error { + d, err := os.Open(dir) + if err != nil { + return err + } + if err := d.Sync(); err != nil { + d.Close() + return err + } + return d.Close() } func (s *FileStorage) Close() error {