Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 34 additions & 17 deletions internal/raft/raft_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,12 @@ func (t memTransport) InstallSnapshot(_ context.Context, to string, r *pb.Snapsh
if err != nil {
return nil, err
}
return n.HandleInstallSnapshot(r), nil
resp := n.HandleInstallSnapshot(r)
// A node that died while handling the RPC never gets to reply.
if _, err := t.nw.target(t.from, to); err != nil {
return nil, err
}
return resp, nil
}

// listFSM records every applied command so tests can compare histories.
Expand Down Expand Up @@ -96,29 +101,39 @@ func (f *listFSM) history() []string {
}

type cluster struct {
t *testing.T
nw *network
ids []string
nodes map[string]*Node
fsms map[string]*listFSM
storages map[string]*MemoryStorage
snapAt uint64
t *testing.T
nw *network
ids []string
nodes map[string]*Node
fsms map[string]*listFSM
snapAt uint64
// open returns the storage a (re)started node recovers from.
open func(id string) Storage
}

func newCluster(t *testing.T, size int, snapshotThreshold uint64) *cluster {
mem := map[string]*MemoryStorage{}
return newClusterWith(t, size, snapshotThreshold, func(id string) Storage {
if mem[id] == nil {
mem[id] = NewMemoryStorage()
}
return mem[id]
})
}

func newClusterWith(t *testing.T, size int, snapshotThreshold uint64, open func(id string) Storage) *cluster {
c := &cluster{
t: t,
nw: &network{nodes: map[string]*Node{}, down: map[string]bool{}},
nodes: map[string]*Node{},
fsms: map[string]*listFSM{},
storages: map[string]*MemoryStorage{},
snapAt: snapshotThreshold,
t: t,
nw: &network{nodes: map[string]*Node{}, down: map[string]bool{}},
nodes: map[string]*Node{},
fsms: map[string]*listFSM{},
snapAt: snapshotThreshold,
open: open,
}
for i := 1; i <= size; i++ {
c.ids = append(c.ids, fmt.Sprintf("n%d", i))
}
for _, id := range c.ids {
c.storages[id] = NewMemoryStorage()
c.start(id)
}
t.Cleanup(func() {
Expand All @@ -129,7 +144,9 @@ func newCluster(t *testing.T, size int, snapshotThreshold uint64) *cluster {
return c
}

func (c *cluster) start(id string) {
func (c *cluster) start(id string) { c.startWith(id, c.open(id)) }

func (c *cluster) startWith(id string, storage Storage) {
peers := map[string]string{}
for _, p := range c.ids {
peers[p] = p // the in-memory transport addresses nodes by id
Expand All @@ -142,7 +159,7 @@ func (c *cluster) start(id string) {
ElectionTimeoutMin: 120 * time.Millisecond,
ElectionTimeoutMax: 240 * time.Millisecond,
SnapshotThreshold: c.snapAt,
Storage: c.storages[id],
Storage: storage,
Transport: memTransport{nw: c.nw, from: id},
FSM: fsm,
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
Expand Down
220 changes: 220 additions & 0 deletions internal/raft/snapshot_crash_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,220 @@
package raft

import (
"fmt"
"os"
"path/filepath"
"sync"
"testing"
"time"

pb "github.com/coolstack-coder/nebulaos/gen/nebula/v1"
)

// crashPoint is where in InstallSnapshot a follower process gets killed.
type crashPoint int

const (
// The snapshot temp file is half written; the old snapshot is untouched.
crashMidSnapshotWrite crashPoint = iota
// The new snapshot is on disk but the log has not been rewritten yet.
crashAfterSnapshotSave
// Snapshot and log are both on disk; the FSM has not been restored yet.
crashAfterLogRewrite
)

func (p crashPoint) String() string {
switch p {
case crashMidSnapshotWrite:
return "mid_snapshot_write"
case crashAfterSnapshotSave:
return "after_snapshot_save"
default:
return "after_log_rewrite"
}
}

// crashingStorage wraps a FileStorage and simulates a SIGKILL at a chosen
// point of a snapshot install. Writes up to the crash point reach the disk,
// every write after it is dropped as if the process had died, and onCrash
// cuts the node off so its in-flight reply is never delivered.
type crashingStorage struct {
Storage
dir string
at crashPoint
onCrash func(snapIndex uint64)

mu sync.Mutex
dead bool
snapIdx uint64
}

func (s *crashingStorage) die() {
s.dead = true
s.onCrash(s.snapIdx)
}

func (s *crashingStorage) SaveSnapshot(index, term uint64, data []byte) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.dead {
return nil
}
s.snapIdx = index
switch s.at {
case crashMidSnapshotWrite:
torn := data[:len(data)/2]
if err := os.WriteFile(filepath.Join(s.dir, "snapshot.bin.tmp"), torn, 0o644); err != nil {
return err
}
s.die()
return nil
case crashAfterSnapshotSave:
if err := s.Storage.SaveSnapshot(index, term, data); err != nil {
return err
}
s.die()
return nil
}
return s.Storage.SaveSnapshot(index, term, data)
}

func (s *crashingStorage) RewriteLog(entries []*pb.LogEntry) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.dead {
return nil
}
if err := s.Storage.RewriteLog(entries); err != nil {
return err
}
if s.at == crashAfterLogRewrite && s.snapIdx != 0 {
s.die()
}
return nil
}

func (s *crashingStorage) SaveState(term uint64, votedFor string) error {
if s.isDead() {
return nil
}
return s.Storage.SaveState(term, votedFor)
}

func (s *crashingStorage) AppendLog(entries []*pb.LogEntry) error {
if s.isDead() {
return nil
}
return s.Storage.AppendLog(entries)
}

func (s *crashingStorage) AppendLogNoSync(entries []*pb.LogEntry) error {
if s.isDead() {
return nil
}
return s.Storage.AppendLogNoSync(entries)
}

func (s *crashingStorage) isDead() bool {
s.mu.Lock()
defer s.mu.Unlock()
return s.dead
}

// newFileCluster runs every node on a real FileStorage under root. Writes
// are not fsynced: a SIGKILL leaves the page cache intact, so fsync does not
// change what a killed process leaves behind.
func newFileCluster(t *testing.T, size int, snapshotThreshold uint64, root string) *cluster {
return newClusterWith(t, size, snapshotThreshold, func(id string) Storage {
s, err := NewFileStorage(filepath.Join(root, id), false)
if err != nil {
t.Fatal(err)
}
return s
})
}

// TestFollowerKilledDuringSnapshotInstall kills a lagging follower at each
// point of InstallSnapshot, lets the leader compact past the snapshot it was
// installing, then restarts it from whatever reached the disk. The follower
// must come back and converge on exactly the leader's state.
func TestFollowerKilledDuringSnapshotInstall(t *testing.T) {
for _, at := range []crashPoint{crashMidSnapshotWrite, crashAfterSnapshotSave, crashAfterLogRewrite} {
t.Run(at.String(), func(t *testing.T) { testKilledDuringInstall(t, at) })
}
}

func testKilledDuringInstall(t *testing.T, at crashPoint) {
root := t.TempDir()
c := newFileCluster(t, 3, 20, root)
l := c.leader()
var victim string
for _, id := range c.ids {
if id != l {
victim = id
break
}
}
write := func(from, to int) {
for i := from; i < to; i++ {
if err := c.apply(l, fmt.Sprintf("v%d", i)); err != nil {
t.Fatal(err)
}
}
}

// Fall far enough behind that the leader has to send a snapshot.
c.setDown(victim, true)
write(0, 60)

// Bring the follower back on storage that dies partway through the
// install.
c.crash(victim)
crashed := make(chan uint64, 1)
c.startWith(victim, &crashingStorage{
Storage: c.open(victim),
dir: filepath.Join(root, victim),
at: at,
onCrash: func(idx uint64) {
c.setDown(victim, true)
crashed <- idx
},
})
c.setDown(victim, false)
var installing uint64
select {
case installing = <-crashed:
case <-time.After(5 * time.Second):
t.Fatal("follower never started installing a snapshot")
}
c.crash(victim)

// While it is down the leader moves on and compacts past the snapshot
// the follower was installing, so the restart cannot just resume it.
write(60, 120)
if s := c.nodes[l].Status(); s.SnapshotIndex <= installing {
t.Fatalf("leader did not compact past index %d (snapshot at %d)", installing, s.SnapshotIndex)
}

// Restart from disk while still cut off and check what survived: a
// snapshot that was fully saved must be kept, a torn one ignored.
c.start(victim)
recovered := c.nodes[victim].Status().SnapshotIndex
if at == crashMidSnapshotWrite && recovered >= installing {
t.Fatalf("torn snapshot %d was loaded (recovered %d)", installing, recovered)
}
if at != crashMidSnapshotWrite && recovered != installing {
t.Fatalf("saved snapshot %d was lost on restart (recovered %d)", installing, recovered)
}
c.checkPrefixes()

c.setDown(victim, false)
c.waitApplied(120, c.ids...)
c.checkPrefixes()
if c.nodes[victim].Status().Stats.SnapshotsLoaded == 0 {
t.Fatal("expected the restarted follower to catch up via a snapshot")
}
if got, want := c.fsms[victim].history(), c.fsms[l].history(); len(got) != len(want) {
t.Fatalf("follower applied %d commands, leader %d", len(got), len(want))
}
}
23 changes: 22 additions & 1 deletion internal/raft/storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,28 @@ func (s *FileStorage) writeAtomic(name string, b []byte) error {
if err := f.Close(); err != nil {
return err
}
return os.Rename(tmp, s.path(name))
if err := os.Rename(tmp, s.path(name)); err != nil {
return err
}
if !s.sync {
return nil
}
// The rename is only durable once the directory is synced. Without this
// a power loss after compaction could keep the rewritten log but lose
// the snapshot it starts from.
return syncDir(s.dir)
}

func syncDir(dir string) error {
d, err := os.Open(dir)
if err != nil {
return err
}
if err := d.Sync(); err != nil {
d.Close()
return err
}
return d.Close()
}

func (s *FileStorage) Close() error {
Expand Down
Loading