diff --git a/client.go b/client.go index 0caa9e1..d4da38d 100644 --- a/client.go +++ b/client.go @@ -136,6 +136,20 @@ func (c *Client) MoveCutoverVerifyWriteSeq(poolUUID uuid.UUID, oid [OIDLen]byte, return err } +// MoveComplete acknowledges durable source cleanup after cutover. Until this +// handshake, the kernel keeps the object switched and rejects another move. +func (c *Client) MoveComplete(poolUUID uuid.UUID, oid [OIDLen]byte, seq uint64) error { + _, err := c.send(CmdMoveComplete, cutoverAttrs(poolUUID, oid, seq, nil)) + return err +} + +// MoveRollback acknowledges durable removal of an interrupted pre-cutover copy. +// It cannot roll back a switched object. +func (c *Client) MoveRollback(poolUUID uuid.UUID, oid [OIDLen]byte, seq uint64) error { + _, err := c.send(CmdMoveRollback, cutoverAttrs(poolUUID, oid, seq, nil)) + return err +} + func cutoverAttrs(poolUUID uuid.UUID, oid [OIDLen]byte, seq uint64, writeSeq *uint64) []netlink.Attribute { attrs := []netlink.Attribute{ {Type: AttrPoolUUID, Data: poolUUID[:]}, diff --git a/controlplane/recovery.go b/controlplane/recovery.go index 2632349..806fda6 100644 --- a/controlplane/recovery.go +++ b/controlplane/recovery.go @@ -8,8 +8,16 @@ import ( ) // Recover scans smoothfs_objects for interrupted movement states and failed -// LUN resume rows that need destination placement reconciliation. -func Recover(ctx context.Context, db *sql.DB) error { +// LUN resume rows that need destination placement reconciliation. A running +// service must pass its kernel client; omitting it retains the historical +// database-only repair for offline callers. +func Recover(ctx context.Context, db *sql.DB, clients ...movementClient) error { + if len(clients) > 0 { + return recoverLive(ctx, db, clients[0]) + } + // Historical database-only reconciliation, retained for offline callers. + // A running service must supply its kernel client. + rows, err := db.QueryContext(ctx, ` SELECT object_id, current_tier_id, intended_tier_id, movement_state, transaction_seq, pin_state, namespace_id, rel_path diff --git a/controlplane/recovery_live.go b/controlplane/recovery_live.go new file mode 100644 index 0000000..7c8094d --- /dev/null +++ b/controlplane/recovery_live.go @@ -0,0 +1,308 @@ +package controlplane + +import ( + "bytes" + "context" + "database/sql" + "encoding/hex" + "errors" + "fmt" + "log" + "os" + "path/filepath" + "strings" + "syscall" + + "golang.org/x/sys/unix" + + "github.com/google/uuid" +) + +type movementRollbackClient interface { + MoveRollback(uuid.UUID, [OIDLen]byte, uint64) error +} + +type interruptedObject struct { + oid, namespace, source, destination, state, pin, rel string + seq uint64 +} + +// The database transition can lag the kernel cutover or completion ack. Never +// choose which copy to delete, or report PLACED, using the SQL state alone. +func recoverLive(ctx context.Context, db *sql.DB, client movementClient) error { + rows, err := db.QueryContext(ctx, `SELECT object_id, namespace_id, current_tier_id, + COALESCE(intended_tier_id,''), movement_state, pin_state, rel_path, transaction_seq + FROM smoothfs_objects WHERE movement_state NOT IN ('placed','cleanup_complete','stale')`) + if err != nil { + return fmt.Errorf("scan in-flight movements: %w", err) + } + var pending []interruptedObject + for rows.Next() { + var p interruptedObject + if err = rows.Scan(&p.oid, &p.namespace, &p.source, &p.destination, &p.state, &p.pin, &p.rel, &p.seq); err != nil { + rows.Close() + return err + } + pending = append(pending, p) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + var failures []error + for _, p := range pending { + if err = recoverObject(ctx, db, client, p); err != nil { + failures = append(failures, fmt.Errorf("recover %s: %w", p.oid, err)) + } + } + return errors.Join(failures...) +} + +func recoverObject(ctx context.Context, db *sql.DB, client movementClient, p interruptedObject) error { + if client == nil { + return errors.New("kernel client unavailable") + } + raw, err := hex.DecodeString(p.oid) + if err != nil || len(raw) != OIDLen { + return errors.New("invalid object id") + } + var oid [OIDLen]byte + copy(oid[:], raw) + var backend, domain, exposed string + if err = db.QueryRowContext(ctx, `SELECT backend_ref,placement_domain,exposed_path FROM managed_namespaces WHERE id=? AND backend_kind='smoothfs'`, p.namespace).Scan(&backend, &domain, &exposed); err != nil { + return err + } + pool, err := uuid.Parse(backend) + if err != nil { + return fmt.Errorf("namespace has no pool UUID: %w", err) + } + ins, err := client.Inspect(pool, oid) + if errors.Is(err, syscall.ENOENT) { + // An asynchronous path index may not contain a recently moved object after + // reboot. Instantiate its persisted name, then confirm the same OID through + // the target pool's INSPECT before selecting or deleting any lower copy. + if err = warmRecoveryObject(exposed, p.rel, oid); err != nil { + return fmt.Errorf("instantiate interrupted object: %w", err) + } + log.Printf("smoothfs: resolved recovery path for object %s", p.oid) + ins, err = client.Inspect(pool, oid) + } + if err != nil { + return fmt.Errorf("inspect: %w", err) + } + if ins == nil { + return errors.New("inspect returned no object") + } + if ins.MovementState != StatePlaced && ins.TransactionSeq != p.seq { + return errors.New("kernel transaction differs from interrupted movement") + } + rel := ins.RelPath + if rel == "" { + rel = p.rel + } + if rel == "" || filepath.IsAbs(rel) || filepath.Clean(rel) != rel || rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { + return errors.New("invalid recovery relative path") + } + type tier struct { + id, lower string + rank uint8 + } + tiers := map[string]tier{} + var current tier + rows, err := db.QueryContext(ctx, `SELECT id,rank,backing_ref FROM tier_targets WHERE placement_domain=? AND backend_kind='smoothfs'`, domain) + if err != nil { + return err + } + for rows.Next() { + var t tier + if err = rows.Scan(&t.id, &t.rank, &t.lower); err != nil { + rows.Close() + return err + } + if !filepath.IsAbs(t.lower) { + rows.Close() + return errors.New("tier has no absolute backing path") + } + tiers[t.id] = t + if t.rank == ins.CurrentTier { + current = t + } + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + if current.id == "" { + return errors.New("kernel tier is not in namespace") + } + // A failed retry or older database-only recovery may have cleared intended. + // Its durable movement log still identifies the two paths for this sequence. + source, destination := p.source, p.destination + if destination == "" { + err = db.QueryRowContext(ctx, `SELECT source_tier,dest_tier FROM smoothfs_movement_log WHERE object_id=? AND transaction_seq=? ORDER BY id DESC LIMIT 1`, p.oid, p.seq).Scan(&source, &destination) + if err != nil && err != sql.ErrNoRows { + return err + } + } + if destination == "" || source == destination { + return errors.New("interrupted movement has no distinct recorded tiers") + } + if _, ok := tiers[source]; !ok { + return errors.New("recorded source tier is unavailable") + } + if _, ok := tiers[destination]; !ok { + return errors.New("recorded destination tier is unavailable") + } + if current.id != source && current.id != destination { + return errors.New("kernel placement is outside recorded movement") + } + authoritative := filepath.Join(current.lower, rel) + live, err := os.Lstat(authoritative) + if err != nil { + return fmt.Errorf("authoritative copy: %w", err) + } + if !live.Mode().IsRegular() { + return errors.New("authoritative copy is not regular") + } + other := destination + if current.id == destination { + other = source + } + otherPath := filepath.Join(tiers[other].lower, rel) + info, err := os.Lstat(otherPath) + if err == nil { + if !info.Mode().IsRegular() || os.SameFile(info, live) { + return errors.New("cleanup path is not a distinct regular copy") + } + } else if !os.IsNotExist(err) { + return err + } + // Validate the phase before touching either path. A switched kernel wins + // even if the process died before recording the SQL 'switched' transition. + switch ins.MovementState { + case StateSwitched: + if current.id != destination { + return errors.New("switched object is not on recorded destination") + } + if _, ok := client.(movementCompletionClient); !ok { + return errors.New("kernel completion acknowledgement unavailable") + } + case StatePlanAccepted, StateDestinationReserved, StateCopyInProgress, StateCopyComplete, StateCopyVerified, StateFailed: + if current.id != source { + return errors.New("pre-cutover object is not on recorded source") + } + if _, ok := client.(movementRollbackClient); !ok { + return errors.New("kernel rollback acknowledgement unavailable") + } + case StatePlaced: + // Mount replay or an ack preceding the final SQL commit already settled it. + default: + return fmt.Errorf("kernel movement is still %s", ins.MovementState) + } + if err == nil { + if err = os.Remove(otherPath); err != nil { + return fmt.Errorf("remove interrupted copy: %w", err) + } + } + // The other name may already be absent after a crash. Still issue its parent + // barrier before telling the kernel or database that cleanup is complete. + parent := filepath.Dir(otherPath) + for { + err = syncDir(parent) + if !os.IsNotExist(err) || parent == filepath.Clean(tiers[other].lower) { + break + } + parent = filepath.Dir(parent) + } + if err != nil { + return fmt.Errorf("sync interrupted cleanup: %w", err) + } + switch ins.MovementState { + case StateSwitched: + err = client.(movementCompletionClient).MoveComplete(pool, oid, ins.TransactionSeq) + case StatePlaced: + default: + err = client.(movementRollbackClient).MoveRollback(pool, oid, ins.TransactionSeq) + } + if err != nil { + return fmt.Errorf("kernel recovery acknowledgement: %w", err) + } + settled, err := client.Inspect(pool, oid) + if err != nil { + return err + } + if settled == nil || settled.MovementState != StatePlaced || settled.CurrentTier != ins.CurrentTier { + return errors.New("kernel recovery did not settle on authoritative tier") + } + if p.pin == string(PinLUN) { + if err = restoreLUNPin(ctx, db, p.namespace, current.id, rel); err != nil { + return err + } + } + state := string(StatePlaced) + if p.pin == string(PinLUN) && p.state == string(StateFailed) { + state = string(StateFailed) + } + _, err = db.ExecContext(ctx, `UPDATE smoothfs_objects SET current_tier_id=?,intended_tier_id=NULL, + movement_state=?,rel_path=?,last_movement_at=CASE WHEN ? THEN datetime('now') ELSE last_movement_at END, + last_committed_cutover_gen=MAX(last_committed_cutover_gen,?), + failure_reason=CASE WHEN ?='failed' THEN failure_reason ELSE 'recovered: kernel placement and cleanup confirmed' END, + updated_at=datetime('now') WHERE object_id=?`, current.id, state, rel, current.id == destination, settled.Generation, state, p.oid) + return err +} + +func warmRecoveryObject(exposed, rel string, oid [OIDLen]byte) error { + if !filepath.IsAbs(exposed) || rel == "" || filepath.IsAbs(rel) || filepath.Clean(rel) != rel || rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { + return errors.New("namespace has no usable exposed recovery path") + } + exposed, err := filepath.EvalSymlinks(exposed) + if err != nil { + return err + } + mounts, err := os.ReadFile("/proc/self/mountinfo") + if err != nil { + return err + } + mount := recoveryMountpoint(string(mounts), exposed) + if mount == "" { + return errors.New("namespace is not below a mounted smoothfs root") + } + path := filepath.Join(mount, rel) + if _, err = os.Lstat(path); err != nil { + return err + } + raw := make([]byte, OIDLen) + n, err := unix.Getxattr(path, "trusted.smoothfs.oid", raw) + if err != nil { + return err + } + if n != OIDLen || !bytes.Equal(raw, oid[:]) { + return errors.New("persisted recovery path belongs to another object") + } + return nil +} + +func recoveryMountpoint(mounts, exposed string) string { + decode := strings.NewReplacer(`\040`, " ", `\011`, "\t", `\012`, "\n", `\134`, `\`) + best := "" + for _, line := range strings.Split(mounts, "\n") { + before, after, ok := strings.Cut(line, " - ") + if !ok { + continue + } + fields, fs := strings.Fields(before), strings.Fields(after) + if len(fields) < 5 || len(fs) < 1 || fs[0] != "smoothfs" || fields[3] != "/" { + continue + } + path := decode.Replace(fields[4]) + if exposed != path && !strings.HasPrefix(exposed, path+string(os.PathSeparator)) { + continue + } + if len(path) > len(best) { + best = path + } + } + return best +} diff --git a/controlplane/recovery_live_test.go b/controlplane/recovery_live_test.go new file mode 100644 index 0000000..8ee26ef --- /dev/null +++ b/controlplane/recovery_live_test.go @@ -0,0 +1,202 @@ +package controlplane + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/google/uuid" +) + +type recoveringMovementClient struct { + fakeMovementClient + result InspectResult + ackErr error + beforeAck func() + completed, rolledBack int +} + +func (c *recoveringMovementClient) Inspect(uuid.UUID, [OIDLen]byte) (*InspectResult, error) { + r := c.result + return &r, nil +} +func (c *recoveringMovementClient) MoveComplete(uuid.UUID, [OIDLen]byte, uint64) error { + c.completed++ + return c.ack() +} +func (c *recoveringMovementClient) MoveRollback(uuid.UUID, [OIDLen]byte, uint64) error { + c.rolledBack++ + return c.ack() +} +func (c *recoveringMovementClient) ack() error { + if c.beforeAck != nil { + c.beforeAck() + } + if c.ackErr != nil { + return c.ackErr + } + c.result.MovementState = StatePlaced + return nil +} + +func TestRecoverLiveUsesKernelPlacementAndDurableCleanup(t *testing.T) { + for _, tc := range []struct { + name string + sql, kernel MovementState + rank uint8 + }{ + {"before-copy", StatePlanAccepted, StatePlanAccepted, 0}, + {"after-copy", StateCopyVerified, StatePlanAccepted, 0}, + {"cutover-before-sql", StateCopyVerified, StateSwitched, 1}, + {"cleanup-before-ack", StateSwitched, StateSwitched, 1}, + {"ack-before-sql", StateSwitched, StatePlaced, 1}, + {"mount-replay-forward", StateCopyVerified, StatePlaced, 1}, + {"mount-replay-rollback", StateSwitched, StatePlaced, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + db := testDB(t) + ns, fast, slow := seedPool(t, db) + fastDir, slowDir := t.TempDir(), t.TempDir() + pool := uuid.New() + if _, err := db.Exec(`UPDATE managed_namespaces SET backend_ref=? WHERE id=?`, pool.String(), ns); err != nil { + t.Fatal(err) + } + for id, path := range map[string]string{fast: fastDir, slow: slowDir} { + if _, err := db.Exec(`UPDATE tier_targets SET backing_ref=? WHERE id=?`, path, id); err != nil { + t.Fatal(err) + } + } + oid := [OIDLen]byte{0x42} + seedMidMove(t, db, ns, fast, slow, tc.sql, 11, oid) + for _, dir := range []string{fastDir, slowDir} { + if err := os.WriteFile(filepath.Join(dir, "chunk"), []byte("bytes"), 0o600); err != nil { + t.Fatal(err) + } + } + canonical, other, tier := fastDir, slowDir, fast + if tc.rank == 1 { + canonical, other, tier = slowDir, fastDir, slow + } + client := &recoveringMovementClient{result: InspectResult{MovementState: tc.kernel, CurrentTier: tc.rank, TransactionSeq: 11, RelPath: "chunk", Generation: 4}} + client.beforeAck = func() { + if _, err := os.Stat(filepath.Join(other, "chunk")); !os.IsNotExist(err) { + t.Fatal("ack preceded deletion") + } + _, _, state, _, _ := readObjectState(t, db, oid) + if state != string(tc.sql) { + t.Fatal("SQL reported recovery before kernel ack") + } + } + if err := Recover(context.Background(), db, client); err != nil { + t.Fatal(err) + } + cur, intended, state, gen, _ := readObjectState(t, db, oid) + if cur.String != tier || intended.Valid || state != "placed" || gen != 4 { + t.Fatalf("recovery cur=%s intended=%v state=%s generation=%d", cur.String, intended, state, gen) + } + if data, err := os.ReadFile(filepath.Join(canonical, "chunk")); err != nil || string(data) != "bytes" { + t.Fatal("authoritative copy lost", err) + } + if _, err := os.Stat(filepath.Join(other, "chunk")); !os.IsNotExist(err) { + t.Fatal("old copy remains") + } + if tc.kernel == StateSwitched && client.completed != 1 { + t.Fatal("missing completion ack") + } + if tc.kernel == StatePlanAccepted && client.rolledBack != 1 { + t.Fatal("missing rollback ack") + } + }) + } +} + +func TestRecoverLiveAckFailurePreservesPendingStateAndRetriesAbsentSource(t *testing.T) { + db := testDB(t) + ns, fast, slow := seedPool(t, db) + a, b := t.TempDir(), t.TempDir() + pool := uuid.New() + db.Exec(`UPDATE managed_namespaces SET backend_ref=? WHERE id=?`, pool.String(), ns) + db.Exec(`UPDATE tier_targets SET backing_ref=? WHERE id=?`, a, fast) + db.Exec(`UPDATE tier_targets SET backing_ref=? WHERE id=?`, b, slow) + oid := [OIDLen]byte{0x43} + seedMidMove(t, db, ns, fast, slow, StateSwitched, 12, oid) + os.WriteFile(filepath.Join(a, "chunk"), []byte("old"), 0o600) + os.WriteFile(filepath.Join(b, "chunk"), []byte("current"), 0o600) + client := &recoveringMovementClient{result: InspectResult{MovementState: StateSwitched, CurrentTier: 1, TransactionSeq: 12, RelPath: "chunk"}, ackErr: errors.New("ack failed")} + if err := Recover(context.Background(), db, client); err == nil { + t.Fatal("ack failure accepted") + } + _, _, state, _, _ := readObjectState(t, db, oid) + if state != "switched" { + t.Fatal("failed acknowledgement finalized SQL") + } + client.ackErr = nil + if err := Recover(context.Background(), db, client); err != nil { + t.Fatal(err) + } + _, _, state, _, _ = readObjectState(t, db, oid) + if state != "placed" { + t.Fatal(state) + } +} + +func TestRecoverLiveRejectsStaleOrAmbiguousCleanup(t *testing.T) { + for _, failure := range []string{"stale", "directory", "invalid-relative-path", "missing-authority", "missing-pool"} { + t.Run(failure, func(t *testing.T) { + db := testDB(t) + ns, fast, slow := seedPool(t, db) + a, b := t.TempDir(), t.TempDir() + db.Exec(`UPDATE managed_namespaces SET backend_ref=? WHERE id=?`, uuid.NewString(), ns) + db.Exec(`UPDATE tier_targets SET backing_ref=? WHERE id=?`, a, fast) + db.Exec(`UPDATE tier_targets SET backing_ref=? WHERE id=?`, b, slow) + oid := [OIDLen]byte{0x44} + seedMidMove(t, db, ns, fast, slow, StateSwitched, 12, oid) + os.WriteFile(filepath.Join(b, "chunk"), []byte("current"), 0o600) + if failure == "directory" { + os.Mkdir(filepath.Join(a, "chunk"), 0o700) + } else { + os.WriteFile(filepath.Join(a, "chunk"), []byte("old"), 0o600) + } + client := &recoveringMovementClient{result: InspectResult{MovementState: StateSwitched, CurrentTier: 1, TransactionSeq: 12, RelPath: "chunk"}} + switch failure { + case "stale": + client.result.TransactionSeq = 13 + case "invalid-relative-path": + client.result.RelPath = "../chunk" + case "missing-authority": + os.Remove(filepath.Join(b, "chunk")) + case "missing-pool": + db.Exec(`UPDATE managed_namespaces SET backend_ref='' WHERE id=?`, ns) + } + if err := Recover(context.Background(), db, client); err == nil { + t.Fatal("unsafe recovery accepted") + } + if _, err := os.Lstat(filepath.Join(a, "chunk")); err != nil { + t.Fatal("removed copy on failed recovery") + } + _, _, state, _, _ := readObjectState(t, db, oid) + if state != "switched" { + t.Fatal(state) + } + if client.completed != 0 || client.rolledBack != 0 { + t.Fatal("acknowledged rejected recovery") + } + }) + } +} + +func TestRecoveryMountpointFindsPoolRootForNamespace(t *testing.T) { + mounts := `10 1 0:20 / /mnt/pbs rw - smoothfs none rw,pool=media +11 1 0:21 / /mnt/pbs2 rw - smoothfs none rw,pool=other +12 1 0:22 /sub /bind-sub rw - smoothfs none rw +13 1 0:23 / /mnt/my\040pool rw - smoothfs none rw +14 1 8:1 / /ordinary rw - ext4 /dev/sda1 rw +` + for exposed, want := range map[string]string{"/mnt/pbs": "/mnt/pbs", "/mnt/pbs/nested/ns": "/mnt/pbs", "/mnt/pbs2/sub": "/mnt/pbs2", "/mnt/pbs-not-a-pool": "", "/bind-sub": "", "/ordinary": "", "/mnt/my pool/ns": "/mnt/my pool"} { + if got := recoveryMountpoint(mounts, exposed); got != want { + t.Errorf("%q: got %q want %q", exposed, got, want) + } + } +} diff --git a/controlplane/service.go b/controlplane/service.go index 312120c..793f343 100644 --- a/controlplane/service.go +++ b/controlplane/service.go @@ -150,7 +150,7 @@ func DiscoverPoolFromDB(ctx context.Context, db *sql.DB, ev *Event) (*Pool, erro } func (s *Service) Run(ctx context.Context) error { - if err := Recover(ctx, s.db); err != nil { + if err := Recover(ctx, s.db, s.client); err != nil { log.Printf("smoothfs: recovery failed: %v", err) } diff --git a/controlplane/testdb_test.go b/controlplane/testdb_test.go index 32cabfa..98e66bc 100644 --- a/controlplane/testdb_test.go +++ b/controlplane/testdb_test.go @@ -40,6 +40,7 @@ func testDB(t *testing.T) *sql.DB { placement_domain TEXT NOT NULL, backend_kind TEXT NOT NULL, backend_ref TEXT NOT NULL DEFAULT '', + exposed_path TEXT NOT NULL DEFAULT '', FOREIGN KEY(placement_domain) REFERENCES placement_domains(id) );`, `CREATE TABLE smoothfs_objects ( diff --git a/controlplane/worker.go b/controlplane/worker.go index 0c4c2a8..8ba7e54 100644 --- a/controlplane/worker.go +++ b/controlplane/worker.go @@ -76,6 +76,10 @@ type writeSeqCutoverClient interface { MoveCutoverVerifyWriteSeq(poolUUID uuid.UUID, oid [OIDLen]byte, seq, writeSeq uint64) error } +type movementCompletionClient interface { + MoveComplete(poolUUID uuid.UUID, oid [OIDLen]byte, seq uint64) error +} + type LUNTargetResumer interface { Resume(ctx context.Context, targetID string) error } @@ -195,11 +199,18 @@ func (w *Worker) execute(ctx context.Context, p MovementPlan) error { srcPath := filepath.Join(p.SourceLowerDir, p.RelPath) dstPath := filepath.Join(p.DestLowerDir, p.RelPath) + // Leave an existing interrupted kernel transaction and its SQL record intact. + if ins.MovementState != "" && ins.MovementState != StatePlaced { + return fmt.Errorf("kernel movement requires recovery: %s", ins.MovementState) + } + // Persist the intent before the first kernel transition. A process may exit + // inside MovePlan, or after cutover but before any later SQL transition. + if err := w.persistIntent(ctx, oid, p); err != nil { + return fmt.Errorf("persist movement intent: %w", err) + } if err := w.client.MovePlan(p.PoolUUID, p.ObjectID, p.DestTierRank, p.TransactionSeq); err != nil { return w.rollbackLUNBeforeSwitch(ctx, p, srcPath, oid, fmt.Errorf("move_plan: %w", err)) } - w.logTransition(ctx, p, "", string(StatePlanAccepted), "") - w.persistObject(ctx, oid, p, StatePlanAccepted) if err := os.MkdirAll(filepath.Dir(dstPath), 0o755); err != nil { return w.rollbackLUNBeforeSwitch(ctx, p, srcPath, oid, fmt.Errorf("mkdir dest parent: %w", err)) @@ -302,8 +313,32 @@ func (w *Worker) execute(ctx context.Context, p MovementPlan) error { } } - if err := os.Remove(srcPath); err != nil && !os.IsNotExist(err) { + // A new movement must not begin until old-source cleanup is durable. Never + // report placed after a failed unlink, directory barrier, or kernel ack. + cleanupFailed := func(err error) error { w.logTransition(ctx, p, string(StateSwitched), string(StateCleanupInProgress), err.Error()) + w.persistObject(ctx, oid, p, StateCleanupInProgress) + return err + } + if err := os.Remove(srcPath); err != nil && !os.IsNotExist(err) { + return cleanupFailed(fmt.Errorf("cleanup source: %w", err)) + } + dir, err := os.Open(filepath.Dir(srcPath)) + if err != nil { + return cleanupFailed(fmt.Errorf("open source parent: %w", err)) + } + err = dir.Sync() + closeErr := dir.Close() + if err != nil { + return cleanupFailed(fmt.Errorf("sync source cleanup: %w", err)) + } + if closeErr != nil { + return cleanupFailed(fmt.Errorf("close source parent: %w", closeErr)) + } + if client, ok := w.client.(movementCompletionClient); ok { + if err := client.MoveComplete(p.PoolUUID, p.ObjectID, p.TransactionSeq); err != nil { + return cleanupFailed(fmt.Errorf("move_complete: %w", err)) + } } w.logTransition(ctx, p, string(StateSwitched), string(StateCleanupComplete), "") w.finalize(ctx, oid, p) @@ -672,3 +707,51 @@ func fileSHA256(path string) ([32]byte, error) { copy(out[:], h.Sum(nil)) return out, nil } + +// An intent must survive a host crash, including on WAL connections configured +// with NORMAL synchronous mode. Use a dedicated connection for this barrier. +func (w *Worker) persistIntent(ctx context.Context, oid string, p MovementPlan) error { + conn, err := w.db.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + var synchronous int + if err = conn.QueryRowContext(ctx, "PRAGMA synchronous").Scan(&synchronous); err != nil { + return err + } + if _, err = conn.ExecContext(ctx, "PRAGMA synchronous=FULL"); err != nil { + return err + } + defer conn.ExecContext(context.Background(), fmt.Sprintf("PRAGMA synchronous=%d", synchronous)) + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + result, err := tx.ExecContext(ctx, `INSERT INTO smoothfs_objects + (object_id,namespace_id,current_tier_id,intended_tier_id,movement_state,transaction_seq,rel_path,updated_at) + VALUES(?,?,?,?,'plan_accepted',?,?,datetime('now')) + ON CONFLICT(object_id) DO UPDATE SET intended_tier_id=excluded.intended_tier_id, + movement_state=excluded.movement_state,transaction_seq=excluded.transaction_seq, + rel_path=excluded.rel_path,updated_at=excluded.updated_at + WHERE smoothfs_objects.movement_state IN ('placed','cleanup_complete','failed')`, + oid, p.NamespaceID, p.SourceTierID, p.DestTierID, p.TransactionSeq, p.RelPath) + if err != nil { + return err + } + n, err := result.RowsAffected() + if err != nil { + return err + } + if n != 1 { + return errors.New("object already has an interrupted movement") + } + _, err = tx.ExecContext(ctx, `INSERT INTO smoothfs_movement_log + (object_id,transaction_seq,from_state,to_state,source_tier,dest_tier,payload_json) + VALUES(?,?,NULL,'plan_accepted',?,?,'{"reason":"intent persisted before kernel plan"}')`, oid, p.TransactionSeq, p.SourceTierID, p.DestTierID) + if err != nil { + return err + } + return tx.Commit() +} diff --git a/controlplane/worker_completion_test.go b/controlplane/worker_completion_test.go new file mode 100644 index 0000000..2b1d6d1 --- /dev/null +++ b/controlplane/worker_completion_test.go @@ -0,0 +1,95 @@ +package controlplane + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/google/uuid" +) + +type completingMovementClient struct { + fakeMovementClient + cutover func() + complete func() error +} + +func (c *completingMovementClient) MoveCutover(uuid.UUID, [OIDLen]byte, uint64) error { + if c.cutover != nil { + c.cutover() + } + return nil +} +func (c *completingMovementClient) MoveComplete(uuid.UUID, [OIDLen]byte, uint64) error { + return c.complete() +} + +func TestWorkerAcknowledgesOnlyDurableCleanup(t *testing.T) { + for _, name := range []string{"success", "ack failure", "unlink failure"} { + t.Run(name, func(t *testing.T) { + db := testDB(t) + ns, srcID, dstID := seedPool(t, db) + srcDir, dstDir := t.TempDir(), t.TempDir() + src := filepath.Join(srcDir, "chunk") + dst := filepath.Join(dstDir, "chunk") + payload := []byte("a completed PBS chunk must remain readable") + if err := os.WriteFile(src, payload, 0600); err != nil { + t.Fatal(err) + } + var oid [OIDLen]byte + oid[0] = 0xb7 + called := false + c := &completingMovementClient{fakeMovementClient: fakeMovementClient{inspectResult: &InspectResult{CurrentTier: 0, RelPath: "chunk", PinState: PinNone}}} + c.complete = func() error { + called = true + if _, err := os.Stat(src); !os.IsNotExist(err) { + t.Fatalf("completion before source cleanup: %v", err) + } + if data, err := os.ReadFile(dst); err != nil || string(data) != string(payload) { + t.Fatalf("destination lost: %v", err) + } + _, _, state, _, _ := readObjectState(t, db, oid) + if state != string(StateSwitched) { + t.Fatalf("state before acknowledgement=%q", state) + } + if name == "ack failure" { + return errors.New("kernel rejected completion") + } + return nil + } + if name == "unlink failure" { + c.cutover = func() { + if err := os.Remove(src); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(src, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(src, "held"), nil, 0600); err != nil { + t.Fatal(err) + } + } + } + plan := MovementPlan{PoolUUID: uuid.New(), ObjectID: oid, NamespaceID: ns, SourceTierID: srcID, SourceLowerDir: srcDir, DestTierID: dstID, DestTierRank: 1, DestLowerDir: dstDir, RelPath: "chunk", TransactionSeq: 77} + err := NewWorker(db, c).Execute(context.Background(), plan) + cur, intended, state, _, _ := readObjectState(t, db, oid) + if name == "success" { + if err != nil || !called || state != string(StatePlaced) || cur.String != dstID || intended.Valid { + t.Fatalf("success: err=%v called=%v state=%q current=%q intended=%v", err, called, state, cur.String, intended) + } + } else { + if err == nil || state != string(StateCleanupInProgress) || cur.String != srcID || intended.String != dstID { + t.Fatalf("failed cleanup reported complete: err=%v state=%q current=%q intended=%v", err, state, cur.String, intended) + } + if called != (name == "ack failure") { + t.Fatalf("completion called=%v after %s", called, name) + } + if data, err := os.ReadFile(dst); err != nil || string(data) != string(payload) { + t.Fatalf("cleanup failure lost destination: %v", err) + } + } + }) + } +} diff --git a/controlplane/worker_intent_test.go b/controlplane/worker_intent_test.go new file mode 100644 index 0000000..009d5bd --- /dev/null +++ b/controlplane/worker_intent_test.go @@ -0,0 +1,102 @@ +package controlplane + +import ( + "context" + "encoding/hex" + "os" + "path/filepath" + "testing" + + "github.com/google/uuid" +) + +type intentMovementClient struct { + fakeMovementClient + beforePlan func() +} + +func (c *intentMovementClient) MovePlan(p uuid.UUID, o [OIDLen]byte, d uint8, s uint64) error { + if c.beforePlan != nil { + c.beforePlan() + } + return c.fakeMovementClient.MovePlan(p, o, d, s) +} +func TestWorkerPersistsFullSyncIntentBeforeKernelPlan(t *testing.T) { + db := testDB(t) + db.SetMaxOpenConns(1) + ns, fast, slow := seedPool(t, db) + if _, err := db.Exec(`PRAGMA synchronous=NORMAL`); err != nil { + t.Fatal(err) + } + a, b := t.TempDir(), t.TempDir() + if err := os.WriteFile(filepath.Join(a, "chunk"), []byte("bytes"), 0o600); err != nil { + t.Fatal(err) + } + oid := [OIDLen]byte{0x51} + c := &intentMovementClient{fakeMovementClient: fakeMovementClient{inspectResult: &InspectResult{MovementState: StatePlaced, RelPath: "chunk"}}} + c.beforePlan = func() { + var state, rel string + var seq int64 + if err := db.QueryRow(`SELECT movement_state,rel_path,transaction_seq FROM smoothfs_objects WHERE object_id=?`, hex.EncodeToString(oid[:])).Scan(&state, &rel, &seq); err != nil { + t.Fatal(err) + } + if state != "plan_accepted" || rel != "chunk" || seq != 17 { + t.Fatalf("missing prior intent: %s %s %d", state, rel, seq) + } + var n int + if err := db.QueryRow(`SELECT count(*) FROM smoothfs_movement_log WHERE object_id=? AND transaction_seq=17`, hex.EncodeToString(oid[:])).Scan(&n); err != nil || n != 1 { + t.Fatal("missing prior log", err, n) + } + } + p := MovementPlan{PoolUUID: uuid.New(), ObjectID: oid, NamespaceID: ns, SourceTierID: fast, SourceLowerDir: a, DestTierID: slow, DestTierRank: 1, DestLowerDir: b, RelPath: "chunk", TransactionSeq: 17} + if err := NewWorker(db, c).Execute(context.Background(), p); err != nil { + t.Fatal(err) + } + var sync int + if err := db.QueryRow(`PRAGMA synchronous`).Scan(&sync); err != nil || sync != 1 { + t.Fatal("connection synchronous setting not restored", err, sync) + } +} + +func TestWorkerDoesNotPlanWithoutCommittedIntent(t *testing.T) { + db := testDB(t) + ns, fast, slow := seedPool(t, db) + if _, err := db.Exec(`CREATE TRIGGER reject_intent BEFORE INSERT ON smoothfs_movement_log BEGIN SELECT RAISE(ABORT,'simulated database write failure'); END`); err != nil { + t.Fatal(err) + } + a, b := t.TempDir(), t.TempDir() + os.WriteFile(filepath.Join(a, "chunk"), []byte("bytes"), 0o600) + c := &fakeMovementClient{inspectResult: &InspectResult{MovementState: StatePlaced, RelPath: "chunk"}} + p := MovementPlan{PoolUUID: uuid.New(), ObjectID: [OIDLen]byte{0x52}, NamespaceID: ns, SourceTierID: fast, SourceLowerDir: a, DestTierID: slow, DestTierRank: 1, DestLowerDir: b, RelPath: "chunk", TransactionSeq: 18} + if err := NewWorker(db, c).Execute(context.Background(), p); err == nil || c.movePlanned { + t.Fatal("kernel plan issued without durable intent", err) + } + var n int + if err := db.QueryRow(`SELECT count(*) FROM smoothfs_objects`).Scan(&n); err != nil || n != 0 { + t.Fatal("intent transaction was not rolled back", err, n) + } +} + +func TestWorkerPreservesInterruptedTransaction(t *testing.T) { + for _, state := range []MovementState{StatePlaced, StateSwitched} { + t.Run(string(state), func(t *testing.T) { + db := testDB(t) + ns, fast, slow := seedPool(t, db) + oid := [OIDLen]byte{0x53} + seedMidMove(t, db, ns, fast, slow, StateSwitched, 19, oid) + c := &fakeMovementClient{inspectResult: &InspectResult{MovementState: state, RelPath: "chunk"}} + p := MovementPlan{PoolUUID: uuid.New(), ObjectID: oid, NamespaceID: ns, SourceTierID: fast, SourceLowerDir: t.TempDir(), DestTierID: slow, DestTierRank: 1, DestLowerDir: t.TempDir(), RelPath: "chunk", TransactionSeq: 20} + if err := NewWorker(db, c).Execute(context.Background(), p); err == nil || c.movePlanned { + t.Fatal("overwrote pending movement", err) + } + _, _, got, _, _ := readObjectState(t, db, oid) + if got != "switched" { + t.Fatal(got) + } + var seq uint64 + if err := db.QueryRow(`SELECT transaction_seq FROM smoothfs_objects`).Scan(&seq); err != nil || seq != 19 { + t.Fatal("old transaction lost", err, seq) + } + }) + } +} diff --git a/src/smoothfs/compat.h b/src/smoothfs/compat.h index bc750b3..2403ea8 100644 --- a/src/smoothfs/compat.h +++ b/src/smoothfs/compat.h @@ -136,11 +136,10 @@ smoothfs_compat_create(struct mnt_idmap *idmap, struct inode *dir, * * Helper does the now-canonical "may-replace-dentry" dance: returns * the dentry the caller should use (either input or replacement), - * or ERR_PTR on failure. NOTE: vfs_mkdir() itself dput()s the - * passed-in dentry whenever it returns a *different* pointer - * (a replacement OR an ERR_PTR), and additionally unlocks the parent - * on error. Callers must therefore NOT dput the original on - * replacement/error, and must not unlock the parent again on error. + * or ERR_PTR on failure. vfs_mkdir() consumes the original on replacement + * or error. Starting with 6.19 it also unlocks the parent on error; this + * helper supplies that unlock on 6.18 to keep one caller contract. Callers + * must not dput the original on replacement/error, or unlock on error. */ static inline struct dentry * smoothfs_compat_mkdir(struct mnt_idmap *idmap, struct inode *dir, @@ -153,8 +152,16 @@ smoothfs_compat_mkdir(struct mnt_idmap *idmap, struct inode *dir, struct dentry *new_d = vfs_mkdir(idmap, dir, dentry, mode); #endif - if (IS_ERR(new_d)) + /* Normalize mkdir's error contract: the child is consumed and the parent + * unlocked. v6.18 consumes only the child; v6.19+ also calls end_creating. + * Success always leaves the caller's parent lock held. + */ + if (IS_ERR(new_d)) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 19, 0) + inode_unlock(dir); +#endif return new_d; + } if (new_d && new_d != dentry) return new_d; return dentry; diff --git a/src/smoothfs/dir.c b/src/smoothfs/dir.c index 059c827..346e879 100644 --- a/src/smoothfs/dir.c +++ b/src/smoothfs/dir.c @@ -157,7 +157,7 @@ static int smoothfs_collect_one_dir(struct smoothfs_dir_cache *cache, return collect.err; } -static struct smoothfs_dir_cache *smoothfs_build_dir_cache(struct file *file) +static struct smoothfs_dir_cache *smoothfs_build_dir_cache_inner(struct file *file) { struct inode *inode = file_inode(file); struct smoothfs_inode_info *si = SMOOTHFS_I(inode); @@ -198,15 +198,41 @@ static struct smoothfs_dir_cache *smoothfs_build_dir_cache(struct file *file) continue; } err = smoothfs_resolve_rel_path_on_tier(sbi, tier, rel_path, &tier_path); - if (err) + if (err == -ENOENT || err == -ENOTDIR) continue; + if (err) + goto fail; if (d_is_dir(tier_path.dentry)) - (void)smoothfs_collect_one_dir(cache, &tier_path, sbi->creator_cred); + err = smoothfs_collect_one_dir(cache, &tier_path, sbi->creator_cred); path_put(&tier_path); + if (err) + goto fail; } kfree(rel_path); return cache; +fail: + /* Partial enumeration can make a backup collector delete live chunks. + * Only a genuinely absent replica is safe to omit. + */ + kfree(rel_path); + smoothfs_dir_cache_free(cache); + return ERR_PTR(err); +} + +/* VFS already checked the caller against the presented upper directory. + * Backing traversal must use the mounter's credentials on every tier, just + * like lookup. Otherwise root-owned private backing parents make readdir + * silently omit spill-tier entries for the presented owner (e.g. PBS). + */ +static struct smoothfs_dir_cache *smoothfs_build_dir_cache(struct file *file) +{ + struct smoothfs_sb_info *sbi = SMOOTHFS_SB(file_inode(file)->i_sb); + const struct cred *old = override_creds(sbi->creator_cred); + struct smoothfs_dir_cache *cache = smoothfs_build_dir_cache_inner(file); + + revert_creds(old); + return cache; } static int smoothfs_opendir(struct inode *inode, struct file *file) @@ -273,11 +299,79 @@ static loff_t smoothfs_dir_llseek(struct file *file, loff_t offset, int whence) return ret; } +/* A directory can have entries on several tiers. Making only its canonical + * lower durable would miss names created by spill or movement on another tier. + * Resolve and sync every contributing lower directory; an explicit durability + * request must not silently skip an unavailable tier or a lower fsync error. + */ +static int smoothfs_dir_fsync(struct file *file, loff_t start, loff_t end, + int datasync) +{ + struct smoothfs_sb_info *sbi = SMOOTHFS_SB(file_inode(file)->i_sb); + struct file *lower = smoothfs_lower_file(file); + char *rel_path; + unsigned int tier; + int err; + + if (IS_ERR(lower)) + return PTR_ERR(lower); + if (!lower) + return -EBADF; + + rel_path = smoothfs_rel_path_from_dentry(file->f_path.dentry); + if (!rel_path) + return -ENOMEM; + + /* Persist deferred identities and placement records before acknowledging + * the namespace barrier. Propagate the placement log's fsync failure too. + */ + smoothfs_oid_wb_drain(sbi); + smoothfs_placement_wb_drain(sbi); + err = vfs_fsync(lower, datasync); + if (err) + goto out; + + for (tier = 0; tier < sbi->ntiers; tier++) { + struct path path; + struct file *dir; + + err = smoothfs_resolve_rel_path_on_tier(sbi, tier, rel_path, &path); + if (err == -ENOENT || err == -ENOTDIR) { + err = 0; + continue; + } + if (err) + goto out; + if (!d_is_dir(path.dentry) || + (path.mnt == lower->f_path.mnt && + path.dentry == lower->f_path.dentry)) { + path_put(&path); + continue; + } + dir = dentry_open(&path, O_RDONLY | O_DIRECTORY, sbi->creator_cred); + path_put(&path); + if (IS_ERR(dir)) { + err = PTR_ERR(dir); + goto out; + } + err = vfs_fsync(dir, datasync); + fput(dir); + if (err) + goto out; + } + if (sbi->placement_log) + err = vfs_fsync(sbi->placement_log, 0); +out: + kfree(rel_path); + return err; +} + const struct file_operations smoothfs_dir_ops = { .owner = THIS_MODULE, .open = smoothfs_opendir, .release = smoothfs_releasedir, .iterate_shared = smoothfs_iterate_shared, + .fsync = smoothfs_dir_fsync, .llseek = smoothfs_dir_llseek, .read = generic_read_dir, }; diff --git a/src/smoothfs/inode.c b/src/smoothfs/inode.c index 5dbea8c..8b2636b 100644 --- a/src/smoothfs/inode.c +++ b/src/smoothfs/inode.c @@ -501,16 +501,10 @@ static int smoothfs_materialize_parent_on_tier(struct mnt_idmap *idmap, revert_creds(old_cred); if (IS_ERR(new_child)) { err = PTR_ERR(new_child); - /* - * vfs_mkdir() (6.15+) already dput()s the - * passed-in dentry when it returns an error - * pointer — a second dput here corrupts the - * dcache (dput() WARN at fs/dcache.c). It does - * NOT unlock the parent: that stays with us - * (see do_mkdirat, where end_creating_path() - * unlocks even on vfs_mkdir error). + /* The compat helper consumed child and released the + * parent lock. Unlocking again underflows i_rwsem on + * v6.19+, hanging later lookups after ENOSPC. */ - inode_unlock(d_inode(cur.dentry)); goto out_err; } if (new_child != child) { @@ -1441,7 +1435,7 @@ static struct dentry *smoothfs_mkdir_inner(struct mnt_idmap *idmap, if (IS_ERR(new_lower)) { err = PTR_ERR(new_lower); /* - * vfs_mkdir() already dput()'d lower and + * The compat helper already consumed lower and * unlocked the parent on error; do not repeat. */ path_put(&parent_path); diff --git a/src/smoothfs/movement.c b/src/smoothfs/movement.c index dcae4a6..910102a 100644 --- a/src/smoothfs/movement.c +++ b/src/smoothfs/movement.c @@ -595,10 +595,55 @@ int smoothfs_movement_cutover(struct smoothfs_sb_info *sbi, goto out_fail; } + /* The userspace mover copies bytes, not trusted identity xattrs. Persist + * the same object identity on the new backing before making it + * authoritative, so eviction or remount cannot mint a second identity. + */ + { + const struct cred *old = override_creds(sbi->creator_cred); + struct file *dest; + + err = smoothfs_write_oid_xattr(dest_path.dentry, si->oid); + if (err == -EEXIST) { + u8 existing[SMOOTHFS_OID_LEN]; + + err = smoothfs_read_oid_xattr(dest_path.dentry, existing); + if (!err && memcmp(existing, si->oid, sizeof(existing))) + err = -ESTALE; + } + if (!err) + err = smoothfs_write_gen_xattr(dest_path.dentry, si->gen); + if (!err) { + dest = dentry_open(&dest_path, O_RDONLY | O_LARGEFILE, + sbi->creator_cred); + if (IS_ERR(dest)) { + err = PTR_ERR(dest); + } else { + err = vfs_fsync(dest, 0); + fput(dest); + } + } + revert_creds(old); + if (err) { + path_put(&dest_path); + goto out_fail; + } + } + /* Swap lower_path: keep refs balanced. The smoothfs dentry's * d_fsdata also points at the OLD lower dentry — update it too, * otherwise smoothfs_d_release dputs a freed pointer at umount. */ + /* Cleanup will free the source lower inode. Its numeric inode may be + * reused immediately by an unrelated file (for example an RRD journal). + * Leaving the old fast-path key mapped to this upper inode redirects + * writes to that unrelated file into the moved object's destination. + */ + smoothfs_lower_ino_map_remove(sbi, si->current_tier, + d_inode(src_dentry)->i_ino); old_path = smoothfs_swap_lower_path(inode, &dest_path); + (void)smoothfs_lower_ino_map_insert(sbi, si->intended_tier, + d_inode(dest_path.dentry)->i_ino, + inode->i_ino); if (rel_path) { smoothfs_path_map_del(sbi, si); kfree(si->rel_path); @@ -702,6 +747,96 @@ void smoothfs_clear_pool_mapping_quiesce(struct smoothfs_sb_info *sbi) up_read(&sbi->inode_lock); } +/* Only userspace knows when the old source has been removed durably. Keep + * SWITCHED until that acknowledgement; resetting during cutover would let a + * second move race the first mover's source cleanup. The transaction sequence + * also makes retries idempotent without accepting a stale mover's completion. + */ +int smoothfs_movement_complete(struct smoothfs_sb_info *sbi, + const u8 oid[SMOOTHFS_OID_LEN], + u64 transaction_seq) +{ + struct smoothfs_inode_info *si; + struct inode *inode; + int err = 0; + + si = smoothfs_lookup_oid_resolve(sbi, oid); + if (!si) + return -ENOENT; + inode = &si->vfs_inode; + inode_lock(inode); + if (si->transaction_seq != transaction_seq) { + err = -ESTALE; + goto out; + } + if (si->movement_state == SMOOTHFS_MS_PLACED) + goto out; + if (si->movement_state != SMOOTHFS_MS_SWITCHED) { + err = -EBUSY; + goto out; + } + err = + smoothfs_placement_record(sbi, oid, SMOOTHFS_MS_PLACED, + si->current_tier, si->current_tier, true); + if (err) + goto out; + si->movement_state = SMOOTHFS_MS_PLACED; + si->intended_tier = si->current_tier; + smoothfs_netlink_emit_move_state(sbi, oid, SMOOTHFS_MS_PLACED, + transaction_seq); +out: + inode_unlock(inode); + return err; +} +/* Recovery may discard a partial destination only before cutover. The + * sequence guard rejects an old recovery task after a newer movement began. + * Unlike the informational abort transition, this acknowledgement durably + * returns the authoritative source to PLACED so movement can resume. + */ +int smoothfs_movement_rollback(struct smoothfs_sb_info *sbi, + const u8 oid[SMOOTHFS_OID_LEN], + u64 transaction_seq) +{ + struct smoothfs_inode_info *si = smoothfs_lookup_oid_resolve(sbi, oid); + struct inode *inode; + int err = 0; + + if (!si) + return -ENOENT; + inode = &si->vfs_inode; + inode_lock(inode); + if (si->transaction_seq != transaction_seq) { + err = -ESTALE; + goto out; + } + switch (si->movement_state) { + case SMOOTHFS_MS_PLACED: + goto out; + case SMOOTHFS_MS_PLAN_ACCEPTED: + case SMOOTHFS_MS_DESTINATION_RESERVED: + case SMOOTHFS_MS_COPY_IN_PROGRESS: + case SMOOTHFS_MS_COPY_COMPLETE: + case SMOOTHFS_MS_COPY_VERIFIED: + case SMOOTHFS_MS_FAILED: + break; + default: + err = -EBUSY; + goto out; + } + err = + smoothfs_placement_record(sbi, oid, SMOOTHFS_MS_PLACED, + si->current_tier, si->current_tier, true); + if (!err) { + si->movement_state = SMOOTHFS_MS_PLACED; + si->intended_tier = si->current_tier; + smoothfs_netlink_emit_move_state(sbi, oid, SMOOTHFS_MS_PLACED, + transaction_seq); + } +out: + inode_unlock(inode); + return err; +} + int smoothfs_movement_abort(struct smoothfs_sb_info *sbi, const u8 oid[SMOOTHFS_OID_LEN], u64 transaction_seq, const char *reason) diff --git a/src/smoothfs/netlink.c b/src/smoothfs/netlink.c index b8b9f9c..7821900 100644 --- a/src/smoothfs/netlink.c +++ b/src/smoothfs/netlink.c @@ -241,6 +241,40 @@ static int doit_move_cutover(struct sk_buff *skb, struct genl_info *info) expected_write_seq, check_write_seq); } +static int doit_move_complete(struct sk_buff *skb, struct genl_info *info) +{ + struct smoothfs_sb_info *sbi; + const u8 *oid; + u64 seq; + + (void)skb; + sbi = resolve_pool(info); + if (IS_ERR(sbi)) + return PTR_ERR(sbi); + if (!info->attrs[SMOOTHFS_ATTR_OBJECT_ID] || + !info->attrs[SMOOTHFS_ATTR_TRANSACTION_SEQ]) + return -EINVAL; + oid = nla_data(info->attrs[SMOOTHFS_ATTR_OBJECT_ID]); + seq = nla_get_u64(info->attrs[SMOOTHFS_ATTR_TRANSACTION_SEQ]); + return smoothfs_movement_complete(sbi, oid, seq); +} +static int doit_move_rollback(struct sk_buff *skb, struct genl_info *info) +{ + struct smoothfs_sb_info *sbi; + const u8 *oid; + u64 seq; + + (void)skb; + sbi = resolve_pool(info); + if (IS_ERR(sbi)) + return PTR_ERR(sbi); + if (!info->attrs[SMOOTHFS_ATTR_OBJECT_ID] || + !info->attrs[SMOOTHFS_ATTR_TRANSACTION_SEQ]) + return -EINVAL; + oid = nla_data(info->attrs[SMOOTHFS_ATTR_OBJECT_ID]); + seq = nla_get_u64(info->attrs[SMOOTHFS_ATTR_TRANSACTION_SEQ]); + return smoothfs_movement_rollback(sbi, oid, seq); +} static int doit_inspect(struct sk_buff *skb, struct genl_info *info) { struct smoothfs_sb_info *sbi; @@ -382,6 +416,10 @@ static const struct genl_multicast_group smoothfs_genl_mcgrps[] = { }; static const struct genl_small_ops smoothfs_genl_ops[] = { + { .cmd = SMOOTHFS_CMD_MOVE_ROLLBACK, .flags = GENL_ADMIN_PERM, + .doit = doit_move_rollback }, + { .cmd = SMOOTHFS_CMD_MOVE_COMPLETE, .flags = GENL_ADMIN_PERM, + .doit = doit_move_complete }, { .cmd = SMOOTHFS_CMD_REGISTER_POOL, .flags = GENL_ADMIN_PERM, .doit = doit_register_pool }, { .cmd = SMOOTHFS_CMD_POLICY_PUSH, .flags = GENL_ADMIN_PERM, diff --git a/src/smoothfs/smoothfs.h b/src/smoothfs/smoothfs.h index 09dad5f..9570210 100644 --- a/src/smoothfs/smoothfs.h +++ b/src/smoothfs/smoothfs.h @@ -814,6 +814,12 @@ int smoothfs_movement_cutover(struct smoothfs_sb_info *sbi, u64 transaction_seq, u64 expected_write_seq, bool check_write_seq); +int smoothfs_movement_complete(struct smoothfs_sb_info *sbi, + const u8 oid[SMOOTHFS_OID_LEN], + u64 transaction_seq); +int smoothfs_movement_rollback(struct smoothfs_sb_info *sbi, + const u8 oid[SMOOTHFS_OID_LEN], + u64 transaction_seq); int smoothfs_movement_abort(struct smoothfs_sb_info *sbi, const u8 oid[SMOOTHFS_OID_LEN], u64 transaction_seq, const char *reason); diff --git a/src/smoothfs/super.c b/src/smoothfs/super.c index d8d1ac8..5d06e7e 100644 --- a/src/smoothfs/super.c +++ b/src/smoothfs/super.c @@ -2421,32 +2421,53 @@ static int smoothfs_show_options(struct seq_file *m, struct dentry *root) } /* - * sync_fs: drain deferred SmoothFS metadata so `sync` / `syncfs` / - * the periodic writeback pass all see up-to-date on-disk state. - * Called with wait=1 for "make it durable", wait=0 for "kick it". - * We flush synchronously when asked to wait and just nudge workers - * when we aren't. + * File data lives in the backing superblocks' address spaces. Draining only + * SmoothFS metadata is not a syncfs durability barrier (PBS uses syncfs when + * completing a backup). Flush each distinct backing FS when VFS asks to wait. */ static int smoothfs_sync_fs(struct super_block *sb, int wait) { struct smoothfs_sb_info *sbi = SMOOTHFS_SB(sb); + int err = 0, ret; + u8 tier, prior; if (!sbi) return 0; if (wait) { - int err = smoothfs_cache_drain(sbi); - if (err) - return err; + ret = smoothfs_cache_drain(sbi); + if (ret) + err = ret; smoothfs_oid_wb_drain(sbi); smoothfs_placement_wb_drain(sbi); - if (sbi->placement_log) - vfs_fsync(sbi->placement_log, 0); + if (sbi->placement_log) { + ret = vfs_fsync(sbi->placement_log, 0); + if (ret && !err) + err = ret; + } + for (tier = 0; tier < sbi->ntiers; tier++) { + struct super_block *lower = + sbi->tiers[tier].lower_path.mnt->mnt_sb; + + for (prior = 0; prior < tier; prior++) + if (sbi->tiers[prior].lower_path.mnt->mnt_sb == + lower) + break; + if (prior != tier) + continue; + /* Match the stacked-filesystem lock contract used by + * overlayfs. */ + down_read(&lower->s_umount); + ret = sync_filesystem(lower); + up_read(&lower->s_umount); + if (ret && !err) + err = ret; + } } else { smoothfs_placement_wb_kick(sbi); if (sbi->oid_wb_ready) mod_delayed_work(sbi->oid_wb_wq, &sbi->oid_wb_work, 0); } - return 0; + return err; } const struct super_operations smoothfs_super_ops = { @@ -2496,8 +2517,24 @@ struct inode *smoothfs_iget(struct super_block *sb, struct path *lower, lower_inode->i_ino); if (ino_no) { inode = ilookup(sb, ino_no); - if (inode) - return inode; + if (inode) { + struct path cached; + bool matches; + + /* Numeric lower inode numbers can be reused. Validate + * the cached upper's actual backing before returning it; + * never attach a fresh file to a moved object's inode. + */ + smoothfs_get_lower_path(inode, &cached); + matches = cached.mnt == lower->mnt && cached.dentry && + d_inode(cached.dentry) == lower_inode; + path_put(&cached); + if (matches) + return inode; + smoothfs_lower_ino_map_remove(sbi, tier_idx, + lower_inode->i_ino); + iput(inode); + } /* Cache hit but inode was evicted — treat as miss * and rebuild state from xattrs. */ } diff --git a/src/smoothfs/test/directory_fsync.sh b/src/smoothfs/test/directory_fsync.sh new file mode 100755 index 0000000..de0cfe4 --- /dev/null +++ b/src/smoothfs/test/directory_fsync.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Namespace barriers must work for both canonical and merged directories. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +. "$SCRIPT_DIR/tier_spill_lib.sh" +SPILL_ROOT=${SMOOTHFS_DIR_FSYNC_ROOT:-/tmp/smoothfs-directory-fsync} +SPILL_UUID=00000000-0000-0000-0000-00000000f51c +trap spill_cleanup EXIT +spill_setup_pool directoryfsync "$SPILL_UUID" + +# Seed a contributing directory on each tier, and one absent from tier 1. +mkdir -p "$SPILL_ROOT/fast/merged" "$SPILL_ROOT/slow/merged" +mkdir "$SPILL_ROOT/fast/canonical" +printf 'slow-tier entry\n' > "$SPILL_ROOT/slow/merged/slow" +python3 - "$SPILL_ROOT/server" <<'PY' +import os +import pathlib +import sys + +root = pathlib.Path(sys.argv[1]) +for directory in (root, root / "canonical", root / "merged"): + fd = os.open(directory, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + os.fdatasync(fd) + source = directory / "barrier.tmp" + target = directory / "barrier" + with source.open("wb") as handle: + handle.write(b"durable namespace entry\n") + handle.flush() + os.fsync(handle.fileno()) + source.rename(target) + os.fsync(fd) + assert target.read_bytes() == b"durable namespace entry\n" + target.unlink() + os.fsync(fd) + finally: + os.close(fd) +assert (root / "merged" / "slow").read_bytes() == b"slow-tier entry\n" +print("directory_fsync: PASS (root, canonical, merged, rename and unlink)") +PY diff --git a/src/smoothfs/test/mkdir_enospc.sh b/src/smoothfs/test/mkdir_enospc.sh new file mode 100755 index 0000000..de87a88 --- /dev/null +++ b/src/smoothfs/test/mkdir_enospc.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# mkdir failure must release the lower parent lock on every supported kernel. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +. "$SCRIPT_DIR/tier_spill_lib.sh" +SPILL_ROOT=${SMOOTHFS_MKDIR_ENOSPC_ROOT:-/tmp/smoothfs-mkdir-enospc} +SPILL_UUID=00000000-0000-0000-0000-00000000f51f +trap spill_cleanup EXIT +spill_setup_pool mkdirenospc "$SPILL_UUID" +python3 - "$SPILL_ROOT" <<'PY' +import errno,os,signal,subprocess,sys +from pathlib import Path +root=Path(sys.argv[1]);server=root/'server' +(server/'fast-parent').mkdir() +for tier in ('fast','slow'): + base=root/tier + with (base/'fill').open('wb',buffering=0) as f: + st=os.statvfs(base) + try:os.posix_fallocate(f.fileno(),0,max(0,st.f_bavail*st.f_frsize-2*1024*1024)) + except OSError as e: + if e.errno!=errno.ENOSPC:raise + f.seek(0,os.SEEK_END) + while True: + try:f.write(bytes(65536)) + except OSError as e: + if e.errno!=errno.ENOSPC:raise + break + # Exhaust remaining inode space too, so the lower mkdir itself must fail. + for n in range(8192): + try:(base/('padding-'+str(n))).mkdir() + except OSError as e: + if e.errno!=errno.ENOSPC:raise + break + else:raise RuntimeError('lower mkdir ENOSPC was not exercised') + print('LOWER_FULL',tier,flush=True) +probe=r''' +import errno,os,sys +from pathlib import Path +p=Path(sys.argv[1]) +for target,missing in [(p/'fast-parent'/'must-fail',p/'missing-after-materialization'),(p/'must-also-fail',p/'missing-after-mkdir')]: + try:target.mkdir() + except OSError as e: + if e.errno!=errno.ENOSPC:raise + else:raise RuntimeError('mkdir unexpectedly succeeded on full tiers') + print('MKDIR_ENOSPC_RETURNED',target.name,flush=True) + try:os.stat(missing) + except FileNotFoundError:pass + else:raise RuntimeError('missing path unexpectedly exists') + print('PARENT_LOOKUP_RESPONSIVE',flush=True) +''' +child=subprocess.Popen([sys.executable,'-c',probe,str(server)],start_new_session=True) +try:code=child.wait(timeout=10) +except subprocess.TimeoutExpired: + # A broken module can leave the probe uninterruptible. Report the exact PID; + # the disposable test guest must be reset before treating cleanup as complete. + os.kill(child.pid,signal.SIGKILL) + print('mkdir_enospc: FAIL (lower parent lock leaked; probe PID '+str(child.pid)+')',flush=True) + os._exit(1) +if code:raise RuntimeError('mkdir probe failed') +for tier in ('fast','slow'):(root/tier/'fill').unlink() +(server/'fast-parent'/'recovered').mkdir() +path=server/'fast-parent'/'recovered'/'data' +with path.open('wb') as f:f.write(b'write after freeing space\n');f.flush();os.fsync(f.fileno()) +assert path.read_bytes()==b'write after freeing space\n' +print('mkdir_enospc: PASS (materialization and direct mkdir ENOSPC, responsive lookup, write recovery)') +PY diff --git a/src/smoothfs/test/movement_inode_reuse.sh b/src/smoothfs/test/movement_inode_reuse.sh new file mode 100755 index 0000000..7d76627 --- /dev/null +++ b/src/smoothfs/test/movement_inode_reuse.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# A freed source inode must never alias the object moved to another tier. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT=${SMOOTHFS_TEST_GO_ROOT:-$(cd "$SCRIPT_DIR/../../.." && pwd)} +. "$SCRIPT_DIR/tier_spill_lib.sh" +SPILL_ROOT=${SMOOTHFS_INODE_REUSE_ROOT:-/tmp/smoothfs-movement-inode-reuse} +SPILL_UUID=00000000-0000-0000-0000-00000000f51d +trap spill_cleanup EXIT +spill_setup_pool inodereuse "$SPILL_UUID" +cat > "$SPILL_ROOT/probe.go" <<'GO' +package main +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "syscall" + smoothfs "github.com/RakuenSoftware/smoothfs" + "github.com/google/uuid" + "golang.org/x/sys/unix" +) +func must(err error) { if err != nil { panic(err) } } +func main() { + root:=os.Args[1]; pool:=uuid.MustParse(os.Args[2]) + upper:=filepath.Join(root,"server","victim") + source:=filepath.Join(root,"fast","victim") + dest:=filepath.Join(root,"slow","victim") + payload:=bytes.Repeat([]byte("PBS chunk identity must survive inode reuse\n"),256) + must(os.WriteFile(upper,payload,0600)) + dir,err:=os.Open(filepath.Join(root,"server"));must(err);must(dir.Sync());must(dir.Close()) + original,err:=os.Stat(source);must(err);oldIno:=original.Sys().(*syscall.Stat_t).Ino + raw:=make([]byte,smoothfs.OIDLen);_,err=unix.Getxattr(upper,"trusted.smoothfs.oid",raw);must(err) + var oid [smoothfs.OIDLen]byte;copy(oid[:],raw) + client,err:=smoothfs.Open();must(err);defer client.Close() + inspect,err:=client.Inspect(pool,oid);must(err) + must(client.MovePlan(pool,oid,1,1)) + in,err:=os.Open(source);must(err);out,err:=os.Create(dest);must(err) + _,err=io.Copy(out,in);must(err);must(out.Sync());must(out.Close());must(in.Close()) + before,err:=os.ReadFile(dest);must(err);if !bytes.Equal(before,payload){panic("copy differs before cutover")} + if inspect.HasWriteSeq {must(client.MoveCutoverVerifyWriteSeq(pool,oid,1,inspect.WriteSeq))} else {must(client.MoveCutover(pool,oid,1))} + must(os.Remove(source)) + // Hold the moved upper inode and its NEW backing file alive, while leaving + // the unlinked source inode free for the backing filesystem to reuse. + held,err:=os.Open(upper);must(err);defer held.Close() + reused:=false + for n:=0;n<16384;n++ { + path:=filepath.Join(root,"server","replacement") + must(os.WriteFile(path,[]byte("unrelated RRD journal data\n"),0600)) + info,err:=os.Stat(filepath.Join(root,"fast","replacement"));must(err) + if info.Sys().(*syscall.Stat_t).Ino==oldIno { + reused=true + actual,err:=os.ReadFile(dest);must(err) + if !bytes.Equal(actual,payload){panic("source inode reuse redirected an unrelated write into the moved object")} + must(os.Remove(path));break + } + must(os.Remove(path)) + } + if !reused {panic("backing filesystem did not reuse the source inode; regression was not exercised")} + actualOID:=make([]byte,smoothfs.OIDLen);_,err=unix.Getxattr(dest,"trusted.smoothfs.oid",actualOID);must(err) + if !bytes.Equal(actualOID,raw){panic("cutover did not preserve destination object identity")} + fmt.Println("movement_inode_reuse: PASS (actual source inode reuse, payload and persistent identity)") +} +GO +go -C "$REPO_ROOT" run "$SPILL_ROOT/probe.go" "$SPILL_ROOT" "$SPILL_UUID" diff --git a/src/smoothfs/test/movement_roundtrip.sh b/src/smoothfs/test/movement_roundtrip.sh new file mode 100755 index 0000000..4aa5559 --- /dev/null +++ b/src/smoothfs/test/movement_roundtrip.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Successful cleanup must permit another movement without inode eviction/reboot. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT=${SMOOTHFS_TEST_GO_ROOT:-$(cd "$SCRIPT_DIR/../../.." && pwd)} +. "$SCRIPT_DIR/tier_spill_lib.sh" +SPILL_ROOT=${SMOOTHFS_ROUNDTRIP_ROOT:-/tmp/smoothfs-movement-roundtrip} +SPILL_UUID=00000000-0000-0000-0000-00000000f51e +trap spill_cleanup EXIT +spill_setup_pool roundtrip "$SPILL_UUID" +cat > "$SPILL_ROOT/probe.go" <<'GO' +package main +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "path/filepath" + smoothfs "github.com/RakuenSoftware/smoothfs" + "github.com/google/uuid" + "golang.org/x/sys/unix" +) +func must(e error){if e!=nil{panic(e)}} +func main(){ + root:=os.Args[1];pool:=uuid.MustParse(os.Args[2]);upper:=filepath.Join(root,"server","chunk") + payload:=bytes.Repeat([]byte("PBS repeated movement sentinel\n"),8192) + must(os.WriteFile(upper,payload,0600)) + held,e:=os.Open(upper);must(e);defer held.Close() + raw:=make([]byte,smoothfs.OIDLen);_,e=unix.Getxattr(upper,"trusted.smoothfs.oid",raw);must(e) + var oid [smoothfs.OIDLen]byte;copy(oid[:],raw) + c,e:=smoothfs.Open();must(e);defer c.Close() + tiers:=[]string{"fast","slow"} + for round:=0;round<4;round++{ + from,to:=round%2,1-round%2;seq:=uint64(500+round) + before,e:=c.Inspect(pool,oid);must(e) + if before.CurrentTier!=uint8(from)||before.MovementState!=smoothfs.StatePlaced{panic("object was not ready for the next movement")} + // A crash before cutover must discard only the partial destination and + // restore movement readiness on the original source. + rollbackSeq:=seq+1000 + must(c.MovePlan(pool,oid,uint8(to),rollbackSeq)) + if e=c.MoveRollback(pool,oid,rollbackSeq+1);!errors.Is(e,unix.ESTALE){panic(fmt.Sprintf("stale rollback: %v",e))} + partial:=filepath.Join(root,tiers[to],"chunk") + must(os.WriteFile(partial,payload[:128],0600));must(os.Remove(partial)) + parent,e:=os.Open(filepath.Dir(partial));must(e);must(parent.Sync());must(parent.Close()) + must(c.MoveRollback(pool,oid,rollbackSeq));must(c.MoveRollback(pool,oid,rollbackSeq)) + rolledBack,e:=c.Inspect(pool,oid);must(e) + if rolledBack.CurrentTier!=uint8(from)||rolledBack.MovementState!=smoothfs.StatePlaced{panic("rollback changed authoritative placement")} + must(c.MovePlan(pool,oid,uint8(to),seq)) + if e=c.MoveComplete(pool,oid,seq);!errors.Is(e,unix.EBUSY){panic(fmt.Sprintf("premature completion: %v",e))} + src:=filepath.Join(root,tiers[from],"chunk");dst:=filepath.Join(root,tiers[to],"chunk") + in,e:=os.Open(src);must(e);out,e:=os.Create(dst);must(e) + _,e=io.Copy(out,in);must(e);must(out.Sync());must(out.Close());must(in.Close()) + planned,e:=c.Inspect(pool,oid);must(e) + fmt.Printf("round=%d seq=%d kernel_seq=%d write_before=%d write_after_rollback=%d write_planned=%d\n",round,seq,planned.TransactionSeq,before.WriteSeq,rolledBack.WriteSeq,planned.WriteSeq) + if rolledBack.HasWriteSeq{must(c.MoveCutoverVerifyWriteSeq(pool,oid,seq,rolledBack.WriteSeq))}else{must(c.MoveCutover(pool,oid,seq))} + if e=c.MoveRollback(pool,oid,seq);!errors.Is(e,unix.EBUSY){panic(fmt.Sprintf("rollback accepted after cutover: %v",e))} + if e=c.MovePlan(pool,oid,uint8(from),seq+100);!errors.Is(e,unix.EBUSY){panic(fmt.Sprintf("new movement before cleanup ack: %v",e))} + if e=c.MoveComplete(pool,oid,seq+1);!errors.Is(e,unix.ESTALE){panic(fmt.Sprintf("stale completion: %v",e))} + must(os.Remove(src));dir,e:=os.Open(filepath.Dir(src));must(e);must(dir.Sync());must(dir.Close()) + must(c.MoveComplete(pool,oid,seq));must(c.MoveComplete(pool,oid,seq)) + after,e:=c.Inspect(pool,oid);must(e) + if after.CurrentTier!=uint8(to)||after.MovementState!=smoothfs.StatePlaced{panic("cleanup did not restore placed state")} + _,e=held.Seek(0,0);must(e);actual,e:=io.ReadAll(held);must(e) + if !bytes.Equal(actual,payload){panic("held file changed after roundtrip")} + } + fmt.Println("movement_roundtrip: PASS (four moves and rollbacks, held descriptor, premature/stale/idempotent acknowledgements)") +} +GO +go -C "$REPO_ROOT" run "$SPILL_ROOT/probe.go" "$SPILL_ROOT" "$SPILL_UUID" diff --git a/src/smoothfs/test/readdir_backing_permissions.sh b/src/smoothfs/test/readdir_backing_permissions.sh new file mode 100755 index 0000000..4829b48 --- /dev/null +++ b/src/smoothfs/test/readdir_backing_permissions.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# The presented owner must see merged entries through restrictive backing dirs. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +. "$SCRIPT_DIR/tier_spill_lib.sh" +SPILL_ROOT=${SMOOTHFS_READDIR_PERMISSIONS_ROOT:-/tmp/smoothfs-readdir-permissions} +SPILL_UUID=00000000-0000-0000-0000-00000000f520 +trap spill_cleanup EXIT +spill_setup_pool readdirpermissions "$SPILL_UUID" +mkdir -p "$SPILL_ROOT/fast/private/merged" "$SPILL_ROOT/slow/private/merged" "$SPILL_ROOT/bind" +chmod 700 "$SPILL_ROOT/fast/private" "$SPILL_ROOT/slow/private" +printf 'fast tier data\n' > "$SPILL_ROOT/fast/private/merged/fast" +printf 'slow tier data\n' > "$SPILL_ROOT/slow/private/merged/slow" +mount --bind "$SPILL_ROOT/server/private/merged" "$SPILL_ROOT/bind" +trap 'umount "$SPILL_ROOT/bind"; spill_cleanup' EXIT +python3 - "$SPILL_ROOT" <<'PY' +from pathlib import Path +import os,subprocess,sys +root=Path(sys.argv[1]);expected={'fast','slow'} +probe=r''' +import os,sys +for path in sys.argv[1:]: + names={e.name for e in os.scandir(path)} + assert names=={'fast','slow'},f'UID {os.geteuid()} merged listing {names}' + for name in names: + with open(os.path.join(path,name)) as f:assert f.read()==name+' tier data\n' +print('PRESENTED_OWNER_MERGED_READDIR_OK',os.geteuid()) +''' +for path in (root/'server/private/merged',root/'bind'):assert set(os.listdir(path))==expected +subprocess.run([sys.executable,'-c',probe,str(root/'server/private/merged'),str(root/'bind')],user=1000,group=1000,extra_groups=[],check=True) +# Privilege applies to lower I/O only; upper directory DAC must still reject +# a different caller before the filesystem iterator runs. +code='import os,sys\ntry:os.listdir(sys.argv[1])\nexcept PermissionError:sys.exit(0)\nraise RuntimeError("upper DAC bypassed")' +subprocess.run([sys.executable,'-c',code,str(root/'server/private/merged')],user=1001,group=1001,extra_groups=[],check=True) +# A broken contributing path must fail enumeration instead of returning a +# partial listing that a backup collector could mistake for a complete tree. +slow=root/'slow/private/merged';saved=root/'slow/private/saved' +slow.rename(saved);slow.symlink_to('merged') +try: + try:os.listdir(root/'server/private/merged') + except OSError as e: + import errno + assert e.errno==errno.ELOOP,e + else:raise RuntimeError('partial listing hid a contributing-tier lookup error') +finally: + slow.unlink();saved.rename(slow) +print('readdir_backing_permissions: PASS (owner union, bind mount, upper DAC, contributing-error propagation)') +PY diff --git a/src/smoothfs/test/run_runtime_harnesses.sh b/src/smoothfs/test/run_runtime_harnesses.sh index 229d812..02e0b7c 100755 --- a/src/smoothfs/test/run_runtime_harnesses.sh +++ b/src/smoothfs/test/run_runtime_harnesses.sh @@ -24,6 +24,10 @@ FAIL_FAST=${SMOOTHFS_RUNTIME_FAIL_FAST:-0} core_tests=( cache_mode.sh + directory_fsync.sh + syncfs_lower.sh + readdir_backing_permissions.sh + mkdir_enospc.sh netlink_receive_close.sh tier_spill_basic_create.sh tier_spill_nested_parent.sh @@ -48,6 +52,8 @@ core_tests=( movement_nested_cutover.sh movement_open_fd_cutover.sh movement_getattr_cutover.sh + movement_inode_reuse.sh + movement_roundtrip.sh write_staging_direct_cutover.sh odirect.sh ) diff --git a/src/smoothfs/test/syncfs_lower.sh b/src/smoothfs/test/syncfs_lower.sh new file mode 100755 index 0000000..0934bc7 --- /dev/null +++ b/src/smoothfs/test/syncfs_lower.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# syncfs on a stacked mount must flush buffered file data on every backing FS. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +. "$SCRIPT_DIR/tier_spill_lib.sh" +SPILL_ROOT=${SMOOTHFS_SYNCFS_ROOT:-/tmp/smoothfs-syncfs-lower} +SPILL_UUID=00000000-0000-0000-0000-00000000f520 +trap spill_cleanup EXIT +spill_setup_pool syncfs-lower "$SPILL_UUID" +python3 - "$SPILL_ROOT" <<'PY' +import ctypes,fcntl,os,struct,sys +from pathlib import Path +root=Path(sys.argv[1]);payload=b'backing filesystem sync boundary\n'*(256*1024) +paths=[root/'fast'/'direct-fast',root/'slow'/'direct-slow'] +for p in paths: + with p.open('wb') as f:f.write(payload) +with (root/'server'/'upper-write').open('wb') as f:f.write(payload) +paths.append(root/'fast'/'upper-write') +def delayed(p): + fd=os.open(p,os.O_RDONLY) + try: + buf=bytearray(struct.pack('