From ffbe6463ee39739b5e4ec95c97b75887a104e037 Mon Sep 17 00:00:00 2001 From: virant Date: Sat, 31 Oct 2026 16:43:01 +0100 Subject: [PATCH 1/7] fix: support fsync on merged directories --- src/smoothfs/dir.c | 68 ++++++++++++++++++++++ src/smoothfs/test/directory_fsync.sh | 41 +++++++++++++ src/smoothfs/test/run_runtime_harnesses.sh | 1 + 3 files changed, 110 insertions(+) create mode 100755 src/smoothfs/test/directory_fsync.sh diff --git a/src/smoothfs/dir.c b/src/smoothfs/dir.c index 059c827..a069332 100644 --- a/src/smoothfs/dir.c +++ b/src/smoothfs/dir.c @@ -273,11 +273,79 @@ static loff_t smoothfs_dir_llseek(struct file *file, loff_t offset, int whence) return ret; } +/* A directory can have entries on several tiers. Making only its canonical + * lower durable would miss names created by spill or movement on another tier. + * Resolve and sync every contributing lower directory; an explicit durability + * request must not silently skip an unavailable tier or a lower fsync error. + */ +static int smoothfs_dir_fsync(struct file *file, loff_t start, loff_t end, + int datasync) +{ + struct smoothfs_sb_info *sbi = SMOOTHFS_SB(file_inode(file)->i_sb); + struct file *lower = smoothfs_lower_file(file); + char *rel_path; + unsigned int tier; + int err; + + if (IS_ERR(lower)) + return PTR_ERR(lower); + if (!lower) + return -EBADF; + + rel_path = smoothfs_rel_path_from_dentry(file->f_path.dentry); + if (!rel_path) + return -ENOMEM; + + /* Persist deferred identities and placement records before acknowledging + * the namespace barrier. Propagate the placement log's fsync failure too. + */ + smoothfs_oid_wb_drain(sbi); + smoothfs_placement_wb_drain(sbi); + err = vfs_fsync(lower, datasync); + if (err) + goto out; + + for (tier = 0; tier < sbi->ntiers; tier++) { + struct path path; + struct file *dir; + + err = smoothfs_resolve_rel_path_on_tier(sbi, tier, rel_path, &path); + if (err == -ENOENT || err == -ENOTDIR) { + err = 0; + continue; + } + if (err) + goto out; + if (!d_is_dir(path.dentry) || + (path.mnt == lower->f_path.mnt && + path.dentry == lower->f_path.dentry)) { + path_put(&path); + continue; + } + dir = dentry_open(&path, O_RDONLY | O_DIRECTORY, sbi->creator_cred); + path_put(&path); + if (IS_ERR(dir)) { + err = PTR_ERR(dir); + goto out; + } + err = vfs_fsync(dir, datasync); + fput(dir); + if (err) + goto out; + } + if (sbi->placement_log) + err = vfs_fsync(sbi->placement_log, 0); +out: + kfree(rel_path); + return err; +} + const struct file_operations smoothfs_dir_ops = { .owner = THIS_MODULE, .open = smoothfs_opendir, .release = smoothfs_releasedir, .iterate_shared = smoothfs_iterate_shared, + .fsync = smoothfs_dir_fsync, .llseek = smoothfs_dir_llseek, .read = generic_read_dir, }; diff --git a/src/smoothfs/test/directory_fsync.sh b/src/smoothfs/test/directory_fsync.sh new file mode 100755 index 0000000..de0cfe4 --- /dev/null +++ b/src/smoothfs/test/directory_fsync.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Namespace barriers must work for both canonical and merged directories. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +. "$SCRIPT_DIR/tier_spill_lib.sh" +SPILL_ROOT=${SMOOTHFS_DIR_FSYNC_ROOT:-/tmp/smoothfs-directory-fsync} +SPILL_UUID=00000000-0000-0000-0000-00000000f51c +trap spill_cleanup EXIT +spill_setup_pool directoryfsync "$SPILL_UUID" + +# Seed a contributing directory on each tier, and one absent from tier 1. +mkdir -p "$SPILL_ROOT/fast/merged" "$SPILL_ROOT/slow/merged" +mkdir "$SPILL_ROOT/fast/canonical" +printf 'slow-tier entry\n' > "$SPILL_ROOT/slow/merged/slow" +python3 - "$SPILL_ROOT/server" <<'PY' +import os +import pathlib +import sys + +root = pathlib.Path(sys.argv[1]) +for directory in (root, root / "canonical", root / "merged"): + fd = os.open(directory, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + os.fdatasync(fd) + source = directory / "barrier.tmp" + target = directory / "barrier" + with source.open("wb") as handle: + handle.write(b"durable namespace entry\n") + handle.flush() + os.fsync(handle.fileno()) + source.rename(target) + os.fsync(fd) + assert target.read_bytes() == b"durable namespace entry\n" + target.unlink() + os.fsync(fd) + finally: + os.close(fd) +assert (root / "merged" / "slow").read_bytes() == b"slow-tier entry\n" +print("directory_fsync: PASS (root, canonical, merged, rename and unlink)") +PY diff --git a/src/smoothfs/test/run_runtime_harnesses.sh b/src/smoothfs/test/run_runtime_harnesses.sh index 229d812..8f51e8a 100755 --- a/src/smoothfs/test/run_runtime_harnesses.sh +++ b/src/smoothfs/test/run_runtime_harnesses.sh @@ -24,6 +24,7 @@ FAIL_FAST=${SMOOTHFS_RUNTIME_FAIL_FAST:-0} core_tests=( cache_mode.sh + directory_fsync.sh netlink_receive_close.sh tier_spill_basic_create.sh tier_spill_nested_parent.sh From c1392ccd1bde24dc89d5166d5d9b403497ca3dda Mon Sep 17 00:00:00 2001 From: virant Date: Sat, 31 Oct 2026 17:07:44 +0100 Subject: [PATCH 2/7] fix: prevent inode reuse from corrupting moved files --- src/smoothfs/movement.c | 45 +++++++++++++++ src/smoothfs/super.c | 20 ++++++- src/smoothfs/test/movement_inode_reuse.sh | 66 ++++++++++++++++++++++ src/smoothfs/test/run_runtime_harnesses.sh | 1 + 4 files changed, 130 insertions(+), 2 deletions(-) create mode 100755 src/smoothfs/test/movement_inode_reuse.sh diff --git a/src/smoothfs/movement.c b/src/smoothfs/movement.c index dcae4a6..94a8a45 100644 --- a/src/smoothfs/movement.c +++ b/src/smoothfs/movement.c @@ -595,10 +595,55 @@ int smoothfs_movement_cutover(struct smoothfs_sb_info *sbi, goto out_fail; } + /* The userspace mover copies bytes, not trusted identity xattrs. Persist + * the same object identity on the new backing before making it + * authoritative, so eviction or remount cannot mint a second identity. + */ + { + const struct cred *old = override_creds(sbi->creator_cred); + struct file *dest; + + err = smoothfs_write_oid_xattr(dest_path.dentry, si->oid); + if (err == -EEXIST) { + u8 existing[SMOOTHFS_OID_LEN]; + + err = smoothfs_read_oid_xattr(dest_path.dentry, existing); + if (!err && memcmp(existing, si->oid, sizeof(existing))) + err = -ESTALE; + } + if (!err) + err = smoothfs_write_gen_xattr(dest_path.dentry, si->gen); + if (!err) { + dest = dentry_open(&dest_path, O_RDONLY | O_LARGEFILE, + sbi->creator_cred); + if (IS_ERR(dest)) { + err = PTR_ERR(dest); + } else { + err = vfs_fsync(dest, 0); + fput(dest); + } + } + revert_creds(old); + if (err) { + path_put(&dest_path); + goto out_fail; + } + } + /* Swap lower_path: keep refs balanced. The smoothfs dentry's * d_fsdata also points at the OLD lower dentry — update it too, * otherwise smoothfs_d_release dputs a freed pointer at umount. */ + /* Cleanup will free the source lower inode. Its numeric inode may be + * reused immediately by an unrelated file (for example an RRD journal). + * Leaving the old fast-path key mapped to this upper inode redirects + * writes to that unrelated file into the moved object's destination. + */ + smoothfs_lower_ino_map_remove(sbi, si->current_tier, + d_inode(src_dentry)->i_ino); old_path = smoothfs_swap_lower_path(inode, &dest_path); + (void)smoothfs_lower_ino_map_insert(sbi, si->intended_tier, + d_inode(dest_path.dentry)->i_ino, + inode->i_ino); if (rel_path) { smoothfs_path_map_del(sbi, si); kfree(si->rel_path); diff --git a/src/smoothfs/super.c b/src/smoothfs/super.c index d8d1ac8..4888aa3 100644 --- a/src/smoothfs/super.c +++ b/src/smoothfs/super.c @@ -2496,8 +2496,24 @@ struct inode *smoothfs_iget(struct super_block *sb, struct path *lower, lower_inode->i_ino); if (ino_no) { inode = ilookup(sb, ino_no); - if (inode) - return inode; + if (inode) { + struct path cached; + bool matches; + + /* Numeric lower inode numbers can be reused. Validate + * the cached upper's actual backing before returning it; + * never attach a fresh file to a moved object's inode. + */ + smoothfs_get_lower_path(inode, &cached); + matches = cached.mnt == lower->mnt && cached.dentry && + d_inode(cached.dentry) == lower_inode; + path_put(&cached); + if (matches) + return inode; + smoothfs_lower_ino_map_remove(sbi, tier_idx, + lower_inode->i_ino); + iput(inode); + } /* Cache hit but inode was evicted — treat as miss * and rebuild state from xattrs. */ } diff --git a/src/smoothfs/test/movement_inode_reuse.sh b/src/smoothfs/test/movement_inode_reuse.sh new file mode 100755 index 0000000..7d76627 --- /dev/null +++ b/src/smoothfs/test/movement_inode_reuse.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# A freed source inode must never alias the object moved to another tier. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT=${SMOOTHFS_TEST_GO_ROOT:-$(cd "$SCRIPT_DIR/../../.." && pwd)} +. "$SCRIPT_DIR/tier_spill_lib.sh" +SPILL_ROOT=${SMOOTHFS_INODE_REUSE_ROOT:-/tmp/smoothfs-movement-inode-reuse} +SPILL_UUID=00000000-0000-0000-0000-00000000f51d +trap spill_cleanup EXIT +spill_setup_pool inodereuse "$SPILL_UUID" +cat > "$SPILL_ROOT/probe.go" <<'GO' +package main +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "syscall" + smoothfs "github.com/RakuenSoftware/smoothfs" + "github.com/google/uuid" + "golang.org/x/sys/unix" +) +func must(err error) { if err != nil { panic(err) } } +func main() { + root:=os.Args[1]; pool:=uuid.MustParse(os.Args[2]) + upper:=filepath.Join(root,"server","victim") + source:=filepath.Join(root,"fast","victim") + dest:=filepath.Join(root,"slow","victim") + payload:=bytes.Repeat([]byte("PBS chunk identity must survive inode reuse\n"),256) + must(os.WriteFile(upper,payload,0600)) + dir,err:=os.Open(filepath.Join(root,"server"));must(err);must(dir.Sync());must(dir.Close()) + original,err:=os.Stat(source);must(err);oldIno:=original.Sys().(*syscall.Stat_t).Ino + raw:=make([]byte,smoothfs.OIDLen);_,err=unix.Getxattr(upper,"trusted.smoothfs.oid",raw);must(err) + var oid [smoothfs.OIDLen]byte;copy(oid[:],raw) + client,err:=smoothfs.Open();must(err);defer client.Close() + inspect,err:=client.Inspect(pool,oid);must(err) + must(client.MovePlan(pool,oid,1,1)) + in,err:=os.Open(source);must(err);out,err:=os.Create(dest);must(err) + _,err=io.Copy(out,in);must(err);must(out.Sync());must(out.Close());must(in.Close()) + before,err:=os.ReadFile(dest);must(err);if !bytes.Equal(before,payload){panic("copy differs before cutover")} + if inspect.HasWriteSeq {must(client.MoveCutoverVerifyWriteSeq(pool,oid,1,inspect.WriteSeq))} else {must(client.MoveCutover(pool,oid,1))} + must(os.Remove(source)) + // Hold the moved upper inode and its NEW backing file alive, while leaving + // the unlinked source inode free for the backing filesystem to reuse. + held,err:=os.Open(upper);must(err);defer held.Close() + reused:=false + for n:=0;n<16384;n++ { + path:=filepath.Join(root,"server","replacement") + must(os.WriteFile(path,[]byte("unrelated RRD journal data\n"),0600)) + info,err:=os.Stat(filepath.Join(root,"fast","replacement"));must(err) + if info.Sys().(*syscall.Stat_t).Ino==oldIno { + reused=true + actual,err:=os.ReadFile(dest);must(err) + if !bytes.Equal(actual,payload){panic("source inode reuse redirected an unrelated write into the moved object")} + must(os.Remove(path));break + } + must(os.Remove(path)) + } + if !reused {panic("backing filesystem did not reuse the source inode; regression was not exercised")} + actualOID:=make([]byte,smoothfs.OIDLen);_,err=unix.Getxattr(dest,"trusted.smoothfs.oid",actualOID);must(err) + if !bytes.Equal(actualOID,raw){panic("cutover did not preserve destination object identity")} + fmt.Println("movement_inode_reuse: PASS (actual source inode reuse, payload and persistent identity)") +} +GO +go -C "$REPO_ROOT" run "$SPILL_ROOT/probe.go" "$SPILL_ROOT" "$SPILL_UUID" diff --git a/src/smoothfs/test/run_runtime_harnesses.sh b/src/smoothfs/test/run_runtime_harnesses.sh index 8f51e8a..3ae718b 100755 --- a/src/smoothfs/test/run_runtime_harnesses.sh +++ b/src/smoothfs/test/run_runtime_harnesses.sh @@ -49,6 +49,7 @@ core_tests=( movement_nested_cutover.sh movement_open_fd_cutover.sh movement_getattr_cutover.sh + movement_inode_reuse.sh write_staging_direct_cutover.sh odirect.sh ) From e2d838d1fe8f68c85a30199ce6771d052a9d4b5e Mon Sep 17 00:00:00 2001 From: virant Date: Sat, 31 Oct 2026 19:12:39 +0100 Subject: [PATCH 3/7] fix: complete durable movement cleanup before allowing another move Add an explicit MOVE_COMPLETE acknowledgement after source unlink and directory fsync. Keep switched state until cleanup succeeds, reject stale or premature acknowledgements, and allow repeat moves without eviction or remount. Cover worker cleanup errors and repeated live kernel movement with a held descriptor. --- client.go | 7 ++ controlplane/worker.go | 30 ++++++- controlplane/worker_completion_test.go | 95 ++++++++++++++++++++++ src/smoothfs/movement.c | 41 ++++++++++ src/smoothfs/netlink.c | 19 +++++ src/smoothfs/smoothfs.h | 3 + src/smoothfs/test/movement_roundtrip.sh | 56 +++++++++++++ src/smoothfs/test/run_runtime_harnesses.sh | 1 + src/smoothfs/uapi_smoothfs.h | 1 + uapi.go | 1 + 10 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 controlplane/worker_completion_test.go create mode 100755 src/smoothfs/test/movement_roundtrip.sh diff --git a/client.go b/client.go index 0caa9e1..2756f3b 100644 --- a/client.go +++ b/client.go @@ -136,6 +136,13 @@ func (c *Client) MoveCutoverVerifyWriteSeq(poolUUID uuid.UUID, oid [OIDLen]byte, return err } +// MoveComplete acknowledges durable source cleanup after cutover. Until this +// handshake, the kernel keeps the object switched and rejects another move. +func (c *Client) MoveComplete(poolUUID uuid.UUID, oid [OIDLen]byte, seq uint64) error { + _, err := c.send(CmdMoveComplete, cutoverAttrs(poolUUID, oid, seq, nil)) + return err +} + func cutoverAttrs(poolUUID uuid.UUID, oid [OIDLen]byte, seq uint64, writeSeq *uint64) []netlink.Attribute { attrs := []netlink.Attribute{ {Type: AttrPoolUUID, Data: poolUUID[:]}, diff --git a/controlplane/worker.go b/controlplane/worker.go index 0c4c2a8..3838cbd 100644 --- a/controlplane/worker.go +++ b/controlplane/worker.go @@ -76,6 +76,10 @@ type writeSeqCutoverClient interface { MoveCutoverVerifyWriteSeq(poolUUID uuid.UUID, oid [OIDLen]byte, seq, writeSeq uint64) error } +type movementCompletionClient interface { + MoveComplete(poolUUID uuid.UUID, oid [OIDLen]byte, seq uint64) error +} + type LUNTargetResumer interface { Resume(ctx context.Context, targetID string) error } @@ -302,8 +306,32 @@ func (w *Worker) execute(ctx context.Context, p MovementPlan) error { } } - if err := os.Remove(srcPath); err != nil && !os.IsNotExist(err) { + // A new movement must not begin until old-source cleanup is durable. Never + // report placed after a failed unlink, directory barrier, or kernel ack. + cleanupFailed := func(err error) error { w.logTransition(ctx, p, string(StateSwitched), string(StateCleanupInProgress), err.Error()) + w.persistObject(ctx, oid, p, StateCleanupInProgress) + return err + } + if err := os.Remove(srcPath); err != nil && !os.IsNotExist(err) { + return cleanupFailed(fmt.Errorf("cleanup source: %w", err)) + } + dir, err := os.Open(filepath.Dir(srcPath)) + if err != nil { + return cleanupFailed(fmt.Errorf("open source parent: %w", err)) + } + err = dir.Sync() + closeErr := dir.Close() + if err != nil { + return cleanupFailed(fmt.Errorf("sync source cleanup: %w", err)) + } + if closeErr != nil { + return cleanupFailed(fmt.Errorf("close source parent: %w", closeErr)) + } + if client, ok := w.client.(movementCompletionClient); ok { + if err := client.MoveComplete(p.PoolUUID, p.ObjectID, p.TransactionSeq); err != nil { + return cleanupFailed(fmt.Errorf("move_complete: %w", err)) + } } w.logTransition(ctx, p, string(StateSwitched), string(StateCleanupComplete), "") w.finalize(ctx, oid, p) diff --git a/controlplane/worker_completion_test.go b/controlplane/worker_completion_test.go new file mode 100644 index 0000000..2b1d6d1 --- /dev/null +++ b/controlplane/worker_completion_test.go @@ -0,0 +1,95 @@ +package controlplane + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/google/uuid" +) + +type completingMovementClient struct { + fakeMovementClient + cutover func() + complete func() error +} + +func (c *completingMovementClient) MoveCutover(uuid.UUID, [OIDLen]byte, uint64) error { + if c.cutover != nil { + c.cutover() + } + return nil +} +func (c *completingMovementClient) MoveComplete(uuid.UUID, [OIDLen]byte, uint64) error { + return c.complete() +} + +func TestWorkerAcknowledgesOnlyDurableCleanup(t *testing.T) { + for _, name := range []string{"success", "ack failure", "unlink failure"} { + t.Run(name, func(t *testing.T) { + db := testDB(t) + ns, srcID, dstID := seedPool(t, db) + srcDir, dstDir := t.TempDir(), t.TempDir() + src := filepath.Join(srcDir, "chunk") + dst := filepath.Join(dstDir, "chunk") + payload := []byte("a completed PBS chunk must remain readable") + if err := os.WriteFile(src, payload, 0600); err != nil { + t.Fatal(err) + } + var oid [OIDLen]byte + oid[0] = 0xb7 + called := false + c := &completingMovementClient{fakeMovementClient: fakeMovementClient{inspectResult: &InspectResult{CurrentTier: 0, RelPath: "chunk", PinState: PinNone}}} + c.complete = func() error { + called = true + if _, err := os.Stat(src); !os.IsNotExist(err) { + t.Fatalf("completion before source cleanup: %v", err) + } + if data, err := os.ReadFile(dst); err != nil || string(data) != string(payload) { + t.Fatalf("destination lost: %v", err) + } + _, _, state, _, _ := readObjectState(t, db, oid) + if state != string(StateSwitched) { + t.Fatalf("state before acknowledgement=%q", state) + } + if name == "ack failure" { + return errors.New("kernel rejected completion") + } + return nil + } + if name == "unlink failure" { + c.cutover = func() { + if err := os.Remove(src); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(src, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(src, "held"), nil, 0600); err != nil { + t.Fatal(err) + } + } + } + plan := MovementPlan{PoolUUID: uuid.New(), ObjectID: oid, NamespaceID: ns, SourceTierID: srcID, SourceLowerDir: srcDir, DestTierID: dstID, DestTierRank: 1, DestLowerDir: dstDir, RelPath: "chunk", TransactionSeq: 77} + err := NewWorker(db, c).Execute(context.Background(), plan) + cur, intended, state, _, _ := readObjectState(t, db, oid) + if name == "success" { + if err != nil || !called || state != string(StatePlaced) || cur.String != dstID || intended.Valid { + t.Fatalf("success: err=%v called=%v state=%q current=%q intended=%v", err, called, state, cur.String, intended) + } + } else { + if err == nil || state != string(StateCleanupInProgress) || cur.String != srcID || intended.String != dstID { + t.Fatalf("failed cleanup reported complete: err=%v state=%q current=%q intended=%v", err, state, cur.String, intended) + } + if called != (name == "ack failure") { + t.Fatalf("completion called=%v after %s", called, name) + } + if data, err := os.ReadFile(dst); err != nil || string(data) != string(payload) { + t.Fatalf("cleanup failure lost destination: %v", err) + } + } + }) + } +} diff --git a/src/smoothfs/movement.c b/src/smoothfs/movement.c index 94a8a45..40fbb44 100644 --- a/src/smoothfs/movement.c +++ b/src/smoothfs/movement.c @@ -747,6 +747,47 @@ void smoothfs_clear_pool_mapping_quiesce(struct smoothfs_sb_info *sbi) up_read(&sbi->inode_lock); } +/* Only userspace knows when the old source has been removed durably. Keep + * SWITCHED until that acknowledgement; resetting during cutover would let a + * second move race the first mover's source cleanup. The transaction sequence + * also makes retries idempotent without accepting a stale mover's completion. + */ +int smoothfs_movement_complete(struct smoothfs_sb_info *sbi, + const u8 oid[SMOOTHFS_OID_LEN], + u64 transaction_seq) +{ + struct smoothfs_inode_info *si; + struct inode *inode; + int err = 0; + + si = smoothfs_lookup_oid_resolve(sbi, oid); + if (!si) + return -ENOENT; + inode = &si->vfs_inode; + inode_lock(inode); + if (si->transaction_seq != transaction_seq) { + err = -ESTALE; + goto out; + } + if (si->movement_state == SMOOTHFS_MS_PLACED) + goto out; + if (si->movement_state != SMOOTHFS_MS_SWITCHED) { + err = -EBUSY; + goto out; + } + err = + smoothfs_placement_record(sbi, oid, SMOOTHFS_MS_PLACED, + si->current_tier, si->current_tier, true); + if (err) + goto out; + si->movement_state = SMOOTHFS_MS_PLACED; + si->intended_tier = si->current_tier; + smoothfs_netlink_emit_move_state(sbi, oid, SMOOTHFS_MS_PLACED, + transaction_seq); +out: + inode_unlock(inode); + return err; +} int smoothfs_movement_abort(struct smoothfs_sb_info *sbi, const u8 oid[SMOOTHFS_OID_LEN], u64 transaction_seq, const char *reason) diff --git a/src/smoothfs/netlink.c b/src/smoothfs/netlink.c index b8b9f9c..b8df939 100644 --- a/src/smoothfs/netlink.c +++ b/src/smoothfs/netlink.c @@ -241,6 +241,23 @@ static int doit_move_cutover(struct sk_buff *skb, struct genl_info *info) expected_write_seq, check_write_seq); } +static int doit_move_complete(struct sk_buff *skb, struct genl_info *info) +{ + struct smoothfs_sb_info *sbi; + const u8 *oid; + u64 seq; + + (void)skb; + sbi = resolve_pool(info); + if (IS_ERR(sbi)) + return PTR_ERR(sbi); + if (!info->attrs[SMOOTHFS_ATTR_OBJECT_ID] || + !info->attrs[SMOOTHFS_ATTR_TRANSACTION_SEQ]) + return -EINVAL; + oid = nla_data(info->attrs[SMOOTHFS_ATTR_OBJECT_ID]); + seq = nla_get_u64(info->attrs[SMOOTHFS_ATTR_TRANSACTION_SEQ]); + return smoothfs_movement_complete(sbi, oid, seq); +} static int doit_inspect(struct sk_buff *skb, struct genl_info *info) { struct smoothfs_sb_info *sbi; @@ -382,6 +399,8 @@ static const struct genl_multicast_group smoothfs_genl_mcgrps[] = { }; static const struct genl_small_ops smoothfs_genl_ops[] = { + { .cmd = SMOOTHFS_CMD_MOVE_COMPLETE, .flags = GENL_ADMIN_PERM, + .doit = doit_move_complete }, { .cmd = SMOOTHFS_CMD_REGISTER_POOL, .flags = GENL_ADMIN_PERM, .doit = doit_register_pool }, { .cmd = SMOOTHFS_CMD_POLICY_PUSH, .flags = GENL_ADMIN_PERM, diff --git a/src/smoothfs/smoothfs.h b/src/smoothfs/smoothfs.h index 09dad5f..f6b1b22 100644 --- a/src/smoothfs/smoothfs.h +++ b/src/smoothfs/smoothfs.h @@ -814,6 +814,9 @@ int smoothfs_movement_cutover(struct smoothfs_sb_info *sbi, u64 transaction_seq, u64 expected_write_seq, bool check_write_seq); +int smoothfs_movement_complete(struct smoothfs_sb_info *sbi, + const u8 oid[SMOOTHFS_OID_LEN], + u64 transaction_seq); int smoothfs_movement_abort(struct smoothfs_sb_info *sbi, const u8 oid[SMOOTHFS_OID_LEN], u64 transaction_seq, const char *reason); diff --git a/src/smoothfs/test/movement_roundtrip.sh b/src/smoothfs/test/movement_roundtrip.sh new file mode 100755 index 0000000..42db1fe --- /dev/null +++ b/src/smoothfs/test/movement_roundtrip.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# Successful cleanup must permit another movement without inode eviction/reboot. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT=${SMOOTHFS_TEST_GO_ROOT:-$(cd "$SCRIPT_DIR/../../.." && pwd)} +. "$SCRIPT_DIR/tier_spill_lib.sh" +SPILL_ROOT=${SMOOTHFS_ROUNDTRIP_ROOT:-/tmp/smoothfs-movement-roundtrip} +SPILL_UUID=00000000-0000-0000-0000-00000000f51e +trap spill_cleanup EXIT +spill_setup_pool roundtrip "$SPILL_UUID" +cat > "$SPILL_ROOT/probe.go" <<'GO' +package main +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "path/filepath" + smoothfs "github.com/RakuenSoftware/smoothfs" + "github.com/google/uuid" + "golang.org/x/sys/unix" +) +func must(e error){if e!=nil{panic(e)}} +func main(){ + root:=os.Args[1];pool:=uuid.MustParse(os.Args[2]);upper:=filepath.Join(root,"server","chunk") + payload:=bytes.Repeat([]byte("PBS repeated movement sentinel\n"),8192) + must(os.WriteFile(upper,payload,0600)) + held,e:=os.Open(upper);must(e);defer held.Close() + raw:=make([]byte,smoothfs.OIDLen);_,e=unix.Getxattr(upper,"trusted.smoothfs.oid",raw);must(e) + var oid [smoothfs.OIDLen]byte;copy(oid[:],raw) + c,e:=smoothfs.Open();must(e);defer c.Close() + tiers:=[]string{"fast","slow"} + for round:=0;round<4;round++{ + from,to:=round%2,1-round%2;seq:=uint64(500+round) + before,e:=c.Inspect(pool,oid);must(e) + if before.CurrentTier!=uint8(from)||before.MovementState!=smoothfs.StatePlaced{panic("object was not ready for the next movement")} + must(c.MovePlan(pool,oid,uint8(to),seq)) + if e=c.MoveComplete(pool,oid,seq);!errors.Is(e,unix.EBUSY){panic(fmt.Sprintf("premature completion: %v",e))} + src:=filepath.Join(root,tiers[from],"chunk");dst:=filepath.Join(root,tiers[to],"chunk") + in,e:=os.Open(src);must(e);out,e:=os.Create(dst);must(e) + _,e=io.Copy(out,in);must(e);must(out.Sync());must(out.Close());must(in.Close()) + if before.HasWriteSeq{must(c.MoveCutoverVerifyWriteSeq(pool,oid,seq,before.WriteSeq))}else{must(c.MoveCutover(pool,oid,seq))} + if e=c.MovePlan(pool,oid,uint8(from),seq+100);!errors.Is(e,unix.EBUSY){panic(fmt.Sprintf("new movement before cleanup ack: %v",e))} + if e=c.MoveComplete(pool,oid,seq+1);!errors.Is(e,unix.ESTALE){panic(fmt.Sprintf("stale completion: %v",e))} + must(os.Remove(src));dir,e:=os.Open(filepath.Dir(src));must(e);must(dir.Sync());must(dir.Close()) + must(c.MoveComplete(pool,oid,seq));must(c.MoveComplete(pool,oid,seq)) + after,e:=c.Inspect(pool,oid);must(e) + if after.CurrentTier!=uint8(to)||after.MovementState!=smoothfs.StatePlaced{panic("cleanup did not restore placed state")} + _,e=held.Seek(0,0);must(e);actual,e:=io.ReadAll(held);must(e) + if !bytes.Equal(actual,payload){panic("held file changed after roundtrip")} + } + fmt.Println("movement_roundtrip: PASS (four moves, held descriptor, premature/stale/idempotent acknowledgements)") +} +GO +go -C "$REPO_ROOT" run "$SPILL_ROOT/probe.go" "$SPILL_ROOT" "$SPILL_UUID" diff --git a/src/smoothfs/test/run_runtime_harnesses.sh b/src/smoothfs/test/run_runtime_harnesses.sh index 3ae718b..5e41946 100755 --- a/src/smoothfs/test/run_runtime_harnesses.sh +++ b/src/smoothfs/test/run_runtime_harnesses.sh @@ -50,6 +50,7 @@ core_tests=( movement_open_fd_cutover.sh movement_getattr_cutover.sh movement_inode_reuse.sh + movement_roundtrip.sh write_staging_direct_cutover.sh odirect.sh ) diff --git a/src/smoothfs/uapi_smoothfs.h b/src/smoothfs/uapi_smoothfs.h index 640cc37..3f1b520 100644 --- a/src/smoothfs/uapi_smoothfs.h +++ b/src/smoothfs/uapi_smoothfs.h @@ -53,6 +53,7 @@ enum { SMOOTHFS_CMD_MOVE_CUTOVER = 13, SMOOTHFS_CMD_REVOKE_MAPPINGS = 14, SMOOTHFS_EVENT_SPILL = 15, + SMOOTHFS_CMD_MOVE_COMPLETE = 16, /* userspace cleanup durable */ __SMOOTHFS_CMD_MAX, }; #define SMOOTHFS_CMD_MAX (__SMOOTHFS_CMD_MAX - 1) diff --git a/uapi.go b/uapi.go index fa380c7..e1ed1c2 100644 --- a/uapi.go +++ b/uapi.go @@ -31,6 +31,7 @@ const ( CmdMoveCutover uint8 = 13 CmdRevokeMappings uint8 = 14 EventSpill uint8 = 15 + CmdMoveComplete uint8 = 16 ) // Generic netlink attributes. Keep in sync with src/smoothfs/uapi_smoothfs.h. From 13b3b03983c7d42a0b04cafe0de70b42d70cadd6 Mon Sep 17 00:00:00 2001 From: virant Date: Sat, 31 Oct 2026 19:28:08 +0100 Subject: [PATCH 4/7] fix: normalize mkdir error cleanup across supported kernels Linux 6.18 leaves the lower parent locked on mkdir error; 6.19 and newer consume the dentry and unlock it. Normalize this in the compatibility helper and remove the materialization path's double unlock. Add an ENOSPC regression for both parent materialization and direct mkdir followed by lookups and write recovery. The old module hung the regression; the fixed module passes on Linux 7.2.11, with both 6.18 and 7.2 builds checked. --- src/smoothfs/compat.h | 19 +++++-- src/smoothfs/inode.c | 14 ++--- src/smoothfs/test/mkdir_enospc.sh | 66 ++++++++++++++++++++++ src/smoothfs/test/run_runtime_harnesses.sh | 1 + 4 files changed, 84 insertions(+), 16 deletions(-) create mode 100755 src/smoothfs/test/mkdir_enospc.sh diff --git a/src/smoothfs/compat.h b/src/smoothfs/compat.h index bc750b3..2403ea8 100644 --- a/src/smoothfs/compat.h +++ b/src/smoothfs/compat.h @@ -136,11 +136,10 @@ smoothfs_compat_create(struct mnt_idmap *idmap, struct inode *dir, * * Helper does the now-canonical "may-replace-dentry" dance: returns * the dentry the caller should use (either input or replacement), - * or ERR_PTR on failure. NOTE: vfs_mkdir() itself dput()s the - * passed-in dentry whenever it returns a *different* pointer - * (a replacement OR an ERR_PTR), and additionally unlocks the parent - * on error. Callers must therefore NOT dput the original on - * replacement/error, and must not unlock the parent again on error. + * or ERR_PTR on failure. vfs_mkdir() consumes the original on replacement + * or error. Starting with 6.19 it also unlocks the parent on error; this + * helper supplies that unlock on 6.18 to keep one caller contract. Callers + * must not dput the original on replacement/error, or unlock on error. */ static inline struct dentry * smoothfs_compat_mkdir(struct mnt_idmap *idmap, struct inode *dir, @@ -153,8 +152,16 @@ smoothfs_compat_mkdir(struct mnt_idmap *idmap, struct inode *dir, struct dentry *new_d = vfs_mkdir(idmap, dir, dentry, mode); #endif - if (IS_ERR(new_d)) + /* Normalize mkdir's error contract: the child is consumed and the parent + * unlocked. v6.18 consumes only the child; v6.19+ also calls end_creating. + * Success always leaves the caller's parent lock held. + */ + if (IS_ERR(new_d)) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 19, 0) + inode_unlock(dir); +#endif return new_d; + } if (new_d && new_d != dentry) return new_d; return dentry; diff --git a/src/smoothfs/inode.c b/src/smoothfs/inode.c index 5dbea8c..8b2636b 100644 --- a/src/smoothfs/inode.c +++ b/src/smoothfs/inode.c @@ -501,16 +501,10 @@ static int smoothfs_materialize_parent_on_tier(struct mnt_idmap *idmap, revert_creds(old_cred); if (IS_ERR(new_child)) { err = PTR_ERR(new_child); - /* - * vfs_mkdir() (6.15+) already dput()s the - * passed-in dentry when it returns an error - * pointer — a second dput here corrupts the - * dcache (dput() WARN at fs/dcache.c). It does - * NOT unlock the parent: that stays with us - * (see do_mkdirat, where end_creating_path() - * unlocks even on vfs_mkdir error). + /* The compat helper consumed child and released the + * parent lock. Unlocking again underflows i_rwsem on + * v6.19+, hanging later lookups after ENOSPC. */ - inode_unlock(d_inode(cur.dentry)); goto out_err; } if (new_child != child) { @@ -1441,7 +1435,7 @@ static struct dentry *smoothfs_mkdir_inner(struct mnt_idmap *idmap, if (IS_ERR(new_lower)) { err = PTR_ERR(new_lower); /* - * vfs_mkdir() already dput()'d lower and + * The compat helper already consumed lower and * unlocked the parent on error; do not repeat. */ path_put(&parent_path); diff --git a/src/smoothfs/test/mkdir_enospc.sh b/src/smoothfs/test/mkdir_enospc.sh new file mode 100755 index 0000000..de87a88 --- /dev/null +++ b/src/smoothfs/test/mkdir_enospc.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# mkdir failure must release the lower parent lock on every supported kernel. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +. "$SCRIPT_DIR/tier_spill_lib.sh" +SPILL_ROOT=${SMOOTHFS_MKDIR_ENOSPC_ROOT:-/tmp/smoothfs-mkdir-enospc} +SPILL_UUID=00000000-0000-0000-0000-00000000f51f +trap spill_cleanup EXIT +spill_setup_pool mkdirenospc "$SPILL_UUID" +python3 - "$SPILL_ROOT" <<'PY' +import errno,os,signal,subprocess,sys +from pathlib import Path +root=Path(sys.argv[1]);server=root/'server' +(server/'fast-parent').mkdir() +for tier in ('fast','slow'): + base=root/tier + with (base/'fill').open('wb',buffering=0) as f: + st=os.statvfs(base) + try:os.posix_fallocate(f.fileno(),0,max(0,st.f_bavail*st.f_frsize-2*1024*1024)) + except OSError as e: + if e.errno!=errno.ENOSPC:raise + f.seek(0,os.SEEK_END) + while True: + try:f.write(bytes(65536)) + except OSError as e: + if e.errno!=errno.ENOSPC:raise + break + # Exhaust remaining inode space too, so the lower mkdir itself must fail. + for n in range(8192): + try:(base/('padding-'+str(n))).mkdir() + except OSError as e: + if e.errno!=errno.ENOSPC:raise + break + else:raise RuntimeError('lower mkdir ENOSPC was not exercised') + print('LOWER_FULL',tier,flush=True) +probe=r''' +import errno,os,sys +from pathlib import Path +p=Path(sys.argv[1]) +for target,missing in [(p/'fast-parent'/'must-fail',p/'missing-after-materialization'),(p/'must-also-fail',p/'missing-after-mkdir')]: + try:target.mkdir() + except OSError as e: + if e.errno!=errno.ENOSPC:raise + else:raise RuntimeError('mkdir unexpectedly succeeded on full tiers') + print('MKDIR_ENOSPC_RETURNED',target.name,flush=True) + try:os.stat(missing) + except FileNotFoundError:pass + else:raise RuntimeError('missing path unexpectedly exists') + print('PARENT_LOOKUP_RESPONSIVE',flush=True) +''' +child=subprocess.Popen([sys.executable,'-c',probe,str(server)],start_new_session=True) +try:code=child.wait(timeout=10) +except subprocess.TimeoutExpired: + # A broken module can leave the probe uninterruptible. Report the exact PID; + # the disposable test guest must be reset before treating cleanup as complete. + os.kill(child.pid,signal.SIGKILL) + print('mkdir_enospc: FAIL (lower parent lock leaked; probe PID '+str(child.pid)+')',flush=True) + os._exit(1) +if code:raise RuntimeError('mkdir probe failed') +for tier in ('fast','slow'):(root/tier/'fill').unlink() +(server/'fast-parent'/'recovered').mkdir() +path=server/'fast-parent'/'recovered'/'data' +with path.open('wb') as f:f.write(b'write after freeing space\n');f.flush();os.fsync(f.fileno()) +assert path.read_bytes()==b'write after freeing space\n' +print('mkdir_enospc: PASS (materialization and direct mkdir ENOSPC, responsive lookup, write recovery)') +PY diff --git a/src/smoothfs/test/run_runtime_harnesses.sh b/src/smoothfs/test/run_runtime_harnesses.sh index 5e41946..3aef8da 100755 --- a/src/smoothfs/test/run_runtime_harnesses.sh +++ b/src/smoothfs/test/run_runtime_harnesses.sh @@ -25,6 +25,7 @@ FAIL_FAST=${SMOOTHFS_RUNTIME_FAIL_FAST:-0} core_tests=( cache_mode.sh directory_fsync.sh + mkdir_enospc.sh netlink_receive_close.sh tier_spill_basic_create.sh tier_spill_nested_parent.sh From 65794ec7eba28487c52365aeef6a12639b9c3b79 Mon Sep 17 00:00:00 2001 From: virant Date: Sat, 31 Oct 2026 19:46:44 +0100 Subject: [PATCH 5/7] fix: enumerate every backing tier with the mount credentials Run merged-directory traversal with the same privileged backing credentials as lookup, preserving upper DAC checks. Caller credentials silently hid spill-tier snapshots below private backing parents from PBS, letting verification skip them and GC delete their live chunks. Propagate contributing-tier traversal/read errors instead of returning a partial listing. Cover UID 1000, bind mounts, upper DAC denial, and lookup-error propagation. --- src/smoothfs/dir.c | 32 +++++++++++-- .../test/readdir_backing_permissions.sh | 48 +++++++++++++++++++ src/smoothfs/test/run_runtime_harnesses.sh | 1 + 3 files changed, 78 insertions(+), 3 deletions(-) create mode 100755 src/smoothfs/test/readdir_backing_permissions.sh diff --git a/src/smoothfs/dir.c b/src/smoothfs/dir.c index a069332..346e879 100644 --- a/src/smoothfs/dir.c +++ b/src/smoothfs/dir.c @@ -157,7 +157,7 @@ static int smoothfs_collect_one_dir(struct smoothfs_dir_cache *cache, return collect.err; } -static struct smoothfs_dir_cache *smoothfs_build_dir_cache(struct file *file) +static struct smoothfs_dir_cache *smoothfs_build_dir_cache_inner(struct file *file) { struct inode *inode = file_inode(file); struct smoothfs_inode_info *si = SMOOTHFS_I(inode); @@ -198,15 +198,41 @@ static struct smoothfs_dir_cache *smoothfs_build_dir_cache(struct file *file) continue; } err = smoothfs_resolve_rel_path_on_tier(sbi, tier, rel_path, &tier_path); - if (err) + if (err == -ENOENT || err == -ENOTDIR) continue; + if (err) + goto fail; if (d_is_dir(tier_path.dentry)) - (void)smoothfs_collect_one_dir(cache, &tier_path, sbi->creator_cred); + err = smoothfs_collect_one_dir(cache, &tier_path, sbi->creator_cred); path_put(&tier_path); + if (err) + goto fail; } kfree(rel_path); return cache; +fail: + /* Partial enumeration can make a backup collector delete live chunks. + * Only a genuinely absent replica is safe to omit. + */ + kfree(rel_path); + smoothfs_dir_cache_free(cache); + return ERR_PTR(err); +} + +/* VFS already checked the caller against the presented upper directory. + * Backing traversal must use the mounter's credentials on every tier, just + * like lookup. Otherwise root-owned private backing parents make readdir + * silently omit spill-tier entries for the presented owner (e.g. PBS). + */ +static struct smoothfs_dir_cache *smoothfs_build_dir_cache(struct file *file) +{ + struct smoothfs_sb_info *sbi = SMOOTHFS_SB(file_inode(file)->i_sb); + const struct cred *old = override_creds(sbi->creator_cred); + struct smoothfs_dir_cache *cache = smoothfs_build_dir_cache_inner(file); + + revert_creds(old); + return cache; } static int smoothfs_opendir(struct inode *inode, struct file *file) diff --git a/src/smoothfs/test/readdir_backing_permissions.sh b/src/smoothfs/test/readdir_backing_permissions.sh new file mode 100755 index 0000000..4829b48 --- /dev/null +++ b/src/smoothfs/test/readdir_backing_permissions.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# The presented owner must see merged entries through restrictive backing dirs. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +. "$SCRIPT_DIR/tier_spill_lib.sh" +SPILL_ROOT=${SMOOTHFS_READDIR_PERMISSIONS_ROOT:-/tmp/smoothfs-readdir-permissions} +SPILL_UUID=00000000-0000-0000-0000-00000000f520 +trap spill_cleanup EXIT +spill_setup_pool readdirpermissions "$SPILL_UUID" +mkdir -p "$SPILL_ROOT/fast/private/merged" "$SPILL_ROOT/slow/private/merged" "$SPILL_ROOT/bind" +chmod 700 "$SPILL_ROOT/fast/private" "$SPILL_ROOT/slow/private" +printf 'fast tier data\n' > "$SPILL_ROOT/fast/private/merged/fast" +printf 'slow tier data\n' > "$SPILL_ROOT/slow/private/merged/slow" +mount --bind "$SPILL_ROOT/server/private/merged" "$SPILL_ROOT/bind" +trap 'umount "$SPILL_ROOT/bind"; spill_cleanup' EXIT +python3 - "$SPILL_ROOT" <<'PY' +from pathlib import Path +import os,subprocess,sys +root=Path(sys.argv[1]);expected={'fast','slow'} +probe=r''' +import os,sys +for path in sys.argv[1:]: + names={e.name for e in os.scandir(path)} + assert names=={'fast','slow'},f'UID {os.geteuid()} merged listing {names}' + for name in names: + with open(os.path.join(path,name)) as f:assert f.read()==name+' tier data\n' +print('PRESENTED_OWNER_MERGED_READDIR_OK',os.geteuid()) +''' +for path in (root/'server/private/merged',root/'bind'):assert set(os.listdir(path))==expected +subprocess.run([sys.executable,'-c',probe,str(root/'server/private/merged'),str(root/'bind')],user=1000,group=1000,extra_groups=[],check=True) +# Privilege applies to lower I/O only; upper directory DAC must still reject +# a different caller before the filesystem iterator runs. +code='import os,sys\ntry:os.listdir(sys.argv[1])\nexcept PermissionError:sys.exit(0)\nraise RuntimeError("upper DAC bypassed")' +subprocess.run([sys.executable,'-c',code,str(root/'server/private/merged')],user=1001,group=1001,extra_groups=[],check=True) +# A broken contributing path must fail enumeration instead of returning a +# partial listing that a backup collector could mistake for a complete tree. +slow=root/'slow/private/merged';saved=root/'slow/private/saved' +slow.rename(saved);slow.symlink_to('merged') +try: + try:os.listdir(root/'server/private/merged') + except OSError as e: + import errno + assert e.errno==errno.ELOOP,e + else:raise RuntimeError('partial listing hid a contributing-tier lookup error') +finally: + slow.unlink();saved.rename(slow) +print('readdir_backing_permissions: PASS (owner union, bind mount, upper DAC, contributing-error propagation)') +PY diff --git a/src/smoothfs/test/run_runtime_harnesses.sh b/src/smoothfs/test/run_runtime_harnesses.sh index 3aef8da..0bc1527 100755 --- a/src/smoothfs/test/run_runtime_harnesses.sh +++ b/src/smoothfs/test/run_runtime_harnesses.sh @@ -25,6 +25,7 @@ FAIL_FAST=${SMOOTHFS_RUNTIME_FAIL_FAST:-0} core_tests=( cache_mode.sh directory_fsync.sh + readdir_backing_permissions.sh mkdir_enospc.sh netlink_receive_close.sh tier_spill_basic_create.sh From bbf034674eeaa5e4e2cb18ed4a6b381c77d5289f Mon Sep 17 00:00:00 2001 From: virant Date: Sat, 31 Oct 2026 20:53:27 +0100 Subject: [PATCH 6/7] fix: reconcile interrupted movement with kernel placement Persist full-sync intent before the first kernel transition. Recover using live placement, durable cleanup and sequence-guarded completion or rollback; instantiate persisted names when the recovery index is cold after reboot. Keep uncertain or failed acknowledgements pending instead of reporting placed. --- client.go | 7 + controlplane/recovery.go | 12 +- controlplane/recovery_live.go | 308 ++++++++++++++++++++++++ controlplane/recovery_live_test.go | 202 ++++++++++++++++ controlplane/service.go | 2 +- controlplane/testdb_test.go | 1 + controlplane/worker.go | 59 ++++- controlplane/worker_intent_test.go | 102 ++++++++ src/smoothfs/movement.c | 49 ++++ src/smoothfs/netlink.c | 19 ++ src/smoothfs/smoothfs.h | 3 + src/smoothfs/test/movement_roundtrip.sh | 18 +- src/smoothfs/uapi_smoothfs.h | 1 + uapi.go | 1 + 14 files changed, 777 insertions(+), 7 deletions(-) create mode 100644 controlplane/recovery_live.go create mode 100644 controlplane/recovery_live_test.go create mode 100644 controlplane/worker_intent_test.go diff --git a/client.go b/client.go index 2756f3b..d4da38d 100644 --- a/client.go +++ b/client.go @@ -143,6 +143,13 @@ func (c *Client) MoveComplete(poolUUID uuid.UUID, oid [OIDLen]byte, seq uint64) return err } +// MoveRollback acknowledges durable removal of an interrupted pre-cutover copy. +// It cannot roll back a switched object. +func (c *Client) MoveRollback(poolUUID uuid.UUID, oid [OIDLen]byte, seq uint64) error { + _, err := c.send(CmdMoveRollback, cutoverAttrs(poolUUID, oid, seq, nil)) + return err +} + func cutoverAttrs(poolUUID uuid.UUID, oid [OIDLen]byte, seq uint64, writeSeq *uint64) []netlink.Attribute { attrs := []netlink.Attribute{ {Type: AttrPoolUUID, Data: poolUUID[:]}, diff --git a/controlplane/recovery.go b/controlplane/recovery.go index 2632349..806fda6 100644 --- a/controlplane/recovery.go +++ b/controlplane/recovery.go @@ -8,8 +8,16 @@ import ( ) // Recover scans smoothfs_objects for interrupted movement states and failed -// LUN resume rows that need destination placement reconciliation. -func Recover(ctx context.Context, db *sql.DB) error { +// LUN resume rows that need destination placement reconciliation. A running +// service must pass its kernel client; omitting it retains the historical +// database-only repair for offline callers. +func Recover(ctx context.Context, db *sql.DB, clients ...movementClient) error { + if len(clients) > 0 { + return recoverLive(ctx, db, clients[0]) + } + // Historical database-only reconciliation, retained for offline callers. + // A running service must supply its kernel client. + rows, err := db.QueryContext(ctx, ` SELECT object_id, current_tier_id, intended_tier_id, movement_state, transaction_seq, pin_state, namespace_id, rel_path diff --git a/controlplane/recovery_live.go b/controlplane/recovery_live.go new file mode 100644 index 0000000..7c8094d --- /dev/null +++ b/controlplane/recovery_live.go @@ -0,0 +1,308 @@ +package controlplane + +import ( + "bytes" + "context" + "database/sql" + "encoding/hex" + "errors" + "fmt" + "log" + "os" + "path/filepath" + "strings" + "syscall" + + "golang.org/x/sys/unix" + + "github.com/google/uuid" +) + +type movementRollbackClient interface { + MoveRollback(uuid.UUID, [OIDLen]byte, uint64) error +} + +type interruptedObject struct { + oid, namespace, source, destination, state, pin, rel string + seq uint64 +} + +// The database transition can lag the kernel cutover or completion ack. Never +// choose which copy to delete, or report PLACED, using the SQL state alone. +func recoverLive(ctx context.Context, db *sql.DB, client movementClient) error { + rows, err := db.QueryContext(ctx, `SELECT object_id, namespace_id, current_tier_id, + COALESCE(intended_tier_id,''), movement_state, pin_state, rel_path, transaction_seq + FROM smoothfs_objects WHERE movement_state NOT IN ('placed','cleanup_complete','stale')`) + if err != nil { + return fmt.Errorf("scan in-flight movements: %w", err) + } + var pending []interruptedObject + for rows.Next() { + var p interruptedObject + if err = rows.Scan(&p.oid, &p.namespace, &p.source, &p.destination, &p.state, &p.pin, &p.rel, &p.seq); err != nil { + rows.Close() + return err + } + pending = append(pending, p) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + var failures []error + for _, p := range pending { + if err = recoverObject(ctx, db, client, p); err != nil { + failures = append(failures, fmt.Errorf("recover %s: %w", p.oid, err)) + } + } + return errors.Join(failures...) +} + +func recoverObject(ctx context.Context, db *sql.DB, client movementClient, p interruptedObject) error { + if client == nil { + return errors.New("kernel client unavailable") + } + raw, err := hex.DecodeString(p.oid) + if err != nil || len(raw) != OIDLen { + return errors.New("invalid object id") + } + var oid [OIDLen]byte + copy(oid[:], raw) + var backend, domain, exposed string + if err = db.QueryRowContext(ctx, `SELECT backend_ref,placement_domain,exposed_path FROM managed_namespaces WHERE id=? AND backend_kind='smoothfs'`, p.namespace).Scan(&backend, &domain, &exposed); err != nil { + return err + } + pool, err := uuid.Parse(backend) + if err != nil { + return fmt.Errorf("namespace has no pool UUID: %w", err) + } + ins, err := client.Inspect(pool, oid) + if errors.Is(err, syscall.ENOENT) { + // An asynchronous path index may not contain a recently moved object after + // reboot. Instantiate its persisted name, then confirm the same OID through + // the target pool's INSPECT before selecting or deleting any lower copy. + if err = warmRecoveryObject(exposed, p.rel, oid); err != nil { + return fmt.Errorf("instantiate interrupted object: %w", err) + } + log.Printf("smoothfs: resolved recovery path for object %s", p.oid) + ins, err = client.Inspect(pool, oid) + } + if err != nil { + return fmt.Errorf("inspect: %w", err) + } + if ins == nil { + return errors.New("inspect returned no object") + } + if ins.MovementState != StatePlaced && ins.TransactionSeq != p.seq { + return errors.New("kernel transaction differs from interrupted movement") + } + rel := ins.RelPath + if rel == "" { + rel = p.rel + } + if rel == "" || filepath.IsAbs(rel) || filepath.Clean(rel) != rel || rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { + return errors.New("invalid recovery relative path") + } + type tier struct { + id, lower string + rank uint8 + } + tiers := map[string]tier{} + var current tier + rows, err := db.QueryContext(ctx, `SELECT id,rank,backing_ref FROM tier_targets WHERE placement_domain=? AND backend_kind='smoothfs'`, domain) + if err != nil { + return err + } + for rows.Next() { + var t tier + if err = rows.Scan(&t.id, &t.rank, &t.lower); err != nil { + rows.Close() + return err + } + if !filepath.IsAbs(t.lower) { + rows.Close() + return errors.New("tier has no absolute backing path") + } + tiers[t.id] = t + if t.rank == ins.CurrentTier { + current = t + } + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + if current.id == "" { + return errors.New("kernel tier is not in namespace") + } + // A failed retry or older database-only recovery may have cleared intended. + // Its durable movement log still identifies the two paths for this sequence. + source, destination := p.source, p.destination + if destination == "" { + err = db.QueryRowContext(ctx, `SELECT source_tier,dest_tier FROM smoothfs_movement_log WHERE object_id=? AND transaction_seq=? ORDER BY id DESC LIMIT 1`, p.oid, p.seq).Scan(&source, &destination) + if err != nil && err != sql.ErrNoRows { + return err + } + } + if destination == "" || source == destination { + return errors.New("interrupted movement has no distinct recorded tiers") + } + if _, ok := tiers[source]; !ok { + return errors.New("recorded source tier is unavailable") + } + if _, ok := tiers[destination]; !ok { + return errors.New("recorded destination tier is unavailable") + } + if current.id != source && current.id != destination { + return errors.New("kernel placement is outside recorded movement") + } + authoritative := filepath.Join(current.lower, rel) + live, err := os.Lstat(authoritative) + if err != nil { + return fmt.Errorf("authoritative copy: %w", err) + } + if !live.Mode().IsRegular() { + return errors.New("authoritative copy is not regular") + } + other := destination + if current.id == destination { + other = source + } + otherPath := filepath.Join(tiers[other].lower, rel) + info, err := os.Lstat(otherPath) + if err == nil { + if !info.Mode().IsRegular() || os.SameFile(info, live) { + return errors.New("cleanup path is not a distinct regular copy") + } + } else if !os.IsNotExist(err) { + return err + } + // Validate the phase before touching either path. A switched kernel wins + // even if the process died before recording the SQL 'switched' transition. + switch ins.MovementState { + case StateSwitched: + if current.id != destination { + return errors.New("switched object is not on recorded destination") + } + if _, ok := client.(movementCompletionClient); !ok { + return errors.New("kernel completion acknowledgement unavailable") + } + case StatePlanAccepted, StateDestinationReserved, StateCopyInProgress, StateCopyComplete, StateCopyVerified, StateFailed: + if current.id != source { + return errors.New("pre-cutover object is not on recorded source") + } + if _, ok := client.(movementRollbackClient); !ok { + return errors.New("kernel rollback acknowledgement unavailable") + } + case StatePlaced: + // Mount replay or an ack preceding the final SQL commit already settled it. + default: + return fmt.Errorf("kernel movement is still %s", ins.MovementState) + } + if err == nil { + if err = os.Remove(otherPath); err != nil { + return fmt.Errorf("remove interrupted copy: %w", err) + } + } + // The other name may already be absent after a crash. Still issue its parent + // barrier before telling the kernel or database that cleanup is complete. + parent := filepath.Dir(otherPath) + for { + err = syncDir(parent) + if !os.IsNotExist(err) || parent == filepath.Clean(tiers[other].lower) { + break + } + parent = filepath.Dir(parent) + } + if err != nil { + return fmt.Errorf("sync interrupted cleanup: %w", err) + } + switch ins.MovementState { + case StateSwitched: + err = client.(movementCompletionClient).MoveComplete(pool, oid, ins.TransactionSeq) + case StatePlaced: + default: + err = client.(movementRollbackClient).MoveRollback(pool, oid, ins.TransactionSeq) + } + if err != nil { + return fmt.Errorf("kernel recovery acknowledgement: %w", err) + } + settled, err := client.Inspect(pool, oid) + if err != nil { + return err + } + if settled == nil || settled.MovementState != StatePlaced || settled.CurrentTier != ins.CurrentTier { + return errors.New("kernel recovery did not settle on authoritative tier") + } + if p.pin == string(PinLUN) { + if err = restoreLUNPin(ctx, db, p.namespace, current.id, rel); err != nil { + return err + } + } + state := string(StatePlaced) + if p.pin == string(PinLUN) && p.state == string(StateFailed) { + state = string(StateFailed) + } + _, err = db.ExecContext(ctx, `UPDATE smoothfs_objects SET current_tier_id=?,intended_tier_id=NULL, + movement_state=?,rel_path=?,last_movement_at=CASE WHEN ? THEN datetime('now') ELSE last_movement_at END, + last_committed_cutover_gen=MAX(last_committed_cutover_gen,?), + failure_reason=CASE WHEN ?='failed' THEN failure_reason ELSE 'recovered: kernel placement and cleanup confirmed' END, + updated_at=datetime('now') WHERE object_id=?`, current.id, state, rel, current.id == destination, settled.Generation, state, p.oid) + return err +} + +func warmRecoveryObject(exposed, rel string, oid [OIDLen]byte) error { + if !filepath.IsAbs(exposed) || rel == "" || filepath.IsAbs(rel) || filepath.Clean(rel) != rel || rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { + return errors.New("namespace has no usable exposed recovery path") + } + exposed, err := filepath.EvalSymlinks(exposed) + if err != nil { + return err + } + mounts, err := os.ReadFile("/proc/self/mountinfo") + if err != nil { + return err + } + mount := recoveryMountpoint(string(mounts), exposed) + if mount == "" { + return errors.New("namespace is not below a mounted smoothfs root") + } + path := filepath.Join(mount, rel) + if _, err = os.Lstat(path); err != nil { + return err + } + raw := make([]byte, OIDLen) + n, err := unix.Getxattr(path, "trusted.smoothfs.oid", raw) + if err != nil { + return err + } + if n != OIDLen || !bytes.Equal(raw, oid[:]) { + return errors.New("persisted recovery path belongs to another object") + } + return nil +} + +func recoveryMountpoint(mounts, exposed string) string { + decode := strings.NewReplacer(`\040`, " ", `\011`, "\t", `\012`, "\n", `\134`, `\`) + best := "" + for _, line := range strings.Split(mounts, "\n") { + before, after, ok := strings.Cut(line, " - ") + if !ok { + continue + } + fields, fs := strings.Fields(before), strings.Fields(after) + if len(fields) < 5 || len(fs) < 1 || fs[0] != "smoothfs" || fields[3] != "/" { + continue + } + path := decode.Replace(fields[4]) + if exposed != path && !strings.HasPrefix(exposed, path+string(os.PathSeparator)) { + continue + } + if len(path) > len(best) { + best = path + } + } + return best +} diff --git a/controlplane/recovery_live_test.go b/controlplane/recovery_live_test.go new file mode 100644 index 0000000..8ee26ef --- /dev/null +++ b/controlplane/recovery_live_test.go @@ -0,0 +1,202 @@ +package controlplane + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/google/uuid" +) + +type recoveringMovementClient struct { + fakeMovementClient + result InspectResult + ackErr error + beforeAck func() + completed, rolledBack int +} + +func (c *recoveringMovementClient) Inspect(uuid.UUID, [OIDLen]byte) (*InspectResult, error) { + r := c.result + return &r, nil +} +func (c *recoveringMovementClient) MoveComplete(uuid.UUID, [OIDLen]byte, uint64) error { + c.completed++ + return c.ack() +} +func (c *recoveringMovementClient) MoveRollback(uuid.UUID, [OIDLen]byte, uint64) error { + c.rolledBack++ + return c.ack() +} +func (c *recoveringMovementClient) ack() error { + if c.beforeAck != nil { + c.beforeAck() + } + if c.ackErr != nil { + return c.ackErr + } + c.result.MovementState = StatePlaced + return nil +} + +func TestRecoverLiveUsesKernelPlacementAndDurableCleanup(t *testing.T) { + for _, tc := range []struct { + name string + sql, kernel MovementState + rank uint8 + }{ + {"before-copy", StatePlanAccepted, StatePlanAccepted, 0}, + {"after-copy", StateCopyVerified, StatePlanAccepted, 0}, + {"cutover-before-sql", StateCopyVerified, StateSwitched, 1}, + {"cleanup-before-ack", StateSwitched, StateSwitched, 1}, + {"ack-before-sql", StateSwitched, StatePlaced, 1}, + {"mount-replay-forward", StateCopyVerified, StatePlaced, 1}, + {"mount-replay-rollback", StateSwitched, StatePlaced, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + db := testDB(t) + ns, fast, slow := seedPool(t, db) + fastDir, slowDir := t.TempDir(), t.TempDir() + pool := uuid.New() + if _, err := db.Exec(`UPDATE managed_namespaces SET backend_ref=? WHERE id=?`, pool.String(), ns); err != nil { + t.Fatal(err) + } + for id, path := range map[string]string{fast: fastDir, slow: slowDir} { + if _, err := db.Exec(`UPDATE tier_targets SET backing_ref=? WHERE id=?`, path, id); err != nil { + t.Fatal(err) + } + } + oid := [OIDLen]byte{0x42} + seedMidMove(t, db, ns, fast, slow, tc.sql, 11, oid) + for _, dir := range []string{fastDir, slowDir} { + if err := os.WriteFile(filepath.Join(dir, "chunk"), []byte("bytes"), 0o600); err != nil { + t.Fatal(err) + } + } + canonical, other, tier := fastDir, slowDir, fast + if tc.rank == 1 { + canonical, other, tier = slowDir, fastDir, slow + } + client := &recoveringMovementClient{result: InspectResult{MovementState: tc.kernel, CurrentTier: tc.rank, TransactionSeq: 11, RelPath: "chunk", Generation: 4}} + client.beforeAck = func() { + if _, err := os.Stat(filepath.Join(other, "chunk")); !os.IsNotExist(err) { + t.Fatal("ack preceded deletion") + } + _, _, state, _, _ := readObjectState(t, db, oid) + if state != string(tc.sql) { + t.Fatal("SQL reported recovery before kernel ack") + } + } + if err := Recover(context.Background(), db, client); err != nil { + t.Fatal(err) + } + cur, intended, state, gen, _ := readObjectState(t, db, oid) + if cur.String != tier || intended.Valid || state != "placed" || gen != 4 { + t.Fatalf("recovery cur=%s intended=%v state=%s generation=%d", cur.String, intended, state, gen) + } + if data, err := os.ReadFile(filepath.Join(canonical, "chunk")); err != nil || string(data) != "bytes" { + t.Fatal("authoritative copy lost", err) + } + if _, err := os.Stat(filepath.Join(other, "chunk")); !os.IsNotExist(err) { + t.Fatal("old copy remains") + } + if tc.kernel == StateSwitched && client.completed != 1 { + t.Fatal("missing completion ack") + } + if tc.kernel == StatePlanAccepted && client.rolledBack != 1 { + t.Fatal("missing rollback ack") + } + }) + } +} + +func TestRecoverLiveAckFailurePreservesPendingStateAndRetriesAbsentSource(t *testing.T) { + db := testDB(t) + ns, fast, slow := seedPool(t, db) + a, b := t.TempDir(), t.TempDir() + pool := uuid.New() + db.Exec(`UPDATE managed_namespaces SET backend_ref=? WHERE id=?`, pool.String(), ns) + db.Exec(`UPDATE tier_targets SET backing_ref=? WHERE id=?`, a, fast) + db.Exec(`UPDATE tier_targets SET backing_ref=? WHERE id=?`, b, slow) + oid := [OIDLen]byte{0x43} + seedMidMove(t, db, ns, fast, slow, StateSwitched, 12, oid) + os.WriteFile(filepath.Join(a, "chunk"), []byte("old"), 0o600) + os.WriteFile(filepath.Join(b, "chunk"), []byte("current"), 0o600) + client := &recoveringMovementClient{result: InspectResult{MovementState: StateSwitched, CurrentTier: 1, TransactionSeq: 12, RelPath: "chunk"}, ackErr: errors.New("ack failed")} + if err := Recover(context.Background(), db, client); err == nil { + t.Fatal("ack failure accepted") + } + _, _, state, _, _ := readObjectState(t, db, oid) + if state != "switched" { + t.Fatal("failed acknowledgement finalized SQL") + } + client.ackErr = nil + if err := Recover(context.Background(), db, client); err != nil { + t.Fatal(err) + } + _, _, state, _, _ = readObjectState(t, db, oid) + if state != "placed" { + t.Fatal(state) + } +} + +func TestRecoverLiveRejectsStaleOrAmbiguousCleanup(t *testing.T) { + for _, failure := range []string{"stale", "directory", "invalid-relative-path", "missing-authority", "missing-pool"} { + t.Run(failure, func(t *testing.T) { + db := testDB(t) + ns, fast, slow := seedPool(t, db) + a, b := t.TempDir(), t.TempDir() + db.Exec(`UPDATE managed_namespaces SET backend_ref=? WHERE id=?`, uuid.NewString(), ns) + db.Exec(`UPDATE tier_targets SET backing_ref=? WHERE id=?`, a, fast) + db.Exec(`UPDATE tier_targets SET backing_ref=? WHERE id=?`, b, slow) + oid := [OIDLen]byte{0x44} + seedMidMove(t, db, ns, fast, slow, StateSwitched, 12, oid) + os.WriteFile(filepath.Join(b, "chunk"), []byte("current"), 0o600) + if failure == "directory" { + os.Mkdir(filepath.Join(a, "chunk"), 0o700) + } else { + os.WriteFile(filepath.Join(a, "chunk"), []byte("old"), 0o600) + } + client := &recoveringMovementClient{result: InspectResult{MovementState: StateSwitched, CurrentTier: 1, TransactionSeq: 12, RelPath: "chunk"}} + switch failure { + case "stale": + client.result.TransactionSeq = 13 + case "invalid-relative-path": + client.result.RelPath = "../chunk" + case "missing-authority": + os.Remove(filepath.Join(b, "chunk")) + case "missing-pool": + db.Exec(`UPDATE managed_namespaces SET backend_ref='' WHERE id=?`, ns) + } + if err := Recover(context.Background(), db, client); err == nil { + t.Fatal("unsafe recovery accepted") + } + if _, err := os.Lstat(filepath.Join(a, "chunk")); err != nil { + t.Fatal("removed copy on failed recovery") + } + _, _, state, _, _ := readObjectState(t, db, oid) + if state != "switched" { + t.Fatal(state) + } + if client.completed != 0 || client.rolledBack != 0 { + t.Fatal("acknowledged rejected recovery") + } + }) + } +} + +func TestRecoveryMountpointFindsPoolRootForNamespace(t *testing.T) { + mounts := `10 1 0:20 / /mnt/pbs rw - smoothfs none rw,pool=media +11 1 0:21 / /mnt/pbs2 rw - smoothfs none rw,pool=other +12 1 0:22 /sub /bind-sub rw - smoothfs none rw +13 1 0:23 / /mnt/my\040pool rw - smoothfs none rw +14 1 8:1 / /ordinary rw - ext4 /dev/sda1 rw +` + for exposed, want := range map[string]string{"/mnt/pbs": "/mnt/pbs", "/mnt/pbs/nested/ns": "/mnt/pbs", "/mnt/pbs2/sub": "/mnt/pbs2", "/mnt/pbs-not-a-pool": "", "/bind-sub": "", "/ordinary": "", "/mnt/my pool/ns": "/mnt/my pool"} { + if got := recoveryMountpoint(mounts, exposed); got != want { + t.Errorf("%q: got %q want %q", exposed, got, want) + } + } +} diff --git a/controlplane/service.go b/controlplane/service.go index 312120c..793f343 100644 --- a/controlplane/service.go +++ b/controlplane/service.go @@ -150,7 +150,7 @@ func DiscoverPoolFromDB(ctx context.Context, db *sql.DB, ev *Event) (*Pool, erro } func (s *Service) Run(ctx context.Context) error { - if err := Recover(ctx, s.db); err != nil { + if err := Recover(ctx, s.db, s.client); err != nil { log.Printf("smoothfs: recovery failed: %v", err) } diff --git a/controlplane/testdb_test.go b/controlplane/testdb_test.go index 32cabfa..98e66bc 100644 --- a/controlplane/testdb_test.go +++ b/controlplane/testdb_test.go @@ -40,6 +40,7 @@ func testDB(t *testing.T) *sql.DB { placement_domain TEXT NOT NULL, backend_kind TEXT NOT NULL, backend_ref TEXT NOT NULL DEFAULT '', + exposed_path TEXT NOT NULL DEFAULT '', FOREIGN KEY(placement_domain) REFERENCES placement_domains(id) );`, `CREATE TABLE smoothfs_objects ( diff --git a/controlplane/worker.go b/controlplane/worker.go index 3838cbd..8ba7e54 100644 --- a/controlplane/worker.go +++ b/controlplane/worker.go @@ -199,11 +199,18 @@ func (w *Worker) execute(ctx context.Context, p MovementPlan) error { srcPath := filepath.Join(p.SourceLowerDir, p.RelPath) dstPath := filepath.Join(p.DestLowerDir, p.RelPath) + // Leave an existing interrupted kernel transaction and its SQL record intact. + if ins.MovementState != "" && ins.MovementState != StatePlaced { + return fmt.Errorf("kernel movement requires recovery: %s", ins.MovementState) + } + // Persist the intent before the first kernel transition. A process may exit + // inside MovePlan, or after cutover but before any later SQL transition. + if err := w.persistIntent(ctx, oid, p); err != nil { + return fmt.Errorf("persist movement intent: %w", err) + } if err := w.client.MovePlan(p.PoolUUID, p.ObjectID, p.DestTierRank, p.TransactionSeq); err != nil { return w.rollbackLUNBeforeSwitch(ctx, p, srcPath, oid, fmt.Errorf("move_plan: %w", err)) } - w.logTransition(ctx, p, "", string(StatePlanAccepted), "") - w.persistObject(ctx, oid, p, StatePlanAccepted) if err := os.MkdirAll(filepath.Dir(dstPath), 0o755); err != nil { return w.rollbackLUNBeforeSwitch(ctx, p, srcPath, oid, fmt.Errorf("mkdir dest parent: %w", err)) @@ -700,3 +707,51 @@ func fileSHA256(path string) ([32]byte, error) { copy(out[:], h.Sum(nil)) return out, nil } + +// An intent must survive a host crash, including on WAL connections configured +// with NORMAL synchronous mode. Use a dedicated connection for this barrier. +func (w *Worker) persistIntent(ctx context.Context, oid string, p MovementPlan) error { + conn, err := w.db.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + var synchronous int + if err = conn.QueryRowContext(ctx, "PRAGMA synchronous").Scan(&synchronous); err != nil { + return err + } + if _, err = conn.ExecContext(ctx, "PRAGMA synchronous=FULL"); err != nil { + return err + } + defer conn.ExecContext(context.Background(), fmt.Sprintf("PRAGMA synchronous=%d", synchronous)) + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + result, err := tx.ExecContext(ctx, `INSERT INTO smoothfs_objects + (object_id,namespace_id,current_tier_id,intended_tier_id,movement_state,transaction_seq,rel_path,updated_at) + VALUES(?,?,?,?,'plan_accepted',?,?,datetime('now')) + ON CONFLICT(object_id) DO UPDATE SET intended_tier_id=excluded.intended_tier_id, + movement_state=excluded.movement_state,transaction_seq=excluded.transaction_seq, + rel_path=excluded.rel_path,updated_at=excluded.updated_at + WHERE smoothfs_objects.movement_state IN ('placed','cleanup_complete','failed')`, + oid, p.NamespaceID, p.SourceTierID, p.DestTierID, p.TransactionSeq, p.RelPath) + if err != nil { + return err + } + n, err := result.RowsAffected() + if err != nil { + return err + } + if n != 1 { + return errors.New("object already has an interrupted movement") + } + _, err = tx.ExecContext(ctx, `INSERT INTO smoothfs_movement_log + (object_id,transaction_seq,from_state,to_state,source_tier,dest_tier,payload_json) + VALUES(?,?,NULL,'plan_accepted',?,?,'{"reason":"intent persisted before kernel plan"}')`, oid, p.TransactionSeq, p.SourceTierID, p.DestTierID) + if err != nil { + return err + } + return tx.Commit() +} diff --git a/controlplane/worker_intent_test.go b/controlplane/worker_intent_test.go new file mode 100644 index 0000000..009d5bd --- /dev/null +++ b/controlplane/worker_intent_test.go @@ -0,0 +1,102 @@ +package controlplane + +import ( + "context" + "encoding/hex" + "os" + "path/filepath" + "testing" + + "github.com/google/uuid" +) + +type intentMovementClient struct { + fakeMovementClient + beforePlan func() +} + +func (c *intentMovementClient) MovePlan(p uuid.UUID, o [OIDLen]byte, d uint8, s uint64) error { + if c.beforePlan != nil { + c.beforePlan() + } + return c.fakeMovementClient.MovePlan(p, o, d, s) +} +func TestWorkerPersistsFullSyncIntentBeforeKernelPlan(t *testing.T) { + db := testDB(t) + db.SetMaxOpenConns(1) + ns, fast, slow := seedPool(t, db) + if _, err := db.Exec(`PRAGMA synchronous=NORMAL`); err != nil { + t.Fatal(err) + } + a, b := t.TempDir(), t.TempDir() + if err := os.WriteFile(filepath.Join(a, "chunk"), []byte("bytes"), 0o600); err != nil { + t.Fatal(err) + } + oid := [OIDLen]byte{0x51} + c := &intentMovementClient{fakeMovementClient: fakeMovementClient{inspectResult: &InspectResult{MovementState: StatePlaced, RelPath: "chunk"}}} + c.beforePlan = func() { + var state, rel string + var seq int64 + if err := db.QueryRow(`SELECT movement_state,rel_path,transaction_seq FROM smoothfs_objects WHERE object_id=?`, hex.EncodeToString(oid[:])).Scan(&state, &rel, &seq); err != nil { + t.Fatal(err) + } + if state != "plan_accepted" || rel != "chunk" || seq != 17 { + t.Fatalf("missing prior intent: %s %s %d", state, rel, seq) + } + var n int + if err := db.QueryRow(`SELECT count(*) FROM smoothfs_movement_log WHERE object_id=? AND transaction_seq=17`, hex.EncodeToString(oid[:])).Scan(&n); err != nil || n != 1 { + t.Fatal("missing prior log", err, n) + } + } + p := MovementPlan{PoolUUID: uuid.New(), ObjectID: oid, NamespaceID: ns, SourceTierID: fast, SourceLowerDir: a, DestTierID: slow, DestTierRank: 1, DestLowerDir: b, RelPath: "chunk", TransactionSeq: 17} + if err := NewWorker(db, c).Execute(context.Background(), p); err != nil { + t.Fatal(err) + } + var sync int + if err := db.QueryRow(`PRAGMA synchronous`).Scan(&sync); err != nil || sync != 1 { + t.Fatal("connection synchronous setting not restored", err, sync) + } +} + +func TestWorkerDoesNotPlanWithoutCommittedIntent(t *testing.T) { + db := testDB(t) + ns, fast, slow := seedPool(t, db) + if _, err := db.Exec(`CREATE TRIGGER reject_intent BEFORE INSERT ON smoothfs_movement_log BEGIN SELECT RAISE(ABORT,'simulated database write failure'); END`); err != nil { + t.Fatal(err) + } + a, b := t.TempDir(), t.TempDir() + os.WriteFile(filepath.Join(a, "chunk"), []byte("bytes"), 0o600) + c := &fakeMovementClient{inspectResult: &InspectResult{MovementState: StatePlaced, RelPath: "chunk"}} + p := MovementPlan{PoolUUID: uuid.New(), ObjectID: [OIDLen]byte{0x52}, NamespaceID: ns, SourceTierID: fast, SourceLowerDir: a, DestTierID: slow, DestTierRank: 1, DestLowerDir: b, RelPath: "chunk", TransactionSeq: 18} + if err := NewWorker(db, c).Execute(context.Background(), p); err == nil || c.movePlanned { + t.Fatal("kernel plan issued without durable intent", err) + } + var n int + if err := db.QueryRow(`SELECT count(*) FROM smoothfs_objects`).Scan(&n); err != nil || n != 0 { + t.Fatal("intent transaction was not rolled back", err, n) + } +} + +func TestWorkerPreservesInterruptedTransaction(t *testing.T) { + for _, state := range []MovementState{StatePlaced, StateSwitched} { + t.Run(string(state), func(t *testing.T) { + db := testDB(t) + ns, fast, slow := seedPool(t, db) + oid := [OIDLen]byte{0x53} + seedMidMove(t, db, ns, fast, slow, StateSwitched, 19, oid) + c := &fakeMovementClient{inspectResult: &InspectResult{MovementState: state, RelPath: "chunk"}} + p := MovementPlan{PoolUUID: uuid.New(), ObjectID: oid, NamespaceID: ns, SourceTierID: fast, SourceLowerDir: t.TempDir(), DestTierID: slow, DestTierRank: 1, DestLowerDir: t.TempDir(), RelPath: "chunk", TransactionSeq: 20} + if err := NewWorker(db, c).Execute(context.Background(), p); err == nil || c.movePlanned { + t.Fatal("overwrote pending movement", err) + } + _, _, got, _, _ := readObjectState(t, db, oid) + if got != "switched" { + t.Fatal(got) + } + var seq uint64 + if err := db.QueryRow(`SELECT transaction_seq FROM smoothfs_objects`).Scan(&seq); err != nil || seq != 19 { + t.Fatal("old transaction lost", err, seq) + } + }) + } +} diff --git a/src/smoothfs/movement.c b/src/smoothfs/movement.c index 40fbb44..910102a 100644 --- a/src/smoothfs/movement.c +++ b/src/smoothfs/movement.c @@ -788,6 +788,55 @@ int smoothfs_movement_complete(struct smoothfs_sb_info *sbi, inode_unlock(inode); return err; } +/* Recovery may discard a partial destination only before cutover. The + * sequence guard rejects an old recovery task after a newer movement began. + * Unlike the informational abort transition, this acknowledgement durably + * returns the authoritative source to PLACED so movement can resume. + */ +int smoothfs_movement_rollback(struct smoothfs_sb_info *sbi, + const u8 oid[SMOOTHFS_OID_LEN], + u64 transaction_seq) +{ + struct smoothfs_inode_info *si = smoothfs_lookup_oid_resolve(sbi, oid); + struct inode *inode; + int err = 0; + + if (!si) + return -ENOENT; + inode = &si->vfs_inode; + inode_lock(inode); + if (si->transaction_seq != transaction_seq) { + err = -ESTALE; + goto out; + } + switch (si->movement_state) { + case SMOOTHFS_MS_PLACED: + goto out; + case SMOOTHFS_MS_PLAN_ACCEPTED: + case SMOOTHFS_MS_DESTINATION_RESERVED: + case SMOOTHFS_MS_COPY_IN_PROGRESS: + case SMOOTHFS_MS_COPY_COMPLETE: + case SMOOTHFS_MS_COPY_VERIFIED: + case SMOOTHFS_MS_FAILED: + break; + default: + err = -EBUSY; + goto out; + } + err = + smoothfs_placement_record(sbi, oid, SMOOTHFS_MS_PLACED, + si->current_tier, si->current_tier, true); + if (!err) { + si->movement_state = SMOOTHFS_MS_PLACED; + si->intended_tier = si->current_tier; + smoothfs_netlink_emit_move_state(sbi, oid, SMOOTHFS_MS_PLACED, + transaction_seq); + } +out: + inode_unlock(inode); + return err; +} + int smoothfs_movement_abort(struct smoothfs_sb_info *sbi, const u8 oid[SMOOTHFS_OID_LEN], u64 transaction_seq, const char *reason) diff --git a/src/smoothfs/netlink.c b/src/smoothfs/netlink.c index b8df939..7821900 100644 --- a/src/smoothfs/netlink.c +++ b/src/smoothfs/netlink.c @@ -258,6 +258,23 @@ static int doit_move_complete(struct sk_buff *skb, struct genl_info *info) seq = nla_get_u64(info->attrs[SMOOTHFS_ATTR_TRANSACTION_SEQ]); return smoothfs_movement_complete(sbi, oid, seq); } +static int doit_move_rollback(struct sk_buff *skb, struct genl_info *info) +{ + struct smoothfs_sb_info *sbi; + const u8 *oid; + u64 seq; + + (void)skb; + sbi = resolve_pool(info); + if (IS_ERR(sbi)) + return PTR_ERR(sbi); + if (!info->attrs[SMOOTHFS_ATTR_OBJECT_ID] || + !info->attrs[SMOOTHFS_ATTR_TRANSACTION_SEQ]) + return -EINVAL; + oid = nla_data(info->attrs[SMOOTHFS_ATTR_OBJECT_ID]); + seq = nla_get_u64(info->attrs[SMOOTHFS_ATTR_TRANSACTION_SEQ]); + return smoothfs_movement_rollback(sbi, oid, seq); +} static int doit_inspect(struct sk_buff *skb, struct genl_info *info) { struct smoothfs_sb_info *sbi; @@ -399,6 +416,8 @@ static const struct genl_multicast_group smoothfs_genl_mcgrps[] = { }; static const struct genl_small_ops smoothfs_genl_ops[] = { + { .cmd = SMOOTHFS_CMD_MOVE_ROLLBACK, .flags = GENL_ADMIN_PERM, + .doit = doit_move_rollback }, { .cmd = SMOOTHFS_CMD_MOVE_COMPLETE, .flags = GENL_ADMIN_PERM, .doit = doit_move_complete }, { .cmd = SMOOTHFS_CMD_REGISTER_POOL, .flags = GENL_ADMIN_PERM, diff --git a/src/smoothfs/smoothfs.h b/src/smoothfs/smoothfs.h index f6b1b22..9570210 100644 --- a/src/smoothfs/smoothfs.h +++ b/src/smoothfs/smoothfs.h @@ -817,6 +817,9 @@ int smoothfs_movement_cutover(struct smoothfs_sb_info *sbi, int smoothfs_movement_complete(struct smoothfs_sb_info *sbi, const u8 oid[SMOOTHFS_OID_LEN], u64 transaction_seq); +int smoothfs_movement_rollback(struct smoothfs_sb_info *sbi, + const u8 oid[SMOOTHFS_OID_LEN], + u64 transaction_seq); int smoothfs_movement_abort(struct smoothfs_sb_info *sbi, const u8 oid[SMOOTHFS_OID_LEN], u64 transaction_seq, const char *reason); diff --git a/src/smoothfs/test/movement_roundtrip.sh b/src/smoothfs/test/movement_roundtrip.sh index 42db1fe..4aa5559 100755 --- a/src/smoothfs/test/movement_roundtrip.sh +++ b/src/smoothfs/test/movement_roundtrip.sh @@ -35,12 +35,26 @@ func main(){ from,to:=round%2,1-round%2;seq:=uint64(500+round) before,e:=c.Inspect(pool,oid);must(e) if before.CurrentTier!=uint8(from)||before.MovementState!=smoothfs.StatePlaced{panic("object was not ready for the next movement")} + // A crash before cutover must discard only the partial destination and + // restore movement readiness on the original source. + rollbackSeq:=seq+1000 + must(c.MovePlan(pool,oid,uint8(to),rollbackSeq)) + if e=c.MoveRollback(pool,oid,rollbackSeq+1);!errors.Is(e,unix.ESTALE){panic(fmt.Sprintf("stale rollback: %v",e))} + partial:=filepath.Join(root,tiers[to],"chunk") + must(os.WriteFile(partial,payload[:128],0600));must(os.Remove(partial)) + parent,e:=os.Open(filepath.Dir(partial));must(e);must(parent.Sync());must(parent.Close()) + must(c.MoveRollback(pool,oid,rollbackSeq));must(c.MoveRollback(pool,oid,rollbackSeq)) + rolledBack,e:=c.Inspect(pool,oid);must(e) + if rolledBack.CurrentTier!=uint8(from)||rolledBack.MovementState!=smoothfs.StatePlaced{panic("rollback changed authoritative placement")} must(c.MovePlan(pool,oid,uint8(to),seq)) if e=c.MoveComplete(pool,oid,seq);!errors.Is(e,unix.EBUSY){panic(fmt.Sprintf("premature completion: %v",e))} src:=filepath.Join(root,tiers[from],"chunk");dst:=filepath.Join(root,tiers[to],"chunk") in,e:=os.Open(src);must(e);out,e:=os.Create(dst);must(e) _,e=io.Copy(out,in);must(e);must(out.Sync());must(out.Close());must(in.Close()) - if before.HasWriteSeq{must(c.MoveCutoverVerifyWriteSeq(pool,oid,seq,before.WriteSeq))}else{must(c.MoveCutover(pool,oid,seq))} + planned,e:=c.Inspect(pool,oid);must(e) + fmt.Printf("round=%d seq=%d kernel_seq=%d write_before=%d write_after_rollback=%d write_planned=%d\n",round,seq,planned.TransactionSeq,before.WriteSeq,rolledBack.WriteSeq,planned.WriteSeq) + if rolledBack.HasWriteSeq{must(c.MoveCutoverVerifyWriteSeq(pool,oid,seq,rolledBack.WriteSeq))}else{must(c.MoveCutover(pool,oid,seq))} + if e=c.MoveRollback(pool,oid,seq);!errors.Is(e,unix.EBUSY){panic(fmt.Sprintf("rollback accepted after cutover: %v",e))} if e=c.MovePlan(pool,oid,uint8(from),seq+100);!errors.Is(e,unix.EBUSY){panic(fmt.Sprintf("new movement before cleanup ack: %v",e))} if e=c.MoveComplete(pool,oid,seq+1);!errors.Is(e,unix.ESTALE){panic(fmt.Sprintf("stale completion: %v",e))} must(os.Remove(src));dir,e:=os.Open(filepath.Dir(src));must(e);must(dir.Sync());must(dir.Close()) @@ -50,7 +64,7 @@ func main(){ _,e=held.Seek(0,0);must(e);actual,e:=io.ReadAll(held);must(e) if !bytes.Equal(actual,payload){panic("held file changed after roundtrip")} } - fmt.Println("movement_roundtrip: PASS (four moves, held descriptor, premature/stale/idempotent acknowledgements)") + fmt.Println("movement_roundtrip: PASS (four moves and rollbacks, held descriptor, premature/stale/idempotent acknowledgements)") } GO go -C "$REPO_ROOT" run "$SPILL_ROOT/probe.go" "$SPILL_ROOT" "$SPILL_UUID" diff --git a/src/smoothfs/uapi_smoothfs.h b/src/smoothfs/uapi_smoothfs.h index 3f1b520..10737ec 100644 --- a/src/smoothfs/uapi_smoothfs.h +++ b/src/smoothfs/uapi_smoothfs.h @@ -54,6 +54,7 @@ enum { SMOOTHFS_CMD_REVOKE_MAPPINGS = 14, SMOOTHFS_EVENT_SPILL = 15, SMOOTHFS_CMD_MOVE_COMPLETE = 16, /* userspace cleanup durable */ + SMOOTHFS_CMD_MOVE_ROLLBACK = 17, /* partial destination removed */ __SMOOTHFS_CMD_MAX, }; #define SMOOTHFS_CMD_MAX (__SMOOTHFS_CMD_MAX - 1) diff --git a/uapi.go b/uapi.go index e1ed1c2..55c8d29 100644 --- a/uapi.go +++ b/uapi.go @@ -32,6 +32,7 @@ const ( CmdRevokeMappings uint8 = 14 EventSpill uint8 = 15 CmdMoveComplete uint8 = 16 + CmdMoveRollback uint8 = 17 ) // Generic netlink attributes. Keep in sync with src/smoothfs/uapi_smoothfs.h. From e85ea3a251262e4bea95efa8862d91258e76f880 Mon Sep 17 00:00:00 2001 From: virant Date: Sat, 31 Oct 2026 21:05:03 +0100 Subject: [PATCH 7/7] fix: flush every backing filesystem on syncfs A stacked syncfs barrier must flush lower address spaces, not just SmoothFS metadata. Synchronize each distinct lower superblock and preserve the first reported error. Add a real XFS delayed-allocation regression that fails on the old callback and passes after the lower flushes. --- src/smoothfs/super.c | 43 ++++++++++++++++------ src/smoothfs/test/run_runtime_harnesses.sh | 1 + src/smoothfs/test/syncfs_lower.sh | 36 ++++++++++++++++++ 3 files changed, 69 insertions(+), 11 deletions(-) create mode 100755 src/smoothfs/test/syncfs_lower.sh diff --git a/src/smoothfs/super.c b/src/smoothfs/super.c index 4888aa3..5d06e7e 100644 --- a/src/smoothfs/super.c +++ b/src/smoothfs/super.c @@ -2421,32 +2421,53 @@ static int smoothfs_show_options(struct seq_file *m, struct dentry *root) } /* - * sync_fs: drain deferred SmoothFS metadata so `sync` / `syncfs` / - * the periodic writeback pass all see up-to-date on-disk state. - * Called with wait=1 for "make it durable", wait=0 for "kick it". - * We flush synchronously when asked to wait and just nudge workers - * when we aren't. + * File data lives in the backing superblocks' address spaces. Draining only + * SmoothFS metadata is not a syncfs durability barrier (PBS uses syncfs when + * completing a backup). Flush each distinct backing FS when VFS asks to wait. */ static int smoothfs_sync_fs(struct super_block *sb, int wait) { struct smoothfs_sb_info *sbi = SMOOTHFS_SB(sb); + int err = 0, ret; + u8 tier, prior; if (!sbi) return 0; if (wait) { - int err = smoothfs_cache_drain(sbi); - if (err) - return err; + ret = smoothfs_cache_drain(sbi); + if (ret) + err = ret; smoothfs_oid_wb_drain(sbi); smoothfs_placement_wb_drain(sbi); - if (sbi->placement_log) - vfs_fsync(sbi->placement_log, 0); + if (sbi->placement_log) { + ret = vfs_fsync(sbi->placement_log, 0); + if (ret && !err) + err = ret; + } + for (tier = 0; tier < sbi->ntiers; tier++) { + struct super_block *lower = + sbi->tiers[tier].lower_path.mnt->mnt_sb; + + for (prior = 0; prior < tier; prior++) + if (sbi->tiers[prior].lower_path.mnt->mnt_sb == + lower) + break; + if (prior != tier) + continue; + /* Match the stacked-filesystem lock contract used by + * overlayfs. */ + down_read(&lower->s_umount); + ret = sync_filesystem(lower); + up_read(&lower->s_umount); + if (ret && !err) + err = ret; + } } else { smoothfs_placement_wb_kick(sbi); if (sbi->oid_wb_ready) mod_delayed_work(sbi->oid_wb_wq, &sbi->oid_wb_work, 0); } - return 0; + return err; } const struct super_operations smoothfs_super_ops = { diff --git a/src/smoothfs/test/run_runtime_harnesses.sh b/src/smoothfs/test/run_runtime_harnesses.sh index 0bc1527..02e0b7c 100755 --- a/src/smoothfs/test/run_runtime_harnesses.sh +++ b/src/smoothfs/test/run_runtime_harnesses.sh @@ -25,6 +25,7 @@ FAIL_FAST=${SMOOTHFS_RUNTIME_FAIL_FAST:-0} core_tests=( cache_mode.sh directory_fsync.sh + syncfs_lower.sh readdir_backing_permissions.sh mkdir_enospc.sh netlink_receive_close.sh diff --git a/src/smoothfs/test/syncfs_lower.sh b/src/smoothfs/test/syncfs_lower.sh new file mode 100755 index 0000000..0934bc7 --- /dev/null +++ b/src/smoothfs/test/syncfs_lower.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# syncfs on a stacked mount must flush buffered file data on every backing FS. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +. "$SCRIPT_DIR/tier_spill_lib.sh" +SPILL_ROOT=${SMOOTHFS_SYNCFS_ROOT:-/tmp/smoothfs-syncfs-lower} +SPILL_UUID=00000000-0000-0000-0000-00000000f520 +trap spill_cleanup EXIT +spill_setup_pool syncfs-lower "$SPILL_UUID" +python3 - "$SPILL_ROOT" <<'PY' +import ctypes,fcntl,os,struct,sys +from pathlib import Path +root=Path(sys.argv[1]);payload=b'backing filesystem sync boundary\n'*(256*1024) +paths=[root/'fast'/'direct-fast',root/'slow'/'direct-slow'] +for p in paths: + with p.open('wb') as f:f.write(payload) +with (root/'server'/'upper-write').open('wb') as f:f.write(payload) +paths.append(root/'fast'/'upper-write') +def delayed(p): + fd=os.open(p,os.O_RDONLY) + try: + buf=bytearray(struct.pack('