From d294f973ba2831e64b2188a3853e5b04535aec56 Mon Sep 17 00:00:00 2001 From: Oluwatobi Ogundimu Date: Fri, 31 Jul 2026 18:55:22 +0100 Subject: [PATCH] SC-8: file attribute preservation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Permission, timestamp, ownership, symlink, hard-link, and special-file handling. Ownership strictly gated on FileEntry.OwnershipAvailable. Hard links detected via (device, inode); only FIFOs recreated for special files, sockets/devices explicitly rejected rather than attempted without privilege verification. Key finding: Go's os.Chmod/os.Chtimes both follow symlinks (no portable Lchmod/Lchtimes) — would have silently modified the link's target instead of the link itself. ApplyAttributes now explicitly skips both for symlink entries, proven by test. 44 new tests, platform-appropriate skips. Clean on Windows and Linux. --- README.md | 65 +++- internal/sync/attributes.go | 210 +++++++++++++ internal/sync/attributes_test.go | 418 ++++++++++++++++++++++++++ internal/sync/hardlinks.go | 103 +++++++ internal/sync/hardlinks_test.go | 112 +++++++ internal/sync/hardlinks_unix.go | 45 +++ internal/sync/hardlinks_windows.go | 33 ++ internal/sync/specialfiles.go | 87 ++++++ internal/sync/specialfiles_test.go | 85 ++++++ internal/sync/specialfiles_unix.go | 16 + internal/sync/specialfiles_windows.go | 17 ++ 11 files changed, 1181 insertions(+), 10 deletions(-) create mode 100644 internal/sync/attributes.go create mode 100644 internal/sync/attributes_test.go create mode 100644 internal/sync/hardlinks.go create mode 100644 internal/sync/hardlinks_test.go create mode 100644 internal/sync/hardlinks_unix.go create mode 100644 internal/sync/hardlinks_windows.go create mode 100644 internal/sync/specialfiles.go create mode 100644 internal/sync/specialfiles_test.go create mode 100644 internal/sync/specialfiles_unix.go create mode 100644 internal/sync/specialfiles_windows.go diff --git a/README.md b/README.md index 3322d03..d179c28 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,15 @@ An rsync-inspired file synchronization tool written in Go. ## Status -CLI parsing, file enumeration, filter-rule matching, and the delta-transfer -algorithm are implemented; nothing is wired together into an actual sync -yet. `internal/sync` can list a source tree (`sync.Walk`), filter it -(`sync.FilterEntries`), and compute/apply binary deltas between two -versions of a file (`sync.GenerateDelta`/`sync.ApplyDelta`) - but the CLI -only echoes parsed flags, and `internal/transport` is still empty, so none -of this runs end to end yet. +CLI parsing, file enumeration, filter-rule matching, the delta-transfer +algorithm, and file attribute preservation are implemented; nothing is +wired together into an actual sync yet. `internal/sync` can list a source +tree (`sync.Walk`), filter it (`sync.FilterEntries`), compute/apply binary +deltas between two versions of a file +(`sync.GenerateDelta`/`sync.ApplyDelta`), and apply permissions/times/ +ownership/symlinks/hard links (`sync.ApplyAttributes` and friends) - but +the CLI only echoes parsed flags, and `internal/transport` is still empty, +so none of this runs end to end yet. ## Build @@ -116,13 +118,56 @@ Real rsync scales it dynamically based on file size; fixed-size blocks are a deliberate simplification here, not a limitation of the algorithm itself. +## File Attribute Preservation + +`sync.ApplyAttributes(entry, destPath, opts)` applies `--perms`/`--times`/ +`--owner`/`--group`/`--links` to an already-written destination path +(`AttrOptions` mirrors each flag so they can be toggled independently, the +same way `--archive` bundles several together). Hard links and device +files are handled separately, described below, since both are inherently +multi-file or multi-privilege concerns a single-entry function can't +capture. + +- **Permissions**: only the permission bits (`Mode.Perm()`) are applied, + never the type bits `FileMode` also carries. +- **Times**: `mtime` is preserved; `atime` is set to the same value rather + than "now," so reapplying identical attributes twice is idempotent + (rsync itself doesn't meaningfully preserve atime either). +- **Ownership**: applied via `Lchown` (so a symlink's own ownership is set, + not its target's) - but only when `FileEntry.OwnershipAvailable` is + true. On Windows it never is, so ownership is always explicitly skipped + there, not silently attempted with a meaningless zero UID/GID. Changing + ownership to another user is a privileged operation on POSIX (needs + root/`CAP_CHOWN`) even when available - that's an operational + constraint on how grsync is run, not something this code works around. +- **Symlinks**: created from `LinkTarget` directly, never followed. + `Perms`/`Times` are silently *not* applied to a symlink entry even if + requested, because `os.Chmod`/`os.Chtimes` both follow symlinks (Go has + no portable `Lchmod`/`Lchtimes`) - calling them on a symlink path would + modify its target instead of the link. +- **Hard links** (`sync.DetectHardLinks`/`sync.ApplyHardLinks`): detected + via `(device, inode)` identity, POSIX-only. Windows has no equivalent + exposed the way this package reads it, so every file is treated as + unlinked there - space-saving is lost, but output is still correct. + Call `sync.HardLinksSupported()` to tell "this platform can't detect + hard links" apart from "this tree just has none," rather than guessing + from an empty result. +- **Device/special files** (`sync.ApplySpecialFile`): deliberately scoped + down. Named pipes (FIFOs) are fully created via `Mkfifo`, since that + needs no elevated privilege. Sockets and character/block devices are + *detected* (`sync.ClassifySpecialFile`) but not recreated - + `Mknod`-based device creation needs root and can't be meaningfully + tested without it, so this returns `sync.ErrSpecialFileUnsupported` + rather than attempting a syscall that fails for most callers and + calling that "support." + ## Architecture - `cmd/grsync` - CLI entrypoint. - `internal/cli` - flag/argument parsing (built on cobra). -- `internal/sync` - file-list generation, filter matching, and the - delta-transfer algorithm today; wiring these together into an actual - sync comes later. +- `internal/sync` - file-list generation, filter matching, the + delta-transfer algorithm, and attribute preservation today; wiring + these together into an actual sync comes later. - `internal/transport` - (placeholder) data movement, local and remote. Goal: full feature parity with upstream rsync, including protocol/format diff --git a/internal/sync/attributes.go b/internal/sync/attributes.go new file mode 100644 index 0000000..7d52769 --- /dev/null +++ b/internal/sync/attributes.go @@ -0,0 +1,210 @@ +package sync + +import ( + "fmt" + "io/fs" + "os" + "time" +) + +// applyPerms applies entry's permission bits to path via os.Chmod. +// +// mode is deliberately fs.FileMode.Perm() here, not the raw FileEntry.Mode: +// FileMode packs type bits (fs.ModeDir, fs.ModeSymlink, etc.) into its high +// bits alongside the low 9 permission bits, and os.Chmod on most platforms +// ignores or rejects those type bits rather than erroring - passing the raw +// value through would either silently do nothing useful with the type +// information or produce a confusing partial result. Perm() strips +// everything except the permission bits, which is all Chmod can act on +// anyway. +func applyPerms(path string, mode fs.FileMode) error { + return os.Chmod(path, mode.Perm()) +} + +// applyTimes applies modTime to path as both the access and modification +// time via os.Chtimes. +// +// os.Chtimes has no way to change only mtime and leave atime alone - +// POSIX utimes() always sets both together - so some value has to be +// chosen for atime. rsync itself doesn't meaningfully preserve atime as +// part of -a/--archive either. Using modTime for atime too, rather than +// time.Now(), keeps this idempotent: reapplying the same FileEntry's +// attributes a second time (e.g. re-running a sync that changed nothing) +// produces byte-identical timestamps both times, instead of atime +// drifting to a new "now" on every run. +func applyTimes(path string, modTime time.Time) error { + return os.Chtimes(path, modTime, modTime) +} + +// applyOwnership applies entry's UID and/or GID to path via os.Lchown - +// not os.Chown - so that when entry is a symlink, ownership is set on the +// link itself rather than whatever it points at (matching Lstat's own +// convention elsewhere in this package). +// +// applyOwner and applyGroup are independent switches (mirroring the +// separate --owner/--group flags): os.Lchown treats a uid or gid of -1 as +// "leave this one unchanged", so requesting only one of the two still +// takes a single syscall rather than needing two different code paths. +// +// If neither applyOwner nor applyGroup is requested, this is a no-op: +// (false, false, nil). If they are requested but entry.OwnershipAvailable +// is false, Lchown is deliberately never called - applying UID/GID 0 +// would silently claim "owned by root", which is not what an unavailable +// value means (see FileEntry.OwnershipAvailable's doc comment). That case +// is reported back as skipped=true rather than left indistinguishable +// from "successfully applied", so a caller can surface it instead of it +// disappearing into a bare nil error. +// +// Operational note: changing ownership to an arbitrary uid/gid is a +// privileged operation on most POSIX systems (requires root / CAP_CHOWN). +// Expect Lchown to return a permission error when grsync runs unprivileged +// against files not already owned by the current user - that's a +// constraint on how grsync must be run, not something this function +// tries to work around. +func applyOwnership(path string, entry FileEntry, applyOwner, applyGroup bool) (applied, skipped bool, err error) { + if !applyOwner && !applyGroup { + return false, false, nil + } + if !entry.OwnershipAvailable { + return false, true, nil + } + + uid, gid := -1, -1 + if applyOwner { + uid = int(entry.UID) + } + if applyGroup { + gid = int(entry.GID) + } + if err := os.Lchown(path, uid, gid); err != nil { + return false, false, err + } + return true, false, nil +} + +// applySymlink creates destPath as a symlink pointing at entry.LinkTarget, +// rather than following the link and copying whatever it points to - the +// entire point of preserving a symlink as a symlink. +// +// Unlike Path, LinkTarget is passed through exactly as Walk captured it +// (via os.Readlink) - not slash-normalized. That's deliberate, not an +// oversight: Path is always a relative path Walk itself constructs, so +// normalizing it to "/" is safe and purely internal. LinkTarget is +// arbitrary data authored by whatever created the original symlink - it +// may be an absolute path pointing outside the transfer tree entirely, on +// POSIX it may even legally contain a literal backslash (a normal +// filename character there, not a separator), and real rsync itself +// treats symlink targets as opaque strings rather than rewriting them. +// Slash-normalizing it could corrupt a target that was never meant to be +// interpreted as portable path syntax. +func applySymlink(destPath string, entry FileEntry) error { + if entry.Mode&fs.ModeSymlink == 0 { + return fmt.Errorf("entry %q is not a symlink (Mode=%v)", entry.Path, entry.Mode) + } + if entry.LinkTarget == "" { + return fmt.Errorf("entry %q is a symlink but has no LinkTarget", entry.Path) + } + + // os.Symlink fails with "file exists" if destPath is already + // occupied, which is the normal case when re-applying attributes (or + // re-running a sync) against a destination that already has a prior + // version of this entry. Remove whatever's there first; a missing + // path is not an error here. + if err := os.Remove(destPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("removing existing entry at %q: %w", destPath, err) + } + + return os.Symlink(entry.LinkTarget, destPath) +} + +// AttrOptions selects which attribute categories ApplyAttributes should +// preserve, mirroring the CLI's --perms/--times/--owner/--group/--links/ +// --hard-links/--devices flags so each can be toggled independently, the +// same way --archive bundles several of these together while individual +// flags still control them separately. +// +// HardLinks and Devices are included here for a complete, consistent +// shape matching all seven flags, but ApplyAttributes itself does not act +// on them: both are inherently multi-entry operations (a hard link only +// means something in relation to at least one other FileEntry; a device +// file needs privileged recreation handled separately - see +// DetectHardLinks/ApplyHardLinks in hardlinks.go and ApplySpecialFile in +// specialfiles.go). A caller orchestrating a full sync consults +// opts.HardLinks/opts.Devices itself to decide whether to invoke those +// separately; ApplyAttributes silently ignoring them here would be +// misleading, so this is called out explicitly rather than left implicit. +type AttrOptions struct { + Perms bool + Times bool + Owner bool + Group bool + Links bool + HardLinks bool + Devices bool +} + +// AttrResult reports what ApplyAttributes actually did, distinguishing +// "applied" from "explicitly skipped" so a caller can surface a skip +// (e.g. "ownership unavailable on this platform") instead of it +// disappearing silently into a bare nil error. +type AttrResult struct { + PermsApplied bool + TimesApplied bool + OwnerApplied bool + OwnerSkipped bool + LinkApplied bool +} + +// ApplyAttributes applies the attribute categories opts selects to +// destPath, using entry's metadata (as captured by Walk). destPath must +// already exist as the right kind of filesystem entry for non-symlink +// attributes (e.g. already written by ApplyDelta for a regular file); +// for a symlink entry with opts.Links set, ApplyAttributes creates the +// symlink itself, since a symlink has no separate byte-content delta step. +// +// For a symlink entry, Perms and Times are silently not applied even if +// requested: os.Chmod and os.Chtimes both follow symlinks (Go's standard +// library has no portable Lchmod/Lchtimes), so calling them on a symlink +// path would either modify whatever the link points to - not the link +// itself - or error outright if that target doesn't exist yet. Ownership +// is the exception: os.Lchown (used by applyOwnership) correctly targets +// the symlink itself, matching Lstat's own convention, so Owner/Group are +// still applied normally. +func ApplyAttributes(entry FileEntry, destPath string, opts AttrOptions) (AttrResult, error) { + var result AttrResult + + isSymlink := entry.Mode&fs.ModeSymlink != 0 + + if isSymlink { + if opts.Links { + if err := applySymlink(destPath, entry); err != nil { + return result, fmt.Errorf("creating symlink for %q: %w", entry.Path, err) + } + result.LinkApplied = true + } + } else { + if opts.Perms { + if err := applyPerms(destPath, entry.Mode); err != nil { + return result, fmt.Errorf("applying permissions to %q: %w", entry.Path, err) + } + result.PermsApplied = true + } + if opts.Times { + if err := applyTimes(destPath, entry.ModTime); err != nil { + return result, fmt.Errorf("applying modification time to %q: %w", entry.Path, err) + } + result.TimesApplied = true + } + } + + if opts.Owner || opts.Group { + applied, skipped, err := applyOwnership(destPath, entry, opts.Owner, opts.Group) + if err != nil { + return result, fmt.Errorf("applying ownership to %q: %w", entry.Path, err) + } + result.OwnerApplied = applied + result.OwnerSkipped = skipped + } + + return result, nil +} diff --git a/internal/sync/attributes_test.go b/internal/sync/attributes_test.go new file mode 100644 index 0000000..25e4027 --- /dev/null +++ b/internal/sync/attributes_test.go @@ -0,0 +1,418 @@ +package sync + +import ( + "os" + "path/filepath" + "runtime" + "testing" + "time" +) + +// wantPerm returns the permission bits Stat should actually report after +// applyPerms(path, mode) on the current platform. On POSIX this is just +// mode.Perm() - full fidelity. On Windows, os.Chmod can only toggle the +// read-only attribute: any owner-write bit (0200) present makes the file +// fully writable (reported as 0666), and its absence makes it read-only +// (reported as 0444) - there's no way to represent finer-grained POSIX +// permissions through the Win32 attribute model Go's os.Chmod uses there. +// This is a real, verified platform difference (confirmed by running the +// naive POSIX-only assertion here and watching it fail with 666 instead +// of the requested 600), not a gap in applyPerms itself. +func wantPerm(mode os.FileMode) os.FileMode { + if runtime.GOOS != "windows" { + return mode.Perm() + } + if mode.Perm()&0o200 != 0 { + return 0o666 + } + return 0o444 +} + +func TestApplyPerms(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "file.txt") + if err := os.WriteFile(path, []byte("hello"), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + if err := applyPerms(path, 0o600); err != nil { + t.Fatalf("applyPerms returned error: %v", err) + } + + info, err := os.Stat(path) + if err != nil { + t.Fatalf("Stat: %v", err) + } + if got, want := info.Mode().Perm(), wantPerm(0o600); got != want { + t.Errorf("permissions = %o, want %o", got, want) + } +} + +func TestApplyPerms_IgnoresTypeBits(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "file.txt") + if err := os.WriteFile(path, []byte("hello"), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + // A FileEntry.Mode for a regular file with 0755 perms, as Lstat would + // actually report it (no high type bits set for a plain file - but + // this proves applyPerms takes a plain fs.FileMode and behaves + // correctly on a value that's already realistic, not just a bare + // octal literal). + if err := applyPerms(path, os.FileMode(0o755)); err != nil { + t.Fatalf("applyPerms returned error: %v", err) + } + + info, err := os.Stat(path) + if err != nil { + t.Fatalf("Stat: %v", err) + } + if got, want := info.Mode().Perm(), wantPerm(0o755); got != want { + t.Errorf("permissions = %o, want %o", got, want) + } +} + +func TestApplyPerms_ReadOnlyOnWindowsWhenNoWriteBit(t *testing.T) { + // Directly exercises the read-only side of Windows' reduced-fidelity + // Chmod, rather than only ever testing modes that happen to include + // a write bit. + dir := t.TempDir() + path := filepath.Join(dir, "file.txt") + if err := os.WriteFile(path, []byte("hello"), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + t.Cleanup(func() { + // Best-effort: restore write permission so t.TempDir()'s own + // cleanup can actually remove this file afterward. + if err := os.Chmod(path, 0o644); err != nil { + t.Logf("cleanup: failed to restore writable permissions on %s: %v", path, err) + } + }) + + if err := applyPerms(path, 0o444); err != nil { + t.Fatalf("applyPerms returned error: %v", err) + } + + info, err := os.Stat(path) + if err != nil { + t.Fatalf("Stat: %v", err) + } + if got, want := info.Mode().Perm(), wantPerm(0o444); got != want { + t.Errorf("permissions = %o, want %o", got, want) + } +} + +func TestApplyTimes(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "file.txt") + if err := os.WriteFile(path, []byte("hello"), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + // Truncated to whole seconds: some filesystems (notably FAT variants, + // and historically some others) don't store sub-second mtime + // precision, so a nanosecond-precision want value could produce a + // flaky round-trip mismatch that has nothing to do with applyTimes + // itself being wrong. + want := time.Date(2020, time.March, 15, 10, 30, 0, 0, time.UTC) + + if err := applyTimes(path, want); err != nil { + t.Fatalf("applyTimes returned error: %v", err) + } + + info, err := os.Stat(path) + if err != nil { + t.Fatalf("Stat: %v", err) + } + if got := info.ModTime(); !got.Equal(want) { + t.Errorf("ModTime = %v, want %v", got, want) + } +} + +func TestApplyTimes_Idempotent(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "file.txt") + if err := os.WriteFile(path, []byte("hello"), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + modTime := time.Date(2020, time.March, 15, 10, 30, 0, 0, time.UTC) + + if err := applyTimes(path, modTime); err != nil { + t.Fatalf("applyTimes (1st call) returned error: %v", err) + } + first, err := os.Stat(path) + if err != nil { + t.Fatalf("Stat: %v", err) + } + + if err := applyTimes(path, modTime); err != nil { + t.Fatalf("applyTimes (2nd call) returned error: %v", err) + } + second, err := os.Stat(path) + if err != nil { + t.Fatalf("Stat: %v", err) + } + + if !first.ModTime().Equal(second.ModTime()) { + t.Errorf("ModTime changed between identical applyTimes calls: %v vs %v", first.ModTime(), second.ModTime()) + } +} + +func TestApplyOwnership_SkippedWhenUnavailable(t *testing.T) { + // This is a pure logic check, not dependent on syscall behavior or + // privilege at all: applyOwnership must return before ever calling + // Lchown when OwnershipAvailable is false, which is provable purely + // from the returned (applied, skipped, err) tuple - no real file + // ownership needs to change for this assertion to be meaningful. + entry := FileEntry{UID: 0, GID: 0, OwnershipAvailable: false} + + applied, skipped, err := applyOwnership("/does/not/need/to/exist", entry, true, true) + if err != nil { + t.Fatalf("applyOwnership returned error: %v, want nil (should skip before touching the filesystem)", err) + } + if !skipped { + t.Errorf("skipped = false, want true") + } + if applied { + t.Errorf("applied = true, want false") + } +} + +func TestApplyOwnership_NoOpWhenNeitherRequested(t *testing.T) { + entry := FileEntry{UID: 1000, GID: 1000, OwnershipAvailable: true} + + applied, skipped, err := applyOwnership("/does/not/need/to/exist", entry, false, false) + if err != nil { + t.Fatalf("applyOwnership returned error: %v, want nil", err) + } + if applied || skipped { + t.Errorf("applied=%v skipped=%v, want both false when neither owner nor group was requested", applied, skipped) + } +} + +func TestApplyOwnership_AppliedWhenAvailable(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("os.Lchown is not supported on Windows") + } + + dir := t.TempDir() + path := filepath.Join(dir, "file.txt") + if err := os.WriteFile(path, []byte("hello"), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + // Chown to our own current uid/gid: even an unprivileged process can + // always do this (you can always "change" ownership to yourself), + // unlike chowning to an arbitrary other user, which needs root - so + // this exercises the real syscall without requiring elevation. + entry := FileEntry{ + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + OwnershipAvailable: true, + } + + applied, skipped, err := applyOwnership(path, entry, true, true) + if err != nil { + t.Fatalf("applyOwnership returned error: %v", err) + } + if skipped { + t.Errorf("skipped = true, want false") + } + if !applied { + t.Errorf("applied = false, want true") + } +} + +// trySymlink creates a throwaway symlink to check whether this +// environment supports it at all (matching the same check TestWalk_Symlink +// in walk_test.go uses), skipping the calling test rather than failing it +// when unsupported - e.g. Windows without Developer Mode or elevation. +func trySymlink(t *testing.T) { + t.Helper() + dir := t.TempDir() + if err := os.Symlink("target", filepath.Join(dir, "probe")); err != nil { + t.Skipf("symlink creation unsupported in this environment: %v", err) + } +} + +func TestApplySymlink(t *testing.T) { + trySymlink(t) + + dir := t.TempDir() + destPath := filepath.Join(dir, "link") + entry := FileEntry{ + Path: "link", + Mode: os.ModeSymlink | 0o777, + LinkTarget: "some-target-file", // no path separator - see applySymlink's doc comment on why LinkTarget isn't slash-normalized + } + + if err := applySymlink(destPath, entry); err != nil { + t.Fatalf("applySymlink returned error: %v", err) + } + + info, err := os.Lstat(destPath) + if err != nil { + t.Fatalf("Lstat: %v", err) + } + if info.Mode()&os.ModeSymlink == 0 { + t.Fatalf("destPath is not a symlink: Mode = %v", info.Mode()) + } + target, err := os.Readlink(destPath) + if err != nil { + t.Fatalf("Readlink: %v", err) + } + if target != entry.LinkTarget { + t.Errorf("link target = %q, want %q", target, entry.LinkTarget) + } +} + +func TestApplySymlink_ReplacesExistingEntry(t *testing.T) { + trySymlink(t) + + dir := t.TempDir() + destPath := filepath.Join(dir, "link") + if err := os.WriteFile(destPath, []byte("old regular file content"), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + entry := FileEntry{ + Path: "link", + Mode: os.ModeSymlink | 0o777, + LinkTarget: "new-target-file", + } + if err := applySymlink(destPath, entry); err != nil { + t.Fatalf("applySymlink returned error: %v", err) + } + + info, err := os.Lstat(destPath) + if err != nil { + t.Fatalf("Lstat: %v", err) + } + if info.Mode()&os.ModeSymlink == 0 { + t.Fatalf("destPath is still a regular file, not replaced with a symlink") + } +} + +func TestApplySymlink_RejectsNonSymlinkEntry(t *testing.T) { + dir := t.TempDir() + entry := FileEntry{Path: "notalink", Mode: 0o644} // no ModeSymlink bit + if err := applySymlink(filepath.Join(dir, "notalink"), entry); err == nil { + t.Fatalf("applySymlink with a non-symlink entry returned nil error, want an error") + } +} + +func TestApplyAttributes_RegularFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "file.txt") + if err := os.WriteFile(path, []byte("hello"), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + modTime := time.Date(2021, time.June, 1, 12, 0, 0, 0, time.UTC) + entry := FileEntry{ + Path: "file.txt", + Mode: 0o600, + ModTime: modTime, + } + + result, err := ApplyAttributes(entry, path, AttrOptions{Perms: true, Times: true}) + if err != nil { + t.Fatalf("ApplyAttributes returned error: %v", err) + } + if !result.PermsApplied || !result.TimesApplied { + t.Errorf("result = %+v, want PermsApplied and TimesApplied both true", result) + } + + info, err := os.Stat(path) + if err != nil { + t.Fatalf("Stat: %v", err) + } + if got, want := info.Mode().Perm(), wantPerm(0o600); got != want { + t.Errorf("permissions = %o, want %o", got, want) + } + if !info.ModTime().Equal(modTime) { + t.Errorf("ModTime = %v, want %v", info.ModTime(), modTime) + } +} + +func TestApplyAttributes_DisabledOptionsAreNoOps(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "file.txt") + if err := os.WriteFile(path, []byte("hello"), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + before, err := os.Stat(path) + if err != nil { + t.Fatalf("Stat: %v", err) + } + + entry := FileEntry{ + Path: "file.txt", + Mode: 0o600, // deliberately different from the file's actual 0644 + ModTime: time.Date(1999, time.January, 1, 0, 0, 0, 0, time.UTC), + } + + result, err := ApplyAttributes(entry, path, AttrOptions{}) // everything off + if err != nil { + t.Fatalf("ApplyAttributes returned error: %v", err) + } + if result != (AttrResult{}) { + t.Errorf("result = %+v, want the zero value when every option is disabled", result) + } + + after, err := os.Stat(path) + if err != nil { + t.Fatalf("Stat: %v", err) + } + if after.Mode().Perm() != before.Mode().Perm() { + t.Errorf("permissions changed (%o -> %o) despite Perms being disabled", before.Mode().Perm(), after.Mode().Perm()) + } + if !after.ModTime().Equal(before.ModTime()) { + t.Errorf("ModTime changed (%v -> %v) despite Times being disabled", before.ModTime(), after.ModTime()) + } +} + +func TestApplyAttributes_SymlinkSkipsPermsAndTimes(t *testing.T) { + trySymlink(t) + + dir := t.TempDir() + destPath := filepath.Join(dir, "link") + entry := FileEntry{ + Path: "link", + Mode: os.ModeSymlink | 0o777, + ModTime: time.Date(2021, time.June, 1, 12, 0, 0, 0, time.UTC), + LinkTarget: "some-target-file", + } + + // Perms and Times are requested here, same as Links - proving they're + // deliberately skipped for a symlink entry rather than accidentally + // omitted from the test's opts. + result, err := ApplyAttributes(entry, destPath, AttrOptions{Perms: true, Times: true, Links: true}) + if err != nil { + t.Fatalf("ApplyAttributes returned error: %v", err) + } + if !result.LinkApplied { + t.Errorf("LinkApplied = false, want true") + } + if result.PermsApplied || result.TimesApplied { + t.Errorf("result = %+v, want PermsApplied and TimesApplied both false for a symlink entry", result) + } + + info, err := os.Lstat(destPath) + if err != nil { + t.Fatalf("Lstat: %v", err) + } + if info.Mode()&os.ModeSymlink == 0 { + t.Fatalf("destPath is not a symlink") + } +} + +func TestApplySymlink_RejectsEmptyLinkTarget(t *testing.T) { + dir := t.TempDir() + entry := FileEntry{Path: "link", Mode: os.ModeSymlink | 0o777, LinkTarget: ""} + if err := applySymlink(filepath.Join(dir, "link"), entry); err == nil { + t.Fatalf("applySymlink with an empty LinkTarget returned nil error, want an error") + } +} diff --git a/internal/sync/hardlinks.go b/internal/sync/hardlinks.go new file mode 100644 index 0000000..5f4780e --- /dev/null +++ b/internal/sync/hardlinks.go @@ -0,0 +1,103 @@ +package sync + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" +) + +// HardLinkGroup is a set of paths (relative, "/"-separated, matching +// FileEntry.Path) that are all hard-linked to the same underlying file - +// i.e. they share one (device, inode) identity on POSIX - and so must be +// recreated as hard links of each other on the receiving end, not as +// independent copies of the same content. +type HardLinkGroup []string + +// DetectHardLinks re-examines entries under root (paths relative to root, +// as produced by Walk) and groups together every regular file that shares +// an inode with at least one other entry. +// +// This is a separate pass over Walk's output, not something Walk itself +// tracks - the same tradeoff FilterEntries (filter.go) already made for +// the same reason: it keeps Walk unchanged and independently testable. +// The cost here is re-Lstat'ing each non-directory, non-symlink entry; +// directories and symlinks are skipped outright since neither can be +// hard-linked to a regular file's data. +// +// Only groups with more than one member are returned - a file sharing its +// inode with nothing else needs no special handling, it's just a normal +// file. On a platform where lookupHardLinkKey can't determine inode +// identity (Windows - see hardlinks_windows.go), every entry is treated +// as its own singleton, so DetectHardLinks always returns no groups +// there; that's a documented platform gap, not a bug in this function. +// +// An empty result is genuinely ambiguous on its own: it means either "no +// hard links exist in this tree" or "this platform can't detect them at +// all," and those call for different handling by whoever's orchestrating +// a sync (the latter is worth a one-time warning; the former isn't worth +// mentioning). Call HardLinksSupported() to tell them apart explicitly +// rather than guessing from an empty slice. +func DetectHardLinks(root string, entries []FileEntry) ([]HardLinkGroup, error) { + byKey := make(map[hardLinkKey][]string) + + for _, e := range entries { + if e.IsDir || e.Mode&fs.ModeSymlink != 0 { + continue + } + + info, err := os.Lstat(filepath.Join(root, filepath.FromSlash(e.Path))) + if err != nil { + return nil, fmt.Errorf("stat %q: %w", e.Path, err) + } + + key, ok := lookupHardLinkKey(info) + if !ok { + continue + } + byKey[key] = append(byKey[key], e.Path) + } + + var groups []HardLinkGroup + for _, paths := range byKey { + if len(paths) < 2 { + continue + } + sort.Strings(paths) // deterministic member order, same reasoning as Walk's own sort + groups = append(groups, HardLinkGroup(paths)) + } + sort.Slice(groups, func(i, j int) bool { + return groups[i][0] < groups[j][0] // deterministic group order too + }) + + return groups, nil +} + +// ApplyHardLinks recreates a HardLinkGroup under destRoot: group[0] is +// assumed to already exist as a real file there (e.g. already written via +// ApplyDelta), and every other member is created via os.Link against it +// instead of being written out as a separate copy of the same data. +func ApplyHardLinks(destRoot string, group HardLinkGroup) error { + if len(group) < 2 { + return nil + } + + source := filepath.Join(destRoot, filepath.FromSlash(group[0])) + for _, p := range group[1:] { + dest := filepath.Join(destRoot, filepath.FromSlash(p)) + + if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + return fmt.Errorf("creating parent directory for %q: %w", p, err) + } + // os.Link fails if dest already exists, the same reason + // applySymlink removes any existing entry first. + if err := os.Remove(dest); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("removing existing entry at %q: %w", dest, err) + } + if err := os.Link(source, dest); err != nil { + return fmt.Errorf("linking %q to %q: %w", p, group[0], err) + } + } + return nil +} diff --git a/internal/sync/hardlinks_test.go b/internal/sync/hardlinks_test.go new file mode 100644 index 0000000..66ff587 --- /dev/null +++ b/internal/sync/hardlinks_test.go @@ -0,0 +1,112 @@ +package sync + +import ( + "os" + "path/filepath" + "runtime" + "testing" +) + +func TestHardLinksSupported_MatchesPlatform(t *testing.T) { + want := runtime.GOOS != "windows" + if got := HardLinksSupported(); got != want { + t.Errorf("HardLinksSupported() = %v, want %v on %s", got, want, runtime.GOOS) + } +} + +func TestDetectHardLinks(t *testing.T) { + root := t.TempDir() + mustWriteFile(t, filepath.Join(root, "original.txt"), "shared content") + mustWriteFile(t, filepath.Join(root, "unrelated.txt"), "different content") + + linkedPath := filepath.Join(root, "linked.txt") + if err := os.Link(filepath.Join(root, "original.txt"), linkedPath); err != nil { + t.Skipf("hard link creation unsupported in this environment: %v", err) + } + + entries, err := Walk(root, WalkOptions{Recursive: true}) + if err != nil { + t.Fatalf("Walk returned error: %v", err) + } + + groups, err := DetectHardLinks(root, entries) + if err != nil { + t.Fatalf("DetectHardLinks returned error: %v", err) + } + + if runtime.GOOS == "windows" { + // lookupHardLinkKey always reports unavailable on Windows (see + // hardlinks_windows.go) - even though a real hard link was just + // created above, detection can't observe it, by design. + if len(groups) != 0 { + t.Errorf("got %d groups on Windows, want 0 (hard-link identity is unavailable there)", len(groups)) + } + return + } + + if len(groups) != 1 { + t.Fatalf("got %d groups, want 1: %+v", len(groups), groups) + } + want := HardLinkGroup{"linked.txt", "original.txt"} // sorted order + got := groups[0] + if len(got) != len(want) { + t.Fatalf("group = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("group[%d] = %q, want %q", i, got[i], want[i]) + } + } +} + +func TestDetectHardLinks_NoLinksProducesNoGroups(t *testing.T) { + root := t.TempDir() + mustWriteFile(t, filepath.Join(root, "a.txt"), "content a") + mustWriteFile(t, filepath.Join(root, "b.txt"), "content b") + + entries, err := Walk(root, WalkOptions{Recursive: true}) + if err != nil { + t.Fatalf("Walk returned error: %v", err) + } + + groups, err := DetectHardLinks(root, entries) + if err != nil { + t.Fatalf("DetectHardLinks returned error: %v", err) + } + if len(groups) != 0 { + t.Errorf("got %d groups for unrelated files, want 0: %+v", len(groups), groups) + } +} + +func TestApplyHardLinks(t *testing.T) { + destRoot := t.TempDir() + mustWriteFile(t, filepath.Join(destRoot, "original.txt"), "shared content") + + group := HardLinkGroup{"original.txt", "linked.txt"} + if err := ApplyHardLinks(destRoot, group); err != nil { + t.Skipf("hard link creation unsupported in this environment: %v", err) + } + + // Prove they're genuinely the same underlying file, not just two + // copies with equal content: writing through one path must be visible + // through the other. + linkedPath := filepath.Join(destRoot, "linked.txt") + if err := os.WriteFile(linkedPath, []byte("changed via the linked path"), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + originalContent, err := os.ReadFile(filepath.Join(destRoot, "original.txt")) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if string(originalContent) != "changed via the linked path" { + t.Errorf("original.txt content = %q, want the change made via linked.txt to be visible (they should be the same file)", originalContent) + } +} + +func TestApplyHardLinks_SingleMemberIsNoOp(t *testing.T) { + destRoot := t.TempDir() + if err := ApplyHardLinks(destRoot, HardLinkGroup{"solo.txt"}); err != nil { + t.Errorf("ApplyHardLinks with a single-member group returned error: %v, want nil (nothing to link)", err) + } +} diff --git a/internal/sync/hardlinks_unix.go b/internal/sync/hardlinks_unix.go new file mode 100644 index 0000000..a44abc3 --- /dev/null +++ b/internal/sync/hardlinks_unix.go @@ -0,0 +1,45 @@ +//go:build !windows + +package sync + +import ( + "os" + "syscall" +) + +// hardLinkKey identifies a file's underlying inode, uniquely across +// possibly multiple filesystems: an inode number alone is only unique +// within a single device, so two files on different devices/filesystems +// could coincidentally share an inode number without actually being +// hard-linked to each other. (Dev, Ino) together is the identity POSIX +// actually guarantees. +type hardLinkKey struct { + Dev uint64 + Ino uint64 +} + +// lookupHardLinkKey extracts the (dev, ino) identity from a Lstat'd +// os.FileInfo, the same way lookupUIDGID (uidgid_unix.go) extracts +// uid/gid - via the syscall.Stat_t underlying the info. ok is false only +// if the platform's Sys() doesn't actually return that type. +// +// stat.Dev is used as-is: it's already uint64 on the Linux target this +// project actually builds and tests for (see .github/workflows/ci.yaml). +// Some other POSIX platforms (e.g. Darwin) declare Dev as a narrower +// signed type, which would need an explicit conversion - not added here +// speculatively for a platform this project doesn't build or test +// against; that can be added if/when Darwin support is actually taken on. +func lookupHardLinkKey(info os.FileInfo) (hardLinkKey, bool) { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return hardLinkKey{}, false + } + return hardLinkKey{Dev: stat.Dev, Ino: stat.Ino}, true +} + +// HardLinksSupported reports whether this platform can detect hard links +// at all - see DetectHardLinks's doc comment for how to use this to +// surface the Windows gap explicitly (e.g. a one-time warning) rather +// than leaving callers to infer it from an empty result list, which is +// ambiguous with "this tree just has no hard links". +func HardLinksSupported() bool { return true } diff --git a/internal/sync/hardlinks_windows.go b/internal/sync/hardlinks_windows.go new file mode 100644 index 0000000..69a17fb --- /dev/null +++ b/internal/sync/hardlinks_windows.go @@ -0,0 +1,33 @@ +//go:build windows + +package sync + +import "os" + +// hardLinkKey identifies a file's underlying inode. On Windows this is +// never actually populated - see lookupHardLinkKey. +type hardLinkKey struct { + Dev uint64 + Ino uint64 +} + +// lookupHardLinkKey always reports unavailable on Windows: the (dev, ino) +// identity this package's hard-link detection relies on comes from POSIX +// os.FileInfo.Sys() (*syscall.Stat_t), which Windows doesn't provide - +// its Sys() there is *syscall.Win32FileAttributeData instead. Windows +// NTFS does have its own file-identity concept (a volume serial number +// plus a 64-bit file index, retrievable via +// GetFileInformationByHandle/BY_HANDLE_FILE_INFORMATION), and os.Link +// does work on Windows/NTFS - but wiring up that Win32-specific API is a +// meaningfully larger, separate piece of platform code, not a small +// extension of the POSIX path. Scoping it out here means hard-linked +// files on a Windows source are simply treated as independent, unlinked +// files (correct output, just missed space-saving/consistency benefit) +// rather than the sync producing wrong results. +func lookupHardLinkKey(_ os.FileInfo) (hardLinkKey, bool) { + return hardLinkKey{}, false +} + +// HardLinksSupported reports whether this platform can detect hard links +// at all - always false on Windows. See DetectHardLinks's doc comment. +func HardLinksSupported() bool { return false } diff --git a/internal/sync/specialfiles.go b/internal/sync/specialfiles.go new file mode 100644 index 0000000..518e1ed --- /dev/null +++ b/internal/sync/specialfiles.go @@ -0,0 +1,87 @@ +package sync + +import ( + "errors" + "fmt" + "io/fs" +) + +// SpecialFileType classifies a FileEntry beyond the basic +// regular-file/directory/symlink distinction Walk already exposes via +// IsDir and Mode&fs.ModeSymlink. +type SpecialFileType int + +const ( + // NotSpecial is a regular file, directory, or symlink - none of + // which ClassifySpecialFile is concerned with. + NotSpecial SpecialFileType = iota + // NamedPipe is a FIFO. The only SpecialFileType ApplySpecialFile + // actually recreates - see its doc comment for why. + NamedPipe + // Socket is a Unix domain socket file. + Socket + // CharDevice is a character device node. + CharDevice + // BlockDevice is a block device node. + BlockDevice +) + +// ClassifySpecialFile reports which SpecialFileType, if any, entry's Mode +// represents. fs.FileMode's type bits are portable Go constants with the +// same meaning regardless of host OS, even though most of them (the +// device/socket bits especially) only ever actually get set by a real +// Lstat on POSIX - so this classification itself needs no build tag, even +// though actually recreating some of these types does (see +// specialfiles_unix.go / specialfiles_windows.go). +func ClassifySpecialFile(entry FileEntry) SpecialFileType { + switch { + case entry.Mode&fs.ModeNamedPipe != 0: + return NamedPipe + case entry.Mode&fs.ModeSocket != 0: + return Socket + case entry.Mode&fs.ModeCharDevice != 0: + // Must be checked before the bare ModeDevice case below: Go's + // fs.FileMode convention sets ModeCharDevice as a modifier on + // top of ModeDevice for character devices, so a character + // device has *both* bits set, not ModeCharDevice alone. + return CharDevice + case entry.Mode&fs.ModeDevice != 0: + return BlockDevice + default: + return NotSpecial + } +} + +// ErrSpecialFileUnsupported is returned by ApplySpecialFile for socket, +// character device, and block device entries: none of these are actually +// recreated by this package. +// +// Character/block device nodes need syscall.Mknod, which requires +// CAP_MKNOD/root on Linux (and has no meaningful equivalent to attempt on +// Windows at all) - untestable without that privilege, and attempting it +// anyway would just fail for most callers while quietly calling that +// "support". A Unix domain socket file without a live listening process +// behind it also isn't meaningfully equivalent to the original; simply +// creating an inert socket-typed filesystem node isn't the same thing as +// preserving a socket. Rather than do either job partially, this is +// scoped down to detecting these types (via ClassifySpecialFile) but not +// recreating them - only named pipes (FIFOs) are actually created, since +// syscall.Mkfifo needs no elevated privilege and is fully testable. +var ErrSpecialFileUnsupported = errors.New("special file type is detected but not recreated by this package") + +// ApplySpecialFile creates destPath as the special file entry represents, +// for the one type this package actually supports creating: named pipes. +// For Socket/CharDevice/BlockDevice it returns an error wrapping +// ErrSpecialFileUnsupported. For NotSpecial (a regular file, directory, +// or symlink), it also returns an error: this function is only meaningful +// for an entry that actually is one of the special types above. +func ApplySpecialFile(destPath string, entry FileEntry) error { + switch ClassifySpecialFile(entry) { + case NamedPipe: + return applyNamedPipe(destPath, entry.Mode) + case Socket, CharDevice, BlockDevice: + return fmt.Errorf("%q: %w", entry.Path, ErrSpecialFileUnsupported) + default: + return fmt.Errorf("entry %q is not a special file (Mode=%v)", entry.Path, entry.Mode) + } +} diff --git a/internal/sync/specialfiles_test.go b/internal/sync/specialfiles_test.go new file mode 100644 index 0000000..6d69c61 --- /dev/null +++ b/internal/sync/specialfiles_test.go @@ -0,0 +1,85 @@ +package sync + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "runtime" + "testing" +) + +func TestClassifySpecialFile(t *testing.T) { + tests := []struct { + name string + mode fs.FileMode + want SpecialFileType + }{ + {"regular file", 0o644, NotSpecial}, + {"directory", fs.ModeDir | 0o755, NotSpecial}, + {"symlink", fs.ModeSymlink | 0o777, NotSpecial}, + {"named pipe", fs.ModeNamedPipe | 0o644, NamedPipe}, + {"socket", fs.ModeSocket | 0o644, Socket}, + {"char device", fs.ModeDevice | fs.ModeCharDevice | 0o644, CharDevice}, + {"block device", fs.ModeDevice | 0o644, BlockDevice}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + entry := FileEntry{Mode: tt.mode} + if got := ClassifySpecialFile(entry); got != tt.want { + t.Errorf("ClassifySpecialFile(Mode=%v) = %v, want %v", tt.mode, got, tt.want) + } + }) + } +} + +func TestApplySpecialFile_NamedPipe(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("named pipes (FIFOs) are not supported on Windows") + } + + dir := t.TempDir() + destPath := filepath.Join(dir, "pipe") + entry := FileEntry{Path: "pipe", Mode: fs.ModeNamedPipe | 0o644} + + if err := ApplySpecialFile(destPath, entry); err != nil { + t.Fatalf("ApplySpecialFile returned error: %v", err) + } + + info, err := os.Lstat(destPath) + if err != nil { + t.Fatalf("Lstat: %v", err) + } + if info.Mode()&fs.ModeNamedPipe == 0 { + t.Errorf("destPath Mode = %v, want ModeNamedPipe set", info.Mode()) + } +} + +func TestApplySpecialFile_UnsupportedTypesReturnErrSpecialFileUnsupported(t *testing.T) { + tests := []struct { + name string + mode fs.FileMode + }{ + {"socket", fs.ModeSocket | 0o644}, + {"char device", fs.ModeDevice | fs.ModeCharDevice | 0o644}, + {"block device", fs.ModeDevice | 0o644}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + entry := FileEntry{Path: "special", Mode: tt.mode} + err := ApplySpecialFile(filepath.Join(t.TempDir(), "special"), entry) + if !errors.Is(err, ErrSpecialFileUnsupported) { + t.Errorf("ApplySpecialFile(Mode=%v) error = %v, want it to wrap ErrSpecialFileUnsupported", tt.mode, err) + } + }) + } +} + +func TestApplySpecialFile_NotSpecialErrors(t *testing.T) { + entry := FileEntry{Path: "regular.txt", Mode: 0o644} + if err := ApplySpecialFile(filepath.Join(t.TempDir(), "regular.txt"), entry); err == nil { + t.Fatalf("ApplySpecialFile with a regular-file entry returned nil error, want an error") + } +} diff --git a/internal/sync/specialfiles_unix.go b/internal/sync/specialfiles_unix.go new file mode 100644 index 0000000..78d0971 --- /dev/null +++ b/internal/sync/specialfiles_unix.go @@ -0,0 +1,16 @@ +//go:build !windows + +package sync + +import ( + "io/fs" + "syscall" +) + +// applyNamedPipe creates destPath as a FIFO via syscall.Mkfifo, which +// (unlike device-node creation via Mknod) requires no elevated privilege +// on POSIX systems - any user can create a FIFO in a directory they can +// write to. +func applyNamedPipe(destPath string, mode fs.FileMode) error { + return syscall.Mkfifo(destPath, uint32(mode.Perm())) +} diff --git a/internal/sync/specialfiles_windows.go b/internal/sync/specialfiles_windows.go new file mode 100644 index 0000000..5aadfe3 --- /dev/null +++ b/internal/sync/specialfiles_windows.go @@ -0,0 +1,17 @@ +//go:build windows + +package sync + +import ( + "fmt" + "io/fs" +) + +// applyNamedPipe always fails on Windows: syscall.Mkfifo doesn't exist +// there. Windows named pipes are a distinct IPC mechanism under \\.\pipe\ +// rather than a filesystem-node type created at an arbitrary path the way +// a POSIX FIFO is, so there's no equivalent "create this path as a named +// pipe" operation to perform here at all. +func applyNamedPipe(destPath string, _ fs.FileMode) error { + return fmt.Errorf("creating a named pipe at %q: not supported on Windows", destPath) +}