diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6313b56 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf diff --git a/README.md b/README.md index 155ea32..145845b 100644 --- a/README.md +++ b/README.md @@ -4,21 +4,18 @@ An rsync-inspired file synchronization tool written in Go. ## Status -**CLI skeleton only — no sync logic yet.** The command-line interface parses -arguments and flags and prints what it received, but no files are actually -read, compared, transferred, or deleted. `internal/sync` and -`internal/transport` are currently empty packages reserved for that work. +CLI parsing and file enumeration are implemented; data transfer is not. +`internal/sync` builds a sorted file list (`sync.Walk`), but nothing calls +it yet — the CLI only echoes parsed flags — and `internal/transport` is +still empty. ## Build -Requires Go (see [go.mod](go.mod) for the version this module targets). - ```sh go build ./cmd/grsync ``` -This produces a `grsync` (or `grsync.exe` on Windows) binary in the current -directory. To run without building a binary: +Or run directly without building a binary: ```sh go run ./cmd/grsync ... [flags] @@ -30,68 +27,53 @@ go run ./cmd/grsync ... [flags] grsync ... [flags] ``` -At least one `source` and exactly one `destination` are required -(the last positional argument is always the destination), matching rsync's -own `SRC... DEST` grammar rather than restricting to a single source. +At least one `source` and exactly one `destination` are required; the last +argument is always the destination. | Flag | Shorthand | Description | |---|---|---| -| `--archive` | `-a` | archive mode (equivalent to common rsync defaults) | -| `--verbose` | `-v` | increase output verbosity | -| `--compress` | `-z` | compress file data during transfer | +| `--archive` | `-a` | archive mode | +| `--verbose` | `-v` | increase verbosity | +| `--compress` | `-z` | compress data during transfer | | `--recursive` | `-r` | recurse into directories | -| `--dry-run` | `-n` | show what would be transferred without transferring | +| `--dirs` | `-d` | list directories without recursing into them (implied by `-r`) | +| `--dry-run` | `-n` | show what would be transferred | | `--delete` | | delete extraneous files from destination | | `--progress` | | show progress during transfer | -| `--exclude PATTERN` | | exclude files matching PATTERN (repeatable) | -| `--include PATTERN` | | include files matching PATTERN (repeatable) | -| `--filter RULE` | | add a file-filtering RULE (repeatable) | +| `--exclude PATTERN` | | exclude matching files (repeatable) | +| `--include PATTERN` | | include matching files (repeatable) | +| `--filter RULE` | | add a filter rule (repeatable) | -`--exclude`, `--include`, and `--filter` are collected into a single ordered -rule list, not three independent lists — their relative order on the command -line is preserved regardless of which of the three flags produced each rule. -This matches rsync's first-match-wins filter semantics, where rule order is -significant. +`--exclude`/`--include`/`--filter` share one ordered rule list — their +relative order on the command line is preserved, matching rsync's +first-match-wins semantics. -At this stage, running the command only echoes the parsed values back — it -does not transfer any files: +## File Enumeration -```sh -$ go run ./cmd/grsync ./src1 ./src2 ./dst -av --exclude "*.log" --include "keep.log" -sources: [./src1 ./src2] -destination: ./dst -archive: true -verbose: true -... -filters: exclude:*.log, include:keep.log -``` +`sync.Walk`, in the `internal/sync` package, recursively lists a source tree +into a sorted `[]FileEntry` (path, size, mtime, mode, uid/gid, symlink +target). Symlinks are captured via `Lstat`, never followed. + +`-r`/`-d` control how far it descends: + +| Recursive | Dirs | Result | +|---|---|---| +| off | off | directories skipped entirely | +| off | on | directories listed, not descended into | +| on | any | full recursion | + +On Windows, `UID`/`GID` are always `0` — there's no POSIX ownership concept +to read, so `0` means "unavailable," not a real value. ## Architecture -grsync follows the same conceptual split as upstream -[rsync](https://rsync.samba.org/): a **sync layer** that decides *what* needs -to change (comparing source and destination file trees, applying -include/exclude/filter rules) and a **transport layer** that handles *how* -data actually moves (local copies today, with room for remote transports -later). The codebase mirrors that split: - -- `cmd/grsync` — CLI entrypoint (binary wiring only). -- `internal/cli` — argument/flag parsing (built on - [spf13/cobra](https://github.com/spf13/cobra)); owns no sync or transport - logic. -- `internal/sync` — (placeholder) will own comparison and change-planning - logic, analogous to rsync's file-list generation and delta algorithm. -- `internal/transport` — (placeholder) will own how bytes are actually moved - between source and destination. - -grsync's goal is full feature parity with upstream rsync, not just -similarity of CLI flags — including protocol- and format-level -interoperability with the C implementation where specified (for example, -batch mode's `--write-batch`/`--read-batch` file format is required to be -interoperable with the C rsync implementation). None of that is implemented -yet: the wire protocol, delta-transfer algorithm, and batch file format are -still to come in later tickets. This README will be updated as each piece -lands. +- `cmd/grsync` — CLI entrypoint. +- `internal/cli` — flag/argument parsing (built on cobra). +- `internal/sync` — file-list generation today; comparison/delta logic later. +- `internal/transport` — (placeholder) data movement, local and remote. + +Goal: full feature parity with upstream rsync, including protocol/format +interoperability where specified (e.g. batch mode's file format). ## License diff --git a/internal/cli/root.go b/internal/cli/root.go index 3ffcf5c..6e80305 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -42,6 +42,7 @@ type options struct { verbose bool compress bool recursive bool + dirs bool dryRun bool delete bool progress bool @@ -95,6 +96,7 @@ func NewRootCmd() *cobra.Command { flags.BoolVarP(&opts.verbose, "verbose", "v", false, "increase output verbosity") flags.BoolVarP(&opts.compress, "compress", "z", false, "compress file data during transfer") flags.BoolVarP(&opts.recursive, "recursive", "r", false, "recurse into directories") + flags.BoolVarP(&opts.dirs, "dirs", "d", false, "include directories themselves without recursing into their contents (implied by --recursive)") flags.BoolVarP(&opts.dryRun, "dry-run", "n", false, "show what would be transferred without transferring") flags.BoolVar(&opts.delete, "delete", false, "delete extraneous files from destination") flags.BoolVar(&opts.progress, "progress", false, "show progress during transfer") @@ -131,12 +133,13 @@ func run(cmd *cobra.Command, sources []string, destination string, opts *options "verbose: %t\n"+ "compress: %t\n"+ "recursive: %t\n"+ + "dirs: %t\n"+ "dry-run: %t\n"+ "delete: %t\n"+ "progress: %t\n"+ "filters: %s\n", sources, destination, - opts.archive, opts.verbose, opts.compress, opts.recursive, opts.dryRun, + opts.archive, opts.verbose, opts.compress, opts.recursive, opts.dirs, opts.dryRun, opts.delete, opts.progress, rules.String(), ) diff --git a/internal/sync/.gitkeep b/internal/sync/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/internal/sync/uidgid_unix.go b/internal/sync/uidgid_unix.go new file mode 100644 index 0000000..77cc408 --- /dev/null +++ b/internal/sync/uidgid_unix.go @@ -0,0 +1,21 @@ +//go:build !windows + +package sync + +import ( + "os" + "syscall" +) + +// lookupUIDGID extracts the owning user/group ID from a Lstat'd +// os.FileInfo. On POSIX platforms this is available via the +// syscall.Stat_t underlying the info; ok is false only if the platform's +// Sys() doesn't actually return that type (unexpected, but Sys() is +// documented as platform-dependent, so it's not guaranteed). +func lookupUIDGID(info os.FileInfo) (uid, gid uint32, ok bool) { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return 0, 0, false + } + return stat.Uid, stat.Gid, true +} diff --git a/internal/sync/uidgid_windows.go b/internal/sync/uidgid_windows.go new file mode 100644 index 0000000..afa54c7 --- /dev/null +++ b/internal/sync/uidgid_windows.go @@ -0,0 +1,14 @@ +//go:build windows + +package sync + +import "os" + +// lookupUIDGID always reports unavailable on Windows: os.FileInfo.Sys() +// there returns *syscall.Win32FileAttributeData, which carries no POSIX +// uid/gid concept at all (Windows uses SIDs and ACLs instead). Callers must +// treat ok == false as "not populated", not as "owned by uid/gid 0" — +// zero is not a meaningful default here. +func lookupUIDGID(_ os.FileInfo) (uid, gid uint32, ok bool) { + return 0, 0, false +} diff --git a/internal/sync/walk.go b/internal/sync/walk.go new file mode 100644 index 0000000..5574b79 --- /dev/null +++ b/internal/sync/walk.go @@ -0,0 +1,142 @@ +// Package sync builds the file list ("flist") that grsync compares between +// source and destination before any data transfer happens — the same +// planning phase upstream rsync performs before its delta algorithm runs. +package sync + +import ( + "io/fs" + "os" + "path/filepath" + "sort" + "time" +) + +// FileEntry describes a single file, directory, or symlink discovered under +// a source root. Path is always relative to that root, using "/" as the +// separator regardless of host OS — rsync's wire protocol and file lists +// are always "/"-separated, and grsync targets protocol-level +// interoperability, so paths are normalized at collection time rather than +// left OS-native and converted later. +type FileEntry struct { + Path string + Size int64 + ModTime time.Time + Mode fs.FileMode + UID uint32 + GID uint32 + // OwnershipAvailable reports whether UID/GID were actually populated. + // A real uid/gid of 0 (root) is a valid value, so callers must check + // this rather than treating a zero UID/GID as "unavailable" — see + // uidgid_windows.go, where it is always false. + OwnershipAvailable bool + LinkTarget string + IsDir bool +} + +// WalkOptions controls how far Walk descends, mirroring rsync's own +// -r/--recursive and -d/--dirs flags: +// +// - Recursive=false, Dirs=false (rsync's default with neither flag): +// directories are skipped entirely — not listed, not descended into. +// Only regular files/symlinks directly under root are collected. +// - Recursive=false, Dirs=true (-d): directories are listed (so they can +// be created on the receiving end) but their contents are not — Walk +// does not descend into them. +// - Recursive=true (-r): full recursion into every subdirectory, +// regardless of Dirs — this matches rsync, where -r makes -d redundant. +type WalkOptions struct { + Recursive bool + Dirs bool +} + +// Walk collects a FileEntry for every entry found under root, not including +// root itself, subject to opts. Paths are relative to root and +// "/"-separated. +func Walk(root string, opts WalkOptions) ([]FileEntry, error) { + var entries []FileEntry + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if path == root { + // The root itself isn't part of its own file list. + return nil + } + + if !opts.Recursive && d.IsDir() && !opts.Dirs { + // Neither -r nor -d: rsync skips directories entirely rather + // than creating them empty, so this one is neither listed nor + // descended into. + return filepath.SkipDir + } + + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + + // os.Lstat, not os.Stat: a symlink must be reported as itself, not + // resolved to whatever it points at. WalkDir's own DirEntry is + // already lstat-derived (it never follows symlinks when deciding + // what to recurse into), but calling Lstat explicitly here makes + // that guarantee obvious at the call site rather than implicit in + // WalkDir's internals. + info, err := os.Lstat(path) + if err != nil { + return err + } + + entry := FileEntry{ + Path: filepath.ToSlash(rel), + Size: info.Size(), + ModTime: info.ModTime(), + Mode: info.Mode(), + IsDir: info.IsDir(), + } + + // lookupUIDGID is platform-specific (see uidgid_unix.go / + // uidgid_windows.go): on Windows it always reports unavailable, + // leaving UID/GID at their zero value. OwnershipAvailable carries + // that ok flag through so callers can't mistake the zero value for + // a real uid/gid of 0 (root) — see uidgid_windows.go for why. + entry.UID, entry.GID, entry.OwnershipAvailable = lookupUIDGID(info) + + // info.Mode()&fs.ModeSymlink is only ever set by Lstat (Stat + // resolves through it), which is exactly why Lstat was required + // above: this branch would never be reachable otherwise. + if info.Mode()&fs.ModeSymlink != 0 { + target, err := os.Readlink(path) + if err != nil { + return err + } + entry.LinkTarget = target + } + + entries = append(entries, entry) + + if !opts.Recursive && d.IsDir() { + // Whether or not -d caused this directory to be listed above, + // without -r its contents are never descended into. + return filepath.SkipDir + } + + return nil + }) + if err != nil { + return nil, err + } + + // filepath.WalkDir only guarantees lexicographic order within each + // directory, not across the whole tree (a directory's contents are + // interleaved with sibling entries in traversal order, not sorted + // globally). Sorting the flattened list by full path afterward is what + // actually gives deterministic, rsync-comparable ordering; plain Go + // string comparison is byte-wise, matching C's strcmp, which is what + // rsync's own pathname sort relies on. + sort.Slice(entries, func(i, j int) bool { + return entries[i].Path < entries[j].Path + }) + + return entries, nil +} diff --git a/internal/sync/walk_test.go b/internal/sync/walk_test.go new file mode 100644 index 0000000..d57d303 --- /dev/null +++ b/internal/sync/walk_test.go @@ -0,0 +1,257 @@ +package sync + +import ( + "os" + "path/filepath" + "reflect" + "runtime" + "sort" + "strings" + "testing" +) + +func TestWalk_NestedDirectories(t *testing.T) { + root := t.TempDir() + + mustMkdirAll(t, filepath.Join(root, "a", "b")) + mustWriteFile(t, filepath.Join(root, "top.txt"), "top") + mustWriteFile(t, filepath.Join(root, "a", "mid.txt"), "mid") + mustWriteFile(t, filepath.Join(root, "a", "b", "deep.txt"), "deep") + + entries, err := Walk(root, WalkOptions{Recursive: true}) + if err != nil { + t.Fatalf("Walk returned error: %v", err) + } + + want := map[string]bool{ + "a": true, + "a/b": true, + "a/b/deep.txt": false, + "a/mid.txt": false, + "top.txt": false, + } + + got := make(map[string]bool, len(entries)) + for _, e := range entries { + got[e.Path] = e.IsDir + } + + if len(got) != len(want) { + t.Fatalf("got %d entries, want %d: %v", len(got), len(want), got) + } + for path, wantIsDir := range want { + gotIsDir, ok := got[path] + if !ok { + t.Errorf("missing entry for %q", path) + continue + } + if gotIsDir != wantIsDir { + t.Errorf("entry %q: IsDir = %v, want %v", path, gotIsDir, wantIsDir) + } + } +} + +func TestWalk_Symlink(t *testing.T) { + root := t.TempDir() + + mustWriteFile(t, filepath.Join(root, "real.txt"), "hello") + + linkPath := filepath.Join(root, "link.txt") + if err := os.Symlink("real.txt", linkPath); err != nil { + // Creating symlinks on Windows requires Developer Mode or an + // elevated process; treat that as "unsupported here", not a + // failure, rather than forcing every environment to be elevated. + t.Skipf("symlink creation unsupported in this environment: %v", err) + } + + entries, err := Walk(root, WalkOptions{Recursive: true}) + if err != nil { + t.Fatalf("Walk returned error: %v", err) + } + + var link *FileEntry + for i := range entries { + if entries[i].Path == "link.txt" { + link = &entries[i] + } + } + if link == nil { + t.Fatalf("link.txt not found in entries: %v", entries) + } + + if link.Mode&os.ModeSymlink == 0 { + t.Errorf("link.txt: Mode = %v, want ModeSymlink set", link.Mode) + } + if link.IsDir { + t.Errorf("link.txt: IsDir = true, want false") + } + if link.LinkTarget != "real.txt" { + t.Errorf("link.txt: LinkTarget = %q, want %q", link.LinkTarget, "real.txt") + } + + // The symlink must not have been followed: its own Size should reflect + // the length of the link text, not the 5-byte target file's contents. + if link.Size == 5 { + t.Errorf("link.txt: Size = %d looks like the resolved target's size, want the symlink's own size", link.Size) + } +} + +func TestWalk_RecursiveAndDirsFlags(t *testing.T) { + root := t.TempDir() + + mustMkdirAll(t, filepath.Join(root, "sub")) + mustWriteFile(t, filepath.Join(root, "top.txt"), "top") + mustWriteFile(t, filepath.Join(root, "sub", "nested.txt"), "nested") + + paths := func(entries []FileEntry) []string { + got := make([]string, len(entries)) + for i, e := range entries { + got[i] = e.Path + } + sort.Strings(got) + return got + } + + t.Run("neither flag: directories skipped entirely", func(t *testing.T) { + entries, err := Walk(root, WalkOptions{Recursive: false, Dirs: false}) + if err != nil { + t.Fatalf("Walk returned error: %v", err) + } + want := []string{"top.txt"} + if got := paths(entries); !reflect.DeepEqual(got, want) { + t.Errorf("paths = %v, want %v", got, want) + } + }) + + t.Run("dirs only: directory listed but not descended into", func(t *testing.T) { + entries, err := Walk(root, WalkOptions{Recursive: false, Dirs: true}) + if err != nil { + t.Fatalf("Walk returned error: %v", err) + } + want := []string{"sub", "top.txt"} + if got := paths(entries); !reflect.DeepEqual(got, want) { + t.Errorf("paths = %v, want %v", got, want) + } + }) + + t.Run("recursive with dirs=false: full traversal, dirs listed anyway", func(t *testing.T) { + entries, err := Walk(root, WalkOptions{Recursive: true, Dirs: false}) + if err != nil { + t.Fatalf("Walk returned error: %v", err) + } + want := []string{"sub", "sub/nested.txt", "top.txt"} + if got := paths(entries); !reflect.DeepEqual(got, want) { + t.Errorf("paths = %v, want %v", got, want) + } + }) + + t.Run("recursive with dirs=true: identical to dirs=false", func(t *testing.T) { + entries, err := Walk(root, WalkOptions{Recursive: true, Dirs: true}) + if err != nil { + t.Fatalf("Walk returned error: %v", err) + } + want := []string{"sub", "sub/nested.txt", "top.txt"} + if got := paths(entries); !reflect.DeepEqual(got, want) { + t.Errorf("paths = %v, want %v", got, want) + } + }) +} + +func TestWalk_OwnershipAvailable(t *testing.T) { + root := t.TempDir() + mustWriteFile(t, filepath.Join(root, "file.txt"), "content") + + entries, err := Walk(root, WalkOptions{Recursive: true}) + if err != nil { + t.Fatalf("Walk returned error: %v", err) + } + if len(entries) != 1 { + t.Fatalf("got %d entries, want 1: %v", len(entries), entries) + } + entry := entries[0] + + // Windows has no POSIX uid/gid concept; every other platform this repo + // builds for does (see uidgid_unix.go / uidgid_windows.go). + wantAvailable := runtime.GOOS != "windows" + if entry.OwnershipAvailable != wantAvailable { + t.Errorf("OwnershipAvailable = %v, want %v (GOOS=%s)", entry.OwnershipAvailable, wantAvailable, runtime.GOOS) + } + if runtime.GOOS == "windows" && (entry.UID != 0 || entry.GID != 0) { + t.Errorf("UID/GID = %d/%d, want 0/0 when OwnershipAvailable is false", entry.UID, entry.GID) + } +} + +func TestWalk_VaryingFileSizes(t *testing.T) { + root := t.TempDir() + + sizes := map[string]int{ + "empty.txt": 0, + "small.txt": 3, + "large.txt": 10_000, + } + for name, size := range sizes { + mustWriteFile(t, filepath.Join(root, name), strings.Repeat("x", size)) + } + + entries, err := Walk(root, WalkOptions{Recursive: true}) + if err != nil { + t.Fatalf("Walk returned error: %v", err) + } + + got := make(map[string]int64, len(entries)) + for _, e := range entries { + got[e.Path] = e.Size + } + + for name, wantSize := range sizes { + gotSize, ok := got[name] + if !ok { + t.Errorf("missing entry for %q", name) + continue + } + if gotSize != int64(wantSize) { + t.Errorf("%q: Size = %d, want %d", name, gotSize, wantSize) + } + } +} + +func TestWalk_DeterministicSortOrder(t *testing.T) { + root := t.TempDir() + + // Create files in an order that does not match their sorted order, so a + // pass would only happen if Walk actually sorts rather than incidentally + // returning creation order or directory-read order. + for _, name := range []string{"charlie.txt", "alpha.txt", "bravo.txt", "delta.txt"} { + mustWriteFile(t, filepath.Join(root, name), name) + } + + want := []string{"alpha.txt", "bravo.txt", "charlie.txt", "delta.txt"} + + for i := 0; i < 5; i++ { + entries, err := Walk(root, WalkOptions{Recursive: true}) + if err != nil { + t.Fatalf("run %d: Walk returned error: %v", i, err) + } + got := make([]string, len(entries)) + for j, e := range entries { + got[j] = e.Path + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("run %d: order = %v, want %v", i, got, want) + } + } +} + +func mustMkdirAll(t *testing.T, path string) { + t.Helper() + if err := os.MkdirAll(path, 0o755); err != nil { + t.Fatalf("MkdirAll(%q): %v", path, err) + } +} + +func mustWriteFile(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("WriteFile(%q): %v", path, err) + } +}