diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 9af7014..861b5a1 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -13,11 +13,17 @@ jobs: steps: - uses: actions/checkout@v3 - - name: OS Updates - run: "sudo apt update" + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: false + - name: Unit Tests + run: go test -race ./... + - name: Vet + run: go vet ./... - name: Build Bins run: ./scripts/bins.sh - name: E2E - run: ETE_DIR=$HOME ./scripts/e2e.sh + run: ./scripts/e2e.sh - name: Print SHA run: cat scnnr_bins.zip.sha256 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2b41031..0dd5215 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,11 +13,11 @@ release: script: - export CI=true - export VERSION=$CI_PIPELINE_IID - - apk add --no-cache bash - - ETE_DIR=$HOME ./scripts/e2e.sh - - go run cmd/pack/main.go - - go run main.go -m fff -k $(go run cmd/checksum/main.go) -p . - - cat scnnr_bins.zip.sha256 + - apk add --no-cache bash build-base + - go test -race ./... + - go vet ./... + - ./scripts/e2e.sh + - ./scripts/bins.sh artifacts: paths: diff --git a/README.md b/README.md index 04f1e3b..55aea43 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ - [wget](#wget) - [Docker](#docker) - [Performance (scn)](#performance-scn) +- [Tests](#tests) @@ -35,13 +36,15 @@ Scans files (by extension) in a given directory for a keyword. Can be any file, Prints out a `\n` delimited string of each file (filepath in artifact) containing one of the keywords. -Max file descriptors is set to 1024 (linux default) in scnnr scn mode. +Content matching uses concurrent batches of up to 1024 files. Match output follows concurrent processing order. -Has 3 additional modes: FileSizeFinder (find a file above a certain size), FileNameFinder (fuzzy find files with certain keywords in the filename), and FileFingerprintFinder (find files based on their SHA2-256 hash) +Has 3 additional modes: FileSizeFinder (find a file at or above a certain size), FileNameFinder (fuzzy find files with certain keywords in the filename), and FileFingerprintFinder (find files based on their SHA2-256 hash) `scn` mode is the default -_Caveat: will not throw an error if a file cannot be read due to permissions. It is assumed that you know what files you can read/have access to. This allows for a clean/parseable output_ +In scn mode, errors walking directories stop the scan. Errors opening or reading files selected for content scanning are fatal. + +Finder modes panic if the root directory cannot be read. Files with unavailable metadata are skipped, and errors reading child directories are ignored. Failed directory metadata reads can still cause a panic. ### Help @@ -150,58 +153,38 @@ scnnr -e .md -d . -k cache= _using -k_ +These examples use the files in `pkg/testdata/tree`. Matching paths can appear in a different order. + ```bash -$ scnnr -d . -e .md,.go -k fileData,cache -README.md -cmd/scanner.go +$ scnnr -d pkg/testdata/tree -e .txt,.go -k needle,token -xd ignored +pkg/testdata/tree/alpha.txt +pkg/testdata/tree/nested/beta.go ``` ### Keywords with line numbers _using -l_ +Prints the first occurrence of each keyword on each matching line. With multiple keywords, the matching keyword is appended to each result. + ```bash -$ scnnr -d . -e .md,.go -k fileData,cache -l -scnnr_bins/README.md:117:cache -scnnr_bins/README.md:122:cache -scnnr_bins/README.md:129:cache -scnnr_bins/README.md:135:fileData -scnnr_bins/README.md:135:cache -README.md:123:cache -README.md:128:cache -README.md:135:cache -README.md:141:fileData -README.md:141:cache -scnnr_bins/README.md:377:cache -scnnr_bins/README.md:387:cache -README.md:383:cache -README.md:393:cache -pkg/scanner.go:215:fileData -pkg/scanner.go:222:fileData +$ scnnr -d pkg/testdata/tree -e .txt,.go -k needle,token -l -xd ignored +pkg/testdata/tree/alpha.txt:1:needle +pkg/testdata/tree/alpha.txt:2:token +pkg/testdata/tree/nested/beta.go:2:needle ``` ### Keywords with both line numbers and column numbers _using -c_ +Line and column numbers start at 1. Columns count bytes. + ```bash -$ scnnr -d . -e .md,.go -k fileData,cache -c -scnnr_bins/README.md:117:53:cache -scnnr_bins/README.md:122:29:cache -scnnr_bins/README.md:129:22:cache -scnnr_bins/README.md:135:28:fileData -scnnr_bins/README.md:135:37:cache -scnnr_bins/README.md:377:36:cache -scnnr_bins/README.md:387:40:cache -pkg/scanner.go:215:9:fileData -pkg/scanner.go:222:26:fileData -README.md:123:53:cache -README.md:128:29:cache -README.md:135:22:cache -README.md:141:28:fileData -README.md:141:37:cache -README.md:383:36:cache -README.md:393:40:cache +$ scnnr -d pkg/testdata/tree -e .txt,.go -k needle,token -c -xd ignored +pkg/testdata/tree/alpha.txt:1:7:needle +pkg/testdata/tree/alpha.txt:2:8:token +pkg/testdata/tree/nested/beta.go:2:4:needle ``` ### Keywords while excluding dirs and file extensions @@ -209,32 +192,10 @@ README.md:393:40:cache _using -xd and -xe_ ```bash -go run main.go -k main,let,for -p . -c -xd ".git,scnnr_bins,pkg,cmd,.zip" -xe ".yml,.sh,.md" -.editorconfig:11:21:let -.gitignore:3:1:main -.gitignore:4:1:main -.dockerignore:3:1:main -.dockerignore:4:1:main -Dockerfile:14:21:main -Dockerfile:15:25:main -main.go:25:9:main -main.go:36:6:main -main.go:46:8:for -main.go:47:8:for -main.go:48:8:for -main.go:49:8:for -main.go:70:65:for -main.go:78:57:for -main.go:84:23:for -main.go:89:34:for -main.go:125:3:for -main.go:129:3:for -main.go:172:3:for -main.go:179:42:for -main.go:186:3:for -scnnr_bins.zip:2053:93:for -scnnr_bins.zip:26325:8:let -scnnr_bins.zip:32123:53:let +$ scnnr -d pkg/testdata/tree -k needle,token -c -xd ignored -xe .log +pkg/testdata/tree/alpha.txt:1:7:needle +pkg/testdata/tree/alpha.txt:2:8:token +pkg/testdata/tree/nested/beta.go:2:4:needle ``` # File Name Finder (NameFinder) (fnf) @@ -277,9 +238,8 @@ scnnr -m fnf -f DEFCON,Finance,Tax,Return -p /tmp,/usr,/etc,$HOME/Documents Example use to find a file with a known hash: ```bash -$ known_hash="de4f51f97fa690026e225798ff294cd182b93847aaa46fe1e32b848eb9e985bd" -$ go run main.go -m fff -d $HOME/Documents -k $known_hash -/home/selfup/Documents//dotfiles/mac/.bash_profile +$ scnnr -m fff -d pkg/testdata/tree -k e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +pkg/testdata/tree/empty.txt ``` Please refer to the release notes for more details: @@ -296,18 +256,22 @@ Gitlab: https://gitlab.com/selfup/scnnr/-/releases/v1.1.8 size: 1MB,10MB,100MB,1GB,10GB,100GB,1TB ``` -Example use to find any file over 100MB in E:/LotsOfStuff +Example use to find any file at least 100MB in E:/LotsOfStuff ```bash scnnr -m fsf -s 100MB -d E:/LotsOfStuff ``` +Sizes use decimal bytes: 1MB is 1,000,000 bytes. Files equal to the threshold are included. + # Back to Scnnr ### Using the package github.com/selfup/scnnr/pkg ```go import ( + "log" + scnnr "github.com/selfup/scnnr/pkg" ) @@ -328,28 +292,65 @@ if err != nil { } ``` +`Scan` prints to stdout and appends to the scanner's result fields. Reusing a scanner also reprocesses its accumulated candidate files. Use `[]string{""}` for no keywords or all extensions; nil or empty keyword and extension slices can panic. + +With position tracking, each file parse records the first occurrence of each keyword per line. Without position tracking, each parse records at most one matching path and continues reading the file. Duplicate extensions and repeated scans can produce duplicate results. + +The current batching code can omit files: a full final 1024-file batch is not parsed, and the file that flushes a full batch is skipped. Listing paths without keywords does not use this batching. + +`Scan` returns directory traversal errors. Errors opening or reading a file call `log.Fatal`. Invalid regexes can panic when a line is examined. `ScanTo(writer)` directs the same output to a writer. Both methods ignore output write errors. + +The working tree also includes `Search`, `ScanTo`, `WriteResults`, and `ParseSize`. These methods and helpers are absent from v1.3.0. + +`Search` returns results without printing or changing the scanner's result fields. It removes empty strings from keyword and extension lists, returns read and regex errors, and orders results by file path. With `ShowLines` or `ShowCols` enabled, it requires a nonempty keyword and collects all non-overlapping occurrences of each keyword. Without those flags, it stops reading each file after its first match. + +With the scanner above and a writer: + +```go +scanner.ShowCols = true + +result, err := scanner.Search() + +if err != nil { + log.Fatal(err) +} + +err = scnnr.WriteResults(writer, result, scnnr.OutputOptions{ + ShowCols: true, + ShowKeywords: len(scanner.Keywords) > 1, +}) + +if err != nil { + log.Fatal(err) +} +``` + +`WriteResults` returns output write errors. + +All three finder methods use `Scan(directory string)` with no return value and append results to `Files`. NameFinder and SizeFinder construct paths as `directory + directory + filename`. FingerprintFinder uses `directory + Direction + filename`, matches digest substrings, and ignores file read errors. + +`NewFileSizeFinder` panics on invalid sizes. `scnnr.ParseSize` validates the same thresholds and returns an error for an invalid size. + ## Regex ### Using Regex Patterns -`scnnr -e ".js" -d "artifact" -k "cons?" -r T > .results` +`scnnr -e ".js" -d "artifact" -k "cons?" -r=T > .results` According to the godoc for `flag.BoolVar` you can use a few things for boolean flag values: `t, T, 1, true, True, TRUE` -``` -scnnr $ time scnnr -r 1 -d artifact -e .js,.ts,.md -k 'cons*,let?,var?, impor*, expor*' > .results - -real 0m0.748s -user 0m2.398s -sys 0m0.311s +```bash +scnnr -r=1 -d artifact -e .js,.ts,.md -k 'cons*,let?,var?, impor*, expor*' > .results ``` ### Using the package github.com/selfup/scnnr/pkg ```go import ( + "log" + scnnr "github.com/selfup/scnnr/pkg" ) @@ -469,7 +470,16 @@ scnnr.exe Use of goroutines, buffers, streams, mutexes, and simple checks. -Memory in the following example never went above 5.5MB for the entire program. +The following are historical measurements from a JavaScript project. They have not been rerun for the current working tree. The original memory measurement reported usage below 5.5MB. + +To measure the current build, compile once and time the binary using your own directory: + +```bash +go build -o /tmp/scnnr-bench main.go +time /tmp/scnnr-bench -d path/to/files -e .js,.md -k cache +``` + +Timing `go run` also includes build and launch overhead. Timings depend on the files, filesystem cache, and machine used. No matches on 33k files after `npm i` for a JavaScript project as the `artifact`: @@ -520,3 +530,17 @@ sys 0m0.351s $ ls -lahg .results -rw-r--r-- 1 selfup 1.2M Jul 21 00:57 .results ``` + +## Tests + +```bash +go test -race ./... +go vet ./... +./scripts/e2e.sh +``` + +Tests cover reader and writer failures, build execution, filesystem fixtures, matching, accumulated results, and CLI exit statuses. Compile-time checks cover the existing finder and scanner signatures and selected helpers. + +The E2E script creates its own temporary tree and checks output for all four modes. Set `ETE_DIR` to choose the parent directory for those fixtures. + +`go vet` checks for suspicious code patterns. It does not detect unused function or method declarations. diff --git a/cmd/build/main.go b/cmd/build/main.go index 731c899..d7c49fb 100644 --- a/cmd/build/main.go +++ b/cmd/build/main.go @@ -1,146 +1,89 @@ package main import ( + "fmt" + "io" "log" "os" "os/exec" + "path/filepath" + "slices" + "strings" ) -const BIN = "/scnnr" -const EXE = "/scnnr.exe" - -const LINUX_ROOT = "scnnr_bins/linux" -const LINUX_INTEL = LINUX_ROOT + "/intel" -const LINUX_INTEL_BIN = LINUX_INTEL + BIN -const LINUX_ARM = LINUX_ROOT + "/arm" -const LINUX_ARM_BIN = LINUX_ARM + BIN - -const MAC_ROOT = "scnnr_bins/mac" -const MAC_INTEL = MAC_ROOT + "/intel" -const MAC_INTEL_BIN = MAC_INTEL + BIN -const MAC_ARM = MAC_ROOT + "/arm" -const MAC_ARM_BIN = MAC_ARM + BIN - -const WINDOWS = "scnnr_bins/windows" -const WINDOWS_BIN = WINDOWS + EXE - -func main() { - os.MkdirAll(MAC_ROOT, os.ModePerm) - os.MkdirAll(MAC_INTEL, os.ModePerm) - os.MkdirAll(MAC_ARM, os.ModePerm) - - os.MkdirAll(LINUX_ROOT, os.ModePerm) - os.MkdirAll(LINUX_INTEL, os.ModePerm) - os.MkdirAll(LINUX_ARM, os.ModePerm) - - os.MkdirAll(WINDOWS, os.ModePerm) - - if fileExists("main") { - deleteFile("main") - } - - if fileExists("main.exe") { - deleteFile("main.exe") - } - - compile("darwin", "amd64") - mv("main", MAC_INTEL_BIN) - - compile("darwin", "arm64") - mv("main", MAC_ARM_BIN) - - compile("linux", "amd64") - mv("main", LINUX_INTEL_BIN) - - compile("linux", "arm64") - mv("main", LINUX_ARM_BIN) - - compile("windows", "386") - mv("main.exe", WINDOWS_BIN) - - nixFiles := []string{LINUX_ARM_BIN, LINUX_INTEL_BIN, MAC_ARM_BIN, MAC_INTEL_BIN} - - chmod(nixFiles, 0777) - - ci := os.Getenv("CI") - version := os.Getenv("VERSION") - - if ci != "" && version != "" { - versionNumber := []byte(version + "\n") - - err := os.WriteFile("scnnr_bins/version", versionNumber, 0777) - if err != nil { - log.Fatal(err) - } - } - - cp("README.md", "scnnr_bins/README.md") - cp("LICENSE", "scnnr_bins/LICENSE") +type buildTarget struct { + goos, arch, output string } -func compile(goos string, arch string) { - setEnv("CGO_ENABLED", "0") - setEnv("GOOS", goos) - setEnv("GOARCH", arch) - - cmd := exec.Command("go", "build", "main.go") - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - err := cmd.Run() - if err != nil { - log.Fatal(err) - } +var targets = []buildTarget{ + {"darwin", "amd64", "scnnr_bins/mac/intel/scnnr"}, + {"darwin", "arm64", "scnnr_bins/mac/arm/scnnr"}, + {"linux", "amd64", "scnnr_bins/linux/intel/scnnr"}, + {"linux", "arm64", "scnnr_bins/linux/arm/scnnr"}, + {"windows", "386", "scnnr_bins/windows/scnnr.exe"}, } -func mv(old string, new string) { - err := os.Rename(old, new) - if err != nil { +func main() { + if err := buildAll(".", os.Environ(), os.Stdout, os.Stderr, (*exec.Cmd).Run); err != nil { log.Fatal(err) } } -func cp(source string, destination string) { - input, err := os.ReadFile(source) - if err != nil { - log.Fatal(err) - } - - err = os.WriteFile(destination, input, 0777) - if err != nil { - log.Fatal(err) +// buildAll accepts command execution and environment as inputs so build plans +// and failures can be tested without compiling binaries or changing os.Environ. +func buildAll(root string, environment []string, stdout, stderr io.Writer, execute func(*exec.Cmd) error) error { + for _, target := range targets { + output := filepath.Join(root, filepath.FromSlash(target.output)) + if err := os.MkdirAll(filepath.Dir(output), 0755); err != nil { + return err + } + cmd := exec.Command("go", "build", "-o", filepath.FromSlash(target.output), "main.go") + cmd.Dir = root + cmd.Env = targetEnv(environment, target) + cmd.Stdin, cmd.Stdout, cmd.Stderr = os.Stdin, stdout, stderr + if err := execute(cmd); err != nil { + return fmt.Errorf("build %s/%s: %w", target.goos, target.arch, err) + } + if target.goos != "windows" { + if err := os.Chmod(output, 0755); err != nil { + return err + } + } } -} - -func setEnv(key string, value string) { - err := os.Setenv(key, value) - if err != nil { - log.Fatal(err) + if envValue(environment, "CI") != "" && envValue(environment, "VERSION") != "" { + if err := os.WriteFile(filepath.Join(root, "scnnr_bins/version"), []byte(envValue(environment, "VERSION")+"\n"), 0644); err != nil { + return err + } } -} - -func fileExists(filename string) bool { - info, err := os.Stat(filename) - if os.IsNotExist(err) { - return false + for _, name := range []string{"README.md", "LICENSE"} { + data, err := os.ReadFile(filepath.Join(root, name)) + if err != nil { + return err + } + if err := os.WriteFile(filepath.Join(root, "scnnr_bins", name), data, 0644); err != nil { + return err + } } - - return !info.IsDir() + return nil } -func chmod(files []string, mode os.FileMode) { - for _, file := range files { - err := os.Chmod(file, mode) - if err != nil { - log.Fatal(err) +func targetEnv(environment []string, target buildTarget) []string { + var result []string + for _, entry := range environment { + key, _, _ := strings.Cut(entry, "=") + if key != "CGO_ENABLED" && key != "GOOS" && key != "GOARCH" { + result = append(result, entry) } } + return append(result, "CGO_ENABLED=0", "GOOS="+target.goos, "GOARCH="+target.arch) } -func deleteFile(path string) { - err := os.Remove(path) - if err != nil { - log.Fatal(err) +func envValue(environment []string, name string) string { + for _, e := range slices.Backward(environment) { + key, value, _ := strings.Cut(e, "=") + if key == name { + return value + } } + return "" } diff --git a/cmd/build/main_test.go b/cmd/build/main_test.go new file mode 100644 index 0000000..29c5477 --- /dev/null +++ b/cmd/build/main_test.go @@ -0,0 +1,149 @@ +package main + +import ( + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "reflect" + "runtime" + "testing" +) + +func buildFixture(t *testing.T) string { + t.Helper() + root := t.TempDir() + for name, content := range map[string]string{"README.md": "readme\n", "LICENSE": "license\n"} { + if err := os.WriteFile(filepath.Join(root, name), []byte(content), 0600); err != nil { + t.Fatal(err) + } + } + return root +} + +func fakeBuild(cmd *exec.Cmd) error { + return os.WriteFile(filepath.Join(cmd.Dir, cmd.Args[3]), []byte("binary"), 0600) +} + +func TestBuildPlanEnvironmentAndArtifacts(t *testing.T) { + root := buildFixture(t) + environment := []string{"CGO_ENABLED=1", "GOOS=old", "GOARCH=old", "KEEP=value", "CI=true", "VERSION=42"} + before := append([]string(nil), environment...) + processEnv := []string{os.Getenv("GOOS"), os.Getenv("GOARCH"), os.Getenv("CGO_ENABLED")} + want := []struct{ goos, arch, output string }{ + {"darwin", "amd64", "scnnr_bins/mac/intel/scnnr"}, + {"darwin", "arm64", "scnnr_bins/mac/arm/scnnr"}, + {"linux", "amd64", "scnnr_bins/linux/intel/scnnr"}, + {"linux", "arm64", "scnnr_bins/linux/arm/scnnr"}, + {"windows", "386", "scnnr_bins/windows/scnnr.exe"}, + } + calls := 0 + execute := func(cmd *exec.Cmd) error { + if calls >= len(want) { + t.Fatal("unexpected extra build") + } + target := want[calls] + calls++ + args := []string{"go", "build", "-o", filepath.FromSlash(target.output), "main.go"} + if cmd.Dir != root || !reflect.DeepEqual(cmd.Args, args) { + t.Fatalf("build command: dir=%q args=%q; want %q", cmd.Dir, cmd.Args, args) + } + if envValue(cmd.Env, "GOOS") != target.goos || envValue(cmd.Env, "GOARCH") != target.arch || + envValue(cmd.Env, "CGO_ENABLED") != "0" || envValue(cmd.Env, "KEEP") != "value" { + t.Fatalf("wrong target environment for %s/%s", target.goos, target.arch) + } + return fakeBuild(cmd) + } + if err := buildAll(root, environment, io.Discard, io.Discard, execute); err != nil { + t.Fatal(err) + } + if calls != len(want) || !reflect.DeepEqual(environment, before) || + !reflect.DeepEqual(processEnv, []string{os.Getenv("GOOS"), os.Getenv("GOARCH"), os.Getenv("CGO_ENABLED")}) { + t.Fatal("build plan incomplete or environment mutated") + } + for name, want := range map[string]string{"README.md": "readme\n", "LICENSE": "license\n", "version": "42\n"} { + got, err := os.ReadFile(filepath.Join(root, "scnnr_bins", name)) + if err != nil || string(got) != want { + t.Fatalf("artifact %q = %q, %v; want %q", name, got, err, want) + } + } + if runtime.GOOS != "windows" { + info, err := os.Stat(filepath.Join(root, "scnnr_bins/mac/intel/scnnr")) + if err != nil || info.Mode().Perm() != 0755 { + t.Fatalf("binary permissions: info=%v err=%v", info, err) + } + } +} + +func TestBuildCommandFailureStopsPlan(t *testing.T) { + root := buildFixture(t) + wantErr := errors.New("compiler failed") + calls := 0 + err := buildAll(root, nil, io.Discard, io.Discard, func(*exec.Cmd) error { + calls++ + return wantErr + }) + if !errors.Is(err, wantErr) || calls != 1 { + t.Fatalf("build failure: calls=%d error=%v", calls, err) + } + if _, err := os.Stat(filepath.Join(root, "scnnr_bins/README.md")); !errors.Is(err, os.ErrNotExist) { + t.Fatal("failed build continued to copy artifacts") + } +} + +func TestBuildFilesystemFailures(t *testing.T) { + t.Run("directory creation", func(t *testing.T) { + root := buildFixture(t) + if err := os.WriteFile(filepath.Join(root, "scnnr_bins"), []byte("blocked"), 0600); err != nil { + t.Fatal(err) + } + called := false + err := buildAll(root, nil, io.Discard, io.Discard, func(*exec.Cmd) error { called = true; return nil }) + if err == nil || called { + t.Fatalf("directory creation failure: err=%v compilerCalled=%v", err, called) + } + }) + t.Run("binary missing", func(t *testing.T) { + err := buildAll(buildFixture(t), nil, io.Discard, io.Discard, func(*exec.Cmd) error { return nil }) + if !errors.Is(err, os.ErrNotExist) { + t.Fatalf("missing compiler output error=%v", err) + } + }) + t.Run("documentation missing", func(t *testing.T) { + err := buildAll(t.TempDir(), nil, io.Discard, io.Discard, fakeBuild) + if !errors.Is(err, os.ErrNotExist) { + t.Fatalf("missing documentation error=%v", err) + } + }) + t.Run("version output blocked", func(t *testing.T) { + root := buildFixture(t) + if err := os.MkdirAll(filepath.Join(root, "scnnr_bins/version"), 0700); err != nil { + t.Fatal(err) + } + if err := buildAll(root, []string{"CI=1", "VERSION=42"}, io.Discard, io.Discard, fakeBuild); err == nil { + t.Fatal("version write failure ignored") + } + }) + t.Run("documentation output blocked", func(t *testing.T) { + root := buildFixture(t) + if err := os.MkdirAll(filepath.Join(root, "scnnr_bins/README.md"), 0700); err != nil { + t.Fatal(err) + } + if err := buildAll(root, nil, io.Discard, io.Discard, fakeBuild); err == nil { + t.Fatal("documentation write failure ignored") + } + }) +} + +func TestBuildVersionRequiresCIAndVersion(t *testing.T) { + for _, environment := range [][]string{nil, {"CI=1"}, {"VERSION=42"}} { + root := buildFixture(t) + if err := buildAll(root, environment, io.Discard, io.Discard, fakeBuild); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(root, "scnnr_bins/version")); !errors.Is(err, os.ErrNotExist) { + t.Fatal("wrote version without both CI and VERSION") + } + } +} diff --git a/cmd/checksum/main.go b/cmd/checksum/main.go index ea198f6..500f897 100644 --- a/cmd/checksum/main.go +++ b/cmd/checksum/main.go @@ -1,10 +1,13 @@ package main import ( - "crypto/sha256" + "errors" "fmt" + "io" "log" "os" + + "github.com/selfup/scnnr/internal/digest" ) const ( @@ -13,19 +16,31 @@ const ( ) func main() { - zip, err := os.ReadFile(source) - if err != nil { - log.Fatalln(err) + if err := run(source, destination, os.Stdout); err != nil { + log.Fatal(err) } +} - sum := sha256.Sum256(zip) - sumStr := fmt.Sprintf("%x", sum) - sumStrBytes := []byte(sumStr + " " + source + "\n") - - fmt.Println(sumStr) +func checksum(r io.Reader, name string, stdout io.Writer) (string, error) { + sum, err := digest.SHA256(r) + if err != nil { + return "", err + } + if _, err := fmt.Fprintln(stdout, sum); err != nil { + return "", err + } + return sum + " " + name + "\n", nil +} - writeErr := os.WriteFile(destination, sumStrBytes, os.ModePerm) - if writeErr != nil { - log.Fatalln(writeErr) +func run(source, destination string, stdout io.Writer) (err error) { + file, err := os.Open(source) + if err != nil { + return err + } + defer func() { err = errors.Join(err, file.Close()) }() + line, err := checksum(file, source, stdout) + if err != nil { + return err } + return os.WriteFile(destination, []byte(line), 0644) } diff --git a/cmd/checksum/main_test.go b/cmd/checksum/main_test.go new file mode 100644 index 0000000..4088b0c --- /dev/null +++ b/cmd/checksum/main_test.go @@ -0,0 +1,66 @@ +package main + +import ( + "bytes" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +type failingReader struct{ err error } + +func (r failingReader) Read([]byte) (int, error) { return 0, r.err } + +type failingWriter struct{ err error } + +func (w failingWriter) Write([]byte) (int, error) { return 0, w.err } + +func TestChecksumFormatAndFailures(t *testing.T) { + const sum = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + var stdout bytes.Buffer + line, err := checksum(strings.NewReader("abc"), "sample.zip", &stdout) + if err != nil { + t.Fatal(err) + } + if stdout.String() != sum+"\n" || line != sum+" sample.zip\n" { + t.Fatalf("stdout=%q checksum=%q", stdout.String(), line) + } + wantErr := errors.New("I/O failed") + if _, err := checksum(failingReader{wantErr}, "sample.zip", io.Discard); !errors.Is(err, wantErr) { + t.Fatalf("read error=%v", err) + } + if _, err := checksum(strings.NewReader("abc"), "sample.zip", failingWriter{wantErr}); !errors.Is(err, wantErr) { + t.Fatalf("write error=%v", err) + } +} + +func TestRunChecksumFiles(t *testing.T) { + root := t.TempDir() + source, destination := filepath.Join(root, "sample.zip"), filepath.Join(root, "sample.zip.sha256") + if err := os.WriteFile(source, []byte("abc"), 0600); err != nil { + t.Fatal(err) + } + if err := run(source, destination, io.Discard); err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(destination) + if err != nil { + t.Fatal(err) + } + want := "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad " + source + "\n" + if string(got) != want { + t.Fatalf("checksum file=%q; want %q", got, want) + } + if err := run(filepath.Join(root, "missing"), destination, io.Discard); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("missing source error=%v", err) + } + if err := run(source, filepath.Join(root, "missing", "sum"), io.Discard); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("destination error=%v", err) + } + if err := run(root, destination, io.Discard); err == nil { + t.Fatal("directory source accepted") + } +} diff --git a/cmd/pack/main.go b/cmd/pack/main.go index a554e75..c467547 100644 --- a/cmd/pack/main.go +++ b/cmd/pack/main.go @@ -2,112 +2,91 @@ package main import ( "archive/zip" + "errors" "io" "log" "os" "os/exec" + "path" "path/filepath" - "strings" ) func main() { - if fileExists("scnnr_bins") { - deleteFile("scnnr_bins") - } - - if fileExists("scnnr_bins.zip") { - deleteFile("scnnr_bins.zip") - } - - cmd := exec.Command("go", "run", "cmd/build/main.go") - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - cmdErr := cmd.Run() - if cmdErr != nil { - log.Fatal(cmdErr) - } - - zipErr := zipBins("scnnr_bins", "scnnr_bins.zip") - if zipErr != nil { - log.Fatal(zipErr) - } -} - -func fileExists(filename string) bool { - info, err := os.Stat(filename) - if os.IsNotExist(err) { - return false - } - return !info.IsDir() -} - -func deleteFile(path string) { - err := os.Remove(path) + err := run("scnnr_bins", "scnnr_bins.zip", func() error { + cmd := exec.Command("go", "run", "cmd/build/main.go") + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() + }) if err != nil { log.Fatal(err) } } -func zipBins(source string, target string) error { - zipfile, err := os.Create(target) - if err != nil { +func run(source, target string, build func() error) error { + if err := build(); err != nil { return err } + return zipBins(source, target) +} - defer zipfile.Close() - - archive := zip.NewWriter(zipfile) - defer archive.Close() - - info, err := os.Stat(source) +func zipBins(source, target string) (err error) { + if _, err := os.Stat(source); err != nil { + return err + } + file, err := os.Create(target) if err != nil { - return nil + return err } + defer func() { err = errors.Join(err, file.Close()) }() + return writeZip(file, source) +} - var baseDir string - if info.IsDir() { - baseDir = filepath.Base(source) +// writeZip closes the archive before returning so buffered write errors are +// observable. It leaves ownership of the supplied writer with the caller. +func writeZip(w io.Writer, source string) (err error) { + if _, err := os.Stat(source); err != nil { + return err } - - filepath.Walk(source, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err + archive := zip.NewWriter(w) + defer func() { err = errors.Join(err, archive.Close()) }() + base := filepath.Base(filepath.Clean(source)) + return filepath.Walk(source, func(filename string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + return walkErr } - header, err := zip.FileInfoHeader(info) if err != nil { return err } - - if baseDir != "" { - header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source)) + header.Name = base + if info.IsDir() || filename != source { + relative, err := filepath.Rel(source, filename) + if err != nil { + return err + } + header.Name = path.Join(base, filepath.ToSlash(relative)) } - if info.IsDir() { header.Name += "/" } else { header.Method = zip.Deflate } - writer, err := archive.CreateHeader(header) - if err != nil { + if err != nil || info.IsDir() { return err } - - if info.IsDir() { - return nil - } - - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - _, err = io.Copy(writer, file) - return err + return copyFile(writer, filename) }) +} +func copyFile(w io.Writer, filename string) (err error) { + file, err := os.Open(filename) + if err != nil { + return err + } + defer func() { err = errors.Join(err, file.Close()) }() + _, err = io.Copy(w, file) return err } diff --git a/cmd/pack/main_test.go b/cmd/pack/main_test.go new file mode 100644 index 0000000..615ac1b --- /dev/null +++ b/cmd/pack/main_test.go @@ -0,0 +1,134 @@ +package main + +import ( + "archive/zip" + "bytes" + "errors" + "io" + "os" + "path/filepath" + "reflect" + "testing" +) + +type failingWriter struct{ err error } + +func (w failingWriter) Write([]byte) (int, error) { return 0, w.err } + +func archiveFixture(t *testing.T) string { + t.Helper() + root := filepath.Join(t.TempDir(), "bins") + if err := os.MkdirAll(filepath.Join(root, "nested", "empty"), 0700); err != nil { + t.Fatal(err) + } + for name, content := range map[string][]byte{ + "readme.txt": []byte("hello\n"), + filepath.Join("nested", "binary"): {0, 1, 2, 255}, + "empty": {}, + } { + if err := os.WriteFile(filepath.Join(root, name), content, 0600); err != nil { + t.Fatal(err) + } + } + return root +} + +func archiveContents(t *testing.T, data []byte) map[string]string { + t.Helper() + archive, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + t.Fatal(err) + } + contents := make(map[string]string) + for _, entry := range archive.File { + if _, exists := contents[entry.Name]; exists { + t.Fatalf("duplicate archive entry %q", entry.Name) + } + file, err := entry.Open() + if err != nil { + t.Fatal(err) + } + content, err := io.ReadAll(file) + closeErr := file.Close() + if err != nil || closeErr != nil { + t.Fatalf("archive read=%v close=%v", err, closeErr) + } + contents[entry.Name] = string(content) + } + return contents +} + +func TestWriteZipContents(t *testing.T) { + root := archiveFixture(t) + var output bytes.Buffer + if err := writeZip(&output, root); err != nil { + t.Fatal(err) + } + want := map[string]string{ + "bins/": "", "bins/empty": "", "bins/nested/": "", + "bins/nested/binary": string([]byte{0, 1, 2, 255}), + "bins/nested/empty/": "", "bins/readme.txt": "hello\n", + } + if got := archiveContents(t, output.Bytes()); !reflect.DeepEqual(got, want) { + t.Fatalf("archive contents=%v; want %v", got, want) + } + output.Reset() + if err := writeZip(&output, filepath.Join(root, "readme.txt")); err != nil { + t.Fatal(err) + } + if got := archiveContents(t, output.Bytes()); !reflect.DeepEqual(got, map[string]string{"readme.txt": "hello\n"}) { + t.Fatalf("single-file archive=%v", got) + } +} + +func TestArchiveMissingSourceAndOutputErrors(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "out.zip") + if err := zipBins(filepath.Join(root, "missing"), target); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("missing source error=%v", err) + } + if _, err := os.Stat(target); !errors.Is(err, os.ErrNotExist) { + t.Fatal("missing source created an output archive") + } + if err := zipBins(archiveFixture(t), filepath.Join(root, "missing", "out.zip")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("output creation error=%v", err) + } + wantErr := errors.New("archive writer failed") + // The small archive fits in zip.Writer's buffer; the error is surfaced by Close. + if err := writeZip(failingWriter{wantErr}, archiveFixture(t)); !errors.Is(err, wantErr) { + t.Fatalf("archive close/write error=%v", err) + } +} + +func TestArchiveReadFailure(t *testing.T) { + root := archiveFixture(t) + if err := os.Symlink(filepath.Join(root, "missing"), filepath.Join(root, "broken")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + if err := writeZip(io.Discard, root); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("unreadable archive entry error=%v", err) + } +} + +func TestPackBuildFailureAndSuccess(t *testing.T) { + root := archiveFixture(t) + target := filepath.Join(t.TempDir(), "out.zip") + wantErr := errors.New("build failed") + if err := run(root, target, func() error { return wantErr }); !errors.Is(err, wantErr) { + t.Fatalf("build error=%v", err) + } + if _, err := os.Stat(target); !errors.Is(err, os.ErrNotExist) { + t.Fatal("failed build created an archive") + } + called := false + if err := run(root, target, func() error { called = true; return nil }); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(target) + if err != nil { + t.Fatal(err) + } + if !called || archiveContents(t, data)["bins/readme.txt"] != "hello\n" { + t.Fatal("successful build did not produce expected archive") + } +} diff --git a/internal/digest/digest.go b/internal/digest/digest.go new file mode 100644 index 0000000..7b0e3d2 --- /dev/null +++ b/internal/digest/digest.go @@ -0,0 +1,19 @@ +// Package digest provides the streaming checksum shared by finder and release +// commands. +package digest + +import ( + "crypto/sha256" + "fmt" + "io" +) + +func SHA256(r io.Reader) (string, error) { + hash := sha256.New() + + if _, err := io.Copy(hash, r); err != nil { + return "", err + } + + return fmt.Sprintf("%x", hash.Sum(nil)), nil +} diff --git a/internal/digest/digest_test.go b/internal/digest/digest_test.go new file mode 100644 index 0000000..ec259a0 --- /dev/null +++ b/internal/digest/digest_test.go @@ -0,0 +1,27 @@ +package digest + +import ( + "errors" + "strings" + "testing" +) + +type failingReader struct{ err error } + +func (r failingReader) Read([]byte) (int, error) { return 0, r.err } + +func TestSHA256(t *testing.T) { + for _, test := range []struct{ input, want string }{ + {"abc", "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"}, + {"", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, + } { + got, err := SHA256(strings.NewReader(test.input)) + if err != nil || got != test.want { + t.Fatalf("SHA256(%q) = %q, %v; want %q", test.input, got, err, test.want) + } + } + wantErr := errors.New("read failed") + if _, err := SHA256(failingReader{wantErr}); !errors.Is(err, wantErr) { + t.Fatalf("read error = %v", err) + } +} diff --git a/main.go b/main.go index 9a29896..b24b851 100644 --- a/main.go +++ b/main.go @@ -25,21 +25,45 @@ THE SOFTWARE. package main import ( + "errors" "flag" - "fmt" + "io" "log" + "os" "strings" scnnr "github.com/selfup/scnnr/pkg" ) +type options struct { + mode, directory, size string + extensions, keywords, paths, fuzzy, excludeDirs, excludeExts []string + regex, showLines, showCols bool +} + func main() { - var directory string - var extensions []string - var keywords []string + if err := run(os.Args[1:], os.Stdout, os.Stderr); err != nil { + var parseErr *flagError + + if errors.As(err, &parseErr) { + os.Exit(2) + } + + log.Fatal(err) + } +} + +type flagError struct{ error } + +func (e *flagError) Unwrap() error { return e.error } + +func parseOptions(args []string, stderr io.Writer) (options, error) { + flags := flag.NewFlagSet(os.Args[0], flag.ContinueOnError) + flags.SetOutput(stderr) var mode string - flag.StringVar(&mode, "m", "scn", `OPTIONAL + + flags.StringVar(&mode, "m", "scn", `OPTIONAL mode that scnnr will run in options are: @@ -56,135 +80,155 @@ func main() { `) var dir string - flag.StringVar(&dir, "d", ".", `OPTIONAL Scnnr MODE and OPTIONAL FingerrintFinder MODE + flags.StringVar(&dir, "d", ".", `OPTIONAL Scnnr MODE and OPTIONAL FingerrintFinder MODE directory where scnnr will scan default is current directory and all child directories`) var ext string - flag.StringVar(&ext, "e", "", `OPTIONAL Scnnr MODE + flags.StringVar(&ext, "e", "", `OPTIONAL Scnnr MODE a comma delimited list of file extensions to scan if none are given all files will be searched`) var kwd string - flag.StringVar(&kwd, "k", "", `OPTIONAL Scnnr MODE and REQUIRED FingerprintFinder + flags.StringVar(&kwd, "k", "", `OPTIONAL Scnnr MODE and REQUIRED FingerprintFinder scnnr: this is a comma delimited list of characters to look for in a file if no keywords are given - all file paths of given file extensions will be returned if keywords are given - only filepaths of matches will be returned FingerprintFinder: this is a comma delimited list of SHA2-256 hashes to find files by`) var rgx bool - flag.BoolVar(&rgx, "r", false, `OPTIONAL Scnnr MODE + flags.BoolVar(&rgx, "r", false, `OPTIONAL Scnnr MODE if you want to use the regex engine or not defaults to false and will not use the regex engine for scans unless set to a truthy value truthy values are: 1, t, T, true, True, TRUE falsy values are: 0, f, F, false, False, FALSE`) var showLines bool - flag.BoolVar(&showLines, "l", false, `OPTIONAL Scnnr MODE + flags.BoolVar(&showLines, "l", false, `OPTIONAL Scnnr MODE show line numbers for each match (requires -k) when enabled, finds ALL matches in files (no early exit)`) var showCols bool - flag.BoolVar(&showCols, "c", false, `OPTIONAL Scnnr MODE + flags.BoolVar(&showCols, "c", false, `OPTIONAL Scnnr MODE show line and column numbers for each match (requires -k) when enabled, finds ALL matches in files (no early exit)`) var paths string - flag.StringVar(&paths, "p", "", `REQUIRED NameFinder MODE + flags.StringVar(&paths, "p", "", `REQUIRED NameFinder MODE any absolute path - can be comma delimited: Example: $HOME or '/tmp,/usr'`) var fuzzy string - flag.StringVar(&fuzzy, "f", "", `REQUIRED NameFinder MODE + flags.StringVar(&fuzzy, "f", "", `REQUIRED NameFinder MODE fuzzy find the filename(s) contain(s) - can be comma delimited: Example 'wow' or 'wow,omg,lol'`) var size string - flag.StringVar(&size, "s", "", `REQUIRED SizeFinder MODE + flags.StringVar(&size, "s", "", `REQUIRED SizeFinder MODE size: 1MB,10MB,100MB,1GB,10GB,100GB,1TB`) var excludeDirs string - flag.StringVar(&excludeDirs, "xd", "", `OPTIONAL Scnnr MODE + flags.StringVar(&excludeDirs, "xd", "", `OPTIONAL Scnnr MODE comma-delimited list of directory names to exclude from scanning ex: -xd ".git,node_modules,.venv"`) var excludeExts string - flag.StringVar(&excludeExts, "xe", "", `OPTIONAL Scnnr MODE + flags.StringVar(&excludeExts, "xe", "", `OPTIONAL Scnnr MODE comma-delimited list of file extensions to exclude from scanning ex: -xe ".log,.tmp,.json"`) - flag.Parse() + if err := flags.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return options{}, err + } - directory = dir + return options{}, &flagError{err} + } - switch mode { - case "fnf": - scanFuzzy := strings.Split(fuzzy, ",") - scanPaths := strings.Split(paths, ",") + return options{ + mode: mode, directory: dir, size: size, + extensions: splitList(ext), keywords: splitList(kwd), + paths: splitList(paths), fuzzy: splitList(fuzzy), + excludeDirs: optionalList(excludeDirs), excludeExts: optionalList(excludeExts), + regex: rgx, showLines: showLines, showCols: showCols, + }, nil +} - nfnf := scnnr.NewFileNameFinder(scanFuzzy) +func splitList(value string) []string { + return strings.Split(value, ",") +} - for _, path := range scanPaths { - nfnf.Scan(path) - } +func optionalList(value string) []string { + if value == "" { + return nil + } - for _, file := range nfnf.Files { - fmt.Println(file) - } - case "scn": - extensions = strings.Split(ext, ",") - keywords = strings.Split(kwd, ",") + return splitList(value) +} - if (showLines || showCols) && keywords[0] == "" { - log.Fatal("Position tracking flags (-l, -c) require keywords (-k)") +func (o options) validate() error { + switch o.mode { + case "scn": + if (o.showLines || o.showCols) && o.keywords[0] == "" { + return errors.New("Position tracking flags (-l, -c) require keywords (-k)") } + case "fff": + if o.keywords[0] == "" { + return errors.New("-k (known hashes) REQUIRED for FileFingerprintFinder") + } + } - var excludeDirsList []string - var excludeExtsList []string + return nil +} - if excludeDirs != "" { - excludeDirsList = strings.Split(excludeDirs, ",") - } +// run keeps flag state, output, and errors local to each invocation. +func run(args []string, stdout, stderr io.Writer) error { + o, err := parseOptions(args, stderr) - if excludeExts != "" { - excludeExtsList = strings.Split(excludeExts, ",") - } + if errors.Is(err, flag.ErrHelp) { + return nil + } - scanner := scnnr.Scanner{ - Regex: rgx, - Keywords: keywords, - Directory: directory, - FileExtensions: extensions, - ShowLines: showLines, - ShowCols: showCols, - ExcludeDirs: excludeDirsList, - ExcludeExts: excludeExtsList, - } + if err != nil { + return err + } + + if err := o.validate(); err != nil { + return err + } - err := scanner.Scan() + var paths []string - if err != nil { - log.Fatal(err) + switch o.mode { + case "scn": + scanner := scnnr.Scanner{ + Directory: o.directory, FileExtensions: o.extensions, Keywords: o.keywords, + Regex: o.regex, ShowLines: o.showLines, ShowCols: o.showCols, + ExcludeDirs: o.excludeDirs, ExcludeExts: o.excludeExts, } - case "fsf": - nfsf := scnnr.NewFileSizeFinder(size) - nfsf.Scan(directory) + return scanner.ScanTo(stdout) + case "fnf": + finder := scnnr.NewFileNameFinder(o.fuzzy) - for _, file := range nfsf.Files { - fmt.Println(file) + for _, path := range o.paths { + finder.Scan(path) } - case "fff": - keywords = strings.Split(kwd, ",") - if keywords[0] == "" { - log.Fatal("-k (known hashes) REQUIRED for FileFingerprintFinder") - } + paths = finder.Files + case "fsf": + finder := scnnr.NewFileSizeFinder(o.size) + finder.Scan(o.directory) - nfff := scnnr.NewFileFingerprintFinder(keywords) + paths = finder.Files + case "fff": + finder := scnnr.NewFileFingerprintFinder(o.keywords) + finder.Scan(o.directory) - nfff.Scan(directory) + paths = finder.Files + } - for _, file := range nfff.Files { - fmt.Println(file) - } + for _, path := range paths { + io.WriteString(stdout, path+"\n") } + + return nil } diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..4319c08 --- /dev/null +++ b/main_test.go @@ -0,0 +1,224 @@ +package main + +import ( + "bytes" + "errors" + "flag" + "io" + "os" + "os/exec" + "path/filepath" + "reflect" + "slices" + "strings" + "testing" +) + +type failingWriter struct{ err error } + +func (w failingWriter) Write([]byte) (int, error) { return 0, w.err } + +func cliFixture(parts ...string) string { + return filepath.Join(append([]string{"pkg", "testdata", "tree"}, parts...)...) +} + +func assertOutputLines(t *testing.T, got, want string) { + t.Helper() + gotLines := strings.Split(got, "\n") + wantLines := strings.Split(want, "\n") + slices.Sort(gotLines) + slices.Sort(wantLines) + if !slices.Equal(gotLines, wantLines) { + t.Fatalf("output=%q; want the same original lines as %q", got, want) + } +} + +func TestParseOptionsKeepsOriginalDefaultsAndCSV(t *testing.T) { + defaults, err := parseOptions(nil, io.Discard) + if err != nil { + t.Fatal(err) + } + if defaults.mode != "scn" || defaults.directory != "." || + !reflect.DeepEqual(defaults.keywords, []string{""}) || + !reflect.DeepEqual(defaults.extensions, []string{""}) || + defaults.excludeDirs != nil || defaults.excludeExts != nil { + t.Fatalf("original defaults changed: %#v", defaults) + } + opts, err := parseOptions([]string{ + "-d", "sample", "-e", ".go,.txt", "-k", "needle,, token", + "-r", "-c", "-xd", ".git,node_modules", "-xe", ".log", + }, io.Discard) + if err != nil { + t.Fatal(err) + } + if !opts.regex || !opts.showCols || + !reflect.DeepEqual(opts.keywords, []string{"needle", "", " token"}) || + !reflect.DeepEqual(opts.extensions, []string{".go", ".txt"}) { + t.Fatalf("CSV or flags changed: %#v", opts) + } + ignored, err := parseOptions([]string{"positional", "-m", "fff"}, io.Discard) + if err != nil || !reflect.DeepEqual(ignored, defaults) { + t.Fatalf("original positional-argument behavior changed: %#v, %v", ignored, err) + } + if _, err := parseOptions([]string{"-h"}, io.Discard); !errors.Is(err, flag.ErrHelp) { + t.Fatalf("help error=%v", err) + } + again, err := parseOptions(nil, io.Discard) + if err != nil || !reflect.DeepEqual(defaults, again) { + t.Fatal("flag state leaked between invocations") + } +} + +func TestRunOriginalScannerOutput(t *testing.T) { + root := cliFixture() + for _, test := range []struct { + name string + args []string + want string + }{ + {"list paths", []string{"-d", root, "-e", ".go,.log"}, cliFixture("nested", "beta.go") + "\n" + cliFixture("skip.log") + "\n"}, + {"matching paths", []string{"-d", root, "-k", "needle", "-xd", "ignored", "-xe", ".log"}, cliFixture("alpha.txt") + "\n" + cliFixture("nested", "beta.go") + "\n"}, + {"one keyword occurrence per line", []string{"-d", root, "-e", ".txt", "-k", "needle,token", "-l", "-xd", "ignored"}, cliFixture("alpha.txt") + ":1:needle\n" + cliFixture("alpha.txt") + ":2:token\n"}, + {"columns", []string{"-d", root, "-e", ".txt", "-k", "needle", "-c", "-xd", "ignored"}, cliFixture("alpha.txt") + ":1:7\n"}, + {"regex", []string{"-d", root, "-e", ".go", "-k", "n.edle", "-r=true", "-c"}, cliFixture("nested", "beta.go") + ":2:4\n"}, + {"no match retains newline", []string{"-d", root, "-k", "absent"}, "\n"}, + {"empty first CSV element lists files", []string{"-d", root, "-e", ".go", "-k", ",needle"}, cliFixture("nested", "beta.go") + "\n"}, + {"unknown mode succeeds silently", []string{"-m", "unknown"}, ""}, + } { + t.Run(test.name, func(t *testing.T) { + var stdout, stderr bytes.Buffer + if err := run(test.args, &stdout, &stderr); err != nil { + t.Fatal(err) + } + assertOutputLines(t, stdout.String(), test.want) + if stderr.Len() != 0 { + t.Fatalf("stderr=%q", stderr.String()) + } + }) + } +} + +func TestRunOriginalFinders(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "needle.txt"), []byte("abc"), 0600); err != nil { + t.Fatal(err) + } + second := t.TempDir() + if err := os.WriteFile(filepath.Join(second, "needle.go"), []byte("other"), 0600); err != nil { + t.Fatal(err) + } + for _, test := range []struct { + name string + args []string + want string + }{ + {"names include duplicate keyword matches", []string{"-m", "fnf", "-p", root + "," + second, "-f", "needle,.txt"}, root + root + "needle.txt\n" + root + root + "needle.txt\n" + second + second + "needle.go\n"}, + {"missing fuzzy keyword retains match-all", []string{"-m", "fnf", "-p", root}, root + root + "needle.txt\n"}, + {"full hash", []string{"-m", "fff", "-d", root, "-k", "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"}, filepath.Join(root, "needle.txt") + "\n"}, + {"hash substring", []string{"-m", "fff", "-d", root, "-k", "ba7816"}, filepath.Join(root, "needle.txt") + "\n"}, + {"hash no match", []string{"-m", "fff", "-d", root, "-k", "unknown"}, ""}, + } { + t.Run(test.name, func(t *testing.T) { + var stdout bytes.Buffer + if err := run(test.args, &stdout, io.Discard); err != nil { + t.Fatal(err) + } + assertOutputLines(t, stdout.String(), test.want) + }) + } + sizeRoot := t.TempDir() + for name, size := range map[string]int64{"below": 999999, "equal": 1000000, "above": 1000001} { + file, err := os.Create(filepath.Join(sizeRoot, name)) + if err != nil { + t.Fatal(err) + } + err = file.Truncate(size) + closeErr := file.Close() + if err != nil || closeErr != nil { + t.Fatalf("size fixture: %v, %v", err, closeErr) + } + } + var stdout bytes.Buffer + if err := run([]string{"-m", "fsf", "-d", sizeRoot, "-s", "1MB"}, &stdout, io.Discard); err != nil { + t.Fatal(err) + } + assertOutputLines(t, stdout.String(), sizeRoot+sizeRoot+"above\n"+sizeRoot+sizeRoot+"equal\n") +} + +func TestRunOriginalFailures(t *testing.T) { + for _, test := range []struct { + name string + args []string + want string + }{ + {"unknown flag", []string{"-unknown"}, "flag provided but not defined"}, + {"invalid bool", []string{"-r=invalid"}, "invalid"}, + {"lines without keywords", []string{"-l"}, "Position tracking flags (-l, -c) require keywords (-k)"}, + {"columns without keywords", []string{"-c", "-k", ",needle"}, "Position tracking flags (-l, -c) require keywords (-k)"}, + {"fingerprints missing", []string{"-m", "fff"}, "-k (known hashes) REQUIRED for FileFingerprintFinder"}, + {"missing scanner directory", []string{"-d", filepath.Join(t.TempDir(), "missing")}, "missing"}, + } { + t.Run(test.name, func(t *testing.T) { + var stdout, stderr bytes.Buffer + err := run(test.args, &stdout, &stderr) + if err == nil || !strings.Contains(err.Error(), test.want) || stdout.Len() != 0 { + t.Fatalf("error=%v stdout=%q; want %q", err, stdout.String(), test.want) + } + }) + } + wantErr := errors.New("stdout failed") + if err := run([]string{"-d", cliFixture(), "-e", ".go"}, failingWriter{wantErr}, io.Discard); err != nil { + t.Fatalf("original output errors were ignored; got %v", err) + } +} + +func TestRunHelp(t *testing.T) { + var stdout, stderr bytes.Buffer + if err := run([]string{"-h"}, &stdout, &stderr); err != nil { + t.Fatal(err) + } + if stdout.Len() != 0 || !strings.Contains(stderr.String(), "Usage of "+os.Args[0]) { + t.Fatalf("help stdout=%q stderr=%q", stdout.String(), stderr.String()) + } +} + +func TestCLIOriginalExitStatuses(t *testing.T) { + binary := filepath.Join(t.TempDir(), "scnnr") + cmd := exec.Command("go", "build", "-o", binary, "main.go") + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("build CLI: %v\n%s", err, output) + } + for _, test := range []struct { + name string + args []string + code int + message string + }{ + {"help", []string{"-h"}, 0, "Usage of " + binary}, + {"unknown mode", []string{"-m", "unknown"}, 0, ""}, + {"bad flag", []string{"-unknown"}, 2, "flag provided but not defined"}, + {"invalid size panics", []string{"-m", "fsf", "-s", "2MB"}, 2, "panic: please provide a size"}, + {"name root panics", []string{"-m", "fnf", "-f", "needle", "-p", filepath.Join(t.TempDir(), "missing")}, 2, "panic:"}, + {"scanner root returns error", []string{"-d", filepath.Join(t.TempDir(), "missing")}, 1, "missing"}, + {"invalid regex panics", []string{"-d", cliFixture(), "-e", ".go", "-k", "[", "-r"}, 2, "panic: regexp:"}, + {"required positions", []string{"-c"}, 1, "Position tracking flags"}, + } { + t.Run(test.name, func(t *testing.T) { + output, err := exec.Command(binary, test.args...).CombinedOutput() + + code := 0 + + if err != nil { + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) { + t.Fatal(err) + } + code = exitErr.ExitCode() + } + + if code != test.code || !strings.Contains(string(output), test.message) { + t.Fatalf("exit=%d output=%q; want exit=%d containing %q", code, output, test.code, test.message) + } + }) + } +} diff --git a/pkg/api_compatibility_test.go b/pkg/api_compatibility_test.go new file mode 100644 index 0000000..607761a --- /dev/null +++ b/pkg/api_compatibility_test.go @@ -0,0 +1,29 @@ +package scnnr_test + +import ( + "os" + + scnnr "github.com/selfup/scnnr/pkg" +) + +// These assignments fail to compile if a refactor changes an existing public +// method or constructor signature. In particular, finder Scan has no result. +type originalFinder interface{ Scan(string) } +type originalScanner interface{ Scan() error } + +var ( + _ originalFinder = (*scnnr.FileNameFinder)(nil) + _ originalFinder = (*scnnr.FileSizeFinder)(nil) + _ originalFinder = (*scnnr.FileFingerprintFinder)(nil) + _ originalScanner = (*scnnr.Scanner)(nil) + _ func(string) = (*scnnr.FileNameFinder)(nil).Scan + _ func(string) = (*scnnr.FileSizeFinder)(nil).Scan + _ func(string) = (*scnnr.FileFingerprintFinder)(nil).Scan + _ func([]string) *scnnr.FileNameFinder = scnnr.NewFileNameFinder + _ func(string) *scnnr.FileSizeFinder = scnnr.NewFileSizeFinder + _ func([]string) *scnnr.FileFingerprintFinder = scnnr.NewFileFingerprintFinder + _ func(string, string, os.FileInfo) string = scnnr.FullFilePath + _ func() string = scnnr.PathDirection + _ func(string) = scnnr.CheckDirOrPanic + _ func(string, string) ([]os.FileInfo, []os.FileInfo) = scnnr.CollectFilesAndDirs +) diff --git a/pkg/fff.go b/pkg/fff.go index 2210cac..21f8c4f 100644 --- a/pkg/fff.go +++ b/pkg/fff.go @@ -1,14 +1,17 @@ package scnnr import ( - "crypto/sha256" + "bytes" + "errors" "fmt" "os" "strings" "sync" + + "github.com/selfup/scnnr/internal/digest" ) -// FileFingerprintFinder struct contains needed data to perform concurrent operations +// FileFingerprintFinder matches SHA-256 digests, including digest substrings. type FileFingerprintFinder struct { mutex sync.Mutex Direction string @@ -16,64 +19,46 @@ type FileFingerprintFinder struct { FingerPrints []string } -// NewFileFingerprintFinder creates a pointer to FileFingerprintFinder with default values func NewFileFingerprintFinder(fingerPrints []string) *FileFingerprintFinder { - fnf := new(FileFingerprintFinder) - - fnf.Direction = PathDirection() - - fnf.FingerPrints = fingerPrints - - return fnf + return &FileFingerprintFinder{Direction: PathDirection(), FingerPrints: fingerPrints} } -// Scan is a concurrent/parallel directory walker +// Scan preserves the original void API and best-effort reads. Read failures do +// not stop the scan; the bytes returned by ReadFile are hashed, as before. func (f *FileFingerprintFinder) Scan(directory string) { CheckDirOrPanic(directory) + f.findFiles(directory) } func (f *FileFingerprintFinder) findFiles(directory string) { - files, dirs := CollectFilesAndDirs(directory, f.Direction) - - for _, file := range files { - if file != nil { - fullFilePath := FullFilePath(directory, f.Direction, file) - - zip, _ := os.ReadFile(fullFilePath) - - sum := sha256.Sum256(zip) - sumStr := fmt.Sprintf("%x", sum) - - for _, fingerPrint := range f.FingerPrints { - shaString := string(sumStr) - - if strings.Contains(shaString, fingerPrint) { - f.mutex.Lock() - - f.Files = append(f.Files, fullFilePath) - - f.mutex.Unlock() - } + walkFinderFiles(directory, f.Direction, func(directory string, file os.FileInfo) { + path := FullFilePath(directory, f.Direction, file) + data, _ := os.ReadFile(path) + sum, _ := digest.SHA256(bytes.NewReader(data)) + + for _, fingerprint := range f.FingerPrints { + if strings.Contains(sum, fingerprint) { + f.mutex.Lock() + f.Files = append(f.Files, path) + f.mutex.Unlock() } } - } - - dirLen := len(dirs) - if dirLen > 0 { - var dirGroup sync.WaitGroup - - dirGroup.Add(dirLen) + }) +} - for _, dir := range dirs { - go func(dirInfo os.FileInfo, dirName string, pathDirection string) { +func fileDigest(path string) (sum string, err error) { + file, err := os.Open(path) + if err != nil { + return "", fmt.Errorf("open %q: %w", path, err) + } - f.findFiles(dirName + pathDirection + dirInfo.Name()) + defer func() { err = errors.Join(err, file.Close()) }() - dirGroup.Done() - }(dir, directory, f.Direction) - } - - dirGroup.Wait() + sum, err = digest.SHA256(file) + if err != nil { + err = fmt.Errorf("hash %q: %w", path, err) } + + return sum, err } diff --git a/pkg/finders_test.go b/pkg/finders_test.go new file mode 100644 index 0000000..1a57f7f --- /dev/null +++ b/pkg/finders_test.go @@ -0,0 +1,365 @@ +package scnnr + +import ( + "errors" + "os" + "path/filepath" + "reflect" + "slices" + "testing" +) + +func writeFixture(t *testing.T, root, name, content string) string { + t.Helper() + + path := filepath.Join(root, name) + + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + t.Fatal(err) + } + + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } + + return path +} + +func expectPanic(t *testing.T, call func()) any { + t.Helper() + + var value any + + func() { + defer func() { value = recover() }() + call() + }() + + if value == nil { + t.Fatal("expected the original API to panic") + } + + return value +} + +func TestContainsAny(t *testing.T) { + for _, test := range []struct { + value string + keywords []string + want bool + }{ + {"needle.txt", []string{"needle"}, true}, + {"needle.txt", []string{"absent", ".txt"}, true}, + {"needle.txt", []string{"NEEDLE"}, false}, + {"needle.txt", nil, false}, + {"needle.txt", []string{""}, false}, + } { + if got := containsAny(test.value, test.keywords); got != test.want { + t.Errorf("containsAny(%q, %q)=%v; want %v", test.value, test.keywords, got, test.want) + } + } +} + +func TestFileNameFinderLegacyPathsDuplicatesAndAccumulation(t *testing.T) { + first, second := t.TempDir(), t.TempDir() + + writeFixture(t, first, "needle.txt", "abc") + writeFixture(t, first, filepath.Join("nested", "needle.go"), "abc") + writeFixture(t, second, "needle.txt", "abc") + writeFixture(t, first, "unrelated.txt", "needle") + + finder := NewFileNameFinder([]string{"needle", ".go"}) + finder.Scan(first) + finder.Scan(second) + + nested := filepath.Join(first, "nested") + + // These strings pin the original path construction, including its quirks. + want := []string{first + first + "needle.txt", nested + nested + "needle.go", nested + nested + "needle.go", second + second + "needle.txt"} + + if !reflect.DeepEqual(finder.Files, want) { + t.Fatalf("files=%q; want original results %q", finder.Files, want) + } + + expectPanic(t, func() { finder.Scan(filepath.Join(first, "missing")) }) + + if !reflect.DeepEqual(finder.Files, want) { + t.Fatal("root panic changed accumulated results") + } +} + +func TestFinderConstructorsAndInvalidSizePanic(t *testing.T) { + if finder := NewFileNameFinder([]string{"needle"}); finder.Direction != PathDirection() || !reflect.DeepEqual(finder.Keywords, []string{"needle"}) { + t.Fatalf("name finder defaults=%#v", finder) + } + + if finder := NewFileFingerprintFinder([]string{"hash"}); finder.Direction != PathDirection() || !reflect.DeepEqual(finder.FingerPrints, []string{"hash"}) { + t.Fatalf("fingerprint finder defaults=%#v", finder) + } + + for size, want := range map[string]int64{ + "1MB": 1000000, "10MB": 10000000, "100MB": 100000000, + "1GB": 1000000000, "10GB": 10000000000, "100GB": 100000000000, "1TB": 1000000000000, + } { + got, err := ParseSize(size) + + if err != nil || got != want { + t.Errorf("ParseSize(%q)=%d, %v; want %d", size, got, err, want) + } + + finder := NewFileSizeFinder(size) + + if finder.Size != want || finder.Direction != PathDirection() { + t.Errorf("constructor threshold for %q=%d", size, finder.Size) + } + } + for _, size := range []string{"", "0MB", "1MiB", "1mb", "garbage"} { + if _, err := ParseSize(size); err == nil { + t.Errorf("additive ParseSize accepted %q", size) + } + + value := expectPanic(t, func() { NewFileSizeFinder(size) }) + + if value != "please provide a size 1MB 10MB 100MB 1GB 10GB 100GB 1TB" { + t.Fatalf("constructor panic=%v", value) + } + } +} + +func TestFileSizeFinderLegacyThresholdsAndPaths(t *testing.T) { + root := t.TempDir() + + writeFixture(t, root, "below", "ab") + writeFixture(t, root, "equal", "abc") + writeFixture(t, root, filepath.Join("nested", "above"), "abcd") + + finder := NewFileSizeFinder("1MB") + finder.Size = 3 + finder.Scan(root) + + nested := filepath.Join(root, "nested") + + want := []string{root + root + "equal", nested + nested + "above"} + + if !reflect.DeepEqual(finder.Files, want) { + t.Fatalf("threshold files=%q; want %q", finder.Files, want) + } + + finder.Scan(root) + + if !reflect.DeepEqual(finder.Files, append(append([]string(nil), want...), want...)) { + t.Fatal("size finder no longer accumulates") + } + + expectPanic(t, func() { finder.Scan(filepath.Join(root, "missing")) }) + + for _, threshold := range []int64{0, -1} { + all := NewFileSizeFinder("1MB") + + all.Size = threshold + + all.Scan(root) + + if len(all.Files) != 3 { + t.Fatalf("original threshold %d should include all files; got %q", threshold, all.Files) + } + } +} + +func TestFileFingerprintFinderLegacyMatching(t *testing.T) { + const abcHash = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + const emptyHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + root := t.TempDir() + + abc := writeFixture(t, root, "abc.txt", "abc") + empty := writeFixture(t, root, filepath.Join("nested", "empty"), "") + other := writeFixture(t, root, "other", "different") + + for _, test := range []struct { + name string + hashes, want []string + }{ + {"full digest", []string{abcHash}, []string{abc}}, + {"substring", []string{abcHash[:12]}, []string{abc}}, + {"multiple digests", []string{emptyHash, abcHash}, []string{abc, empty}}, + {"overlapping digests produce duplicates", []string{abcHash, abcHash[:12]}, []string{abc, abc}}, + {"no match", []string{"unknown"}, nil}, + {"empty fingerprint matches every digest", []string{""}, []string{abc, other, empty}}, + } { + t.Run(test.name, func(t *testing.T) { + finder := NewFileFingerprintFinder(test.hashes) + + finder.Scan(root) + + if !reflect.DeepEqual(finder.Files, test.want) { + t.Fatalf("files=%q; want %q", finder.Files, test.want) + } + }) + } + + finder := NewFileFingerprintFinder([]string{abcHash}) + + // tests accumulation across calls. + // Scan appends to finder.Files + // so scanning twice should leave []string{abc, abc}. + finder.Scan(root) + finder.Scan(root) + + if !reflect.DeepEqual(finder.Files, []string{abc, abc}) { + t.Fatal("fingerprint finder no longer accumulates") + } + + expectPanic(t, func() { finder.Scan(filepath.Join(root, "missing")) }) + + // The additive reader helper can still report errors independently. + if _, err := fileDigest(filepath.Join(root, "missing")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("fileDigest missing file error=%v", err) + } +} + +func TestFindersContinueAfterBrokenEntries(t *testing.T) { + root := t.TempDir() + + abc := writeFixture(t, root, "a-needle.txt", "abc") + + nested := writeFixture(t, root, filepath.Join("nested", "b-needle.txt"), "abc") + + if err := os.Symlink(filepath.Join(root, "missing"), filepath.Join(root, "broken")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + + name := NewFileNameFinder([]string{"needle"}) + + size := NewFileSizeFinder("1MB") + size.Size = 1 + + hash := NewFileFingerprintFinder([]string{"ba7816"}) + + name.Scan(root) + + size.Scan(root) + + hash.Scan(root) + + nestedDir := filepath.Join(root, "nested") + + wantPaths := []string{root + root + "a-needle.txt", nestedDir + nestedDir + "b-needle.txt"} + + if !reflect.DeepEqual(name.Files, wantPaths) || !reflect.DeepEqual(size.Files, wantPaths) { + t.Fatalf("name/size scans stopped: name=%q size=%q", name.Files, size.Files) + } + + if !reflect.DeepEqual(hash.Files, []string{abc, nested}) { + t.Fatalf("hash scan stopped: %q", hash.Files) + } +} + +func TestFingerprintFinderContinuesAfterReadFailure(t *testing.T) { + root := t.TempDir() + + target := t.TempDir() + + link := filepath.Join(root, "a-unreadable-directory-link") + + if err := os.Symlink(target, link); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + + abc := writeFixture(t, root, "z-abc.txt", "abc") + + const emptyHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + finder := NewFileFingerprintFinder([]string{emptyHash, "ba7816"}) + + finder.Scan(root) + // ReadFile on the directory fails. The original API hashes the returned + // empty bytes and continues to the following readable file. + if !reflect.DeepEqual(finder.Files, []string{link, abc}) { + t.Fatalf("read-failure behavior changed: %q", finder.Files) + } +} + +func TestEmptyNameKeywordsAndCustomDirection(t *testing.T) { + root := t.TempDir() + + writeFixture(t, root, "plain.txt", "abc") + + all := NewFileNameFinder([]string{""}) + + all.Scan(root) + + if !reflect.DeepEqual(all.Files, []string{root + root + "plain.txt"}) { + t.Fatalf("empty keyword behavior=%q", all.Files) + } + + missingMetadata := NewFileNameFinder([]string{""}) + missingMetadata.Direction = "|" + missingMetadata.Scan(root) + + if len(missingMetadata.Files) != 0 { + t.Fatalf("custom direction's failed stats were not skipped: %q", missingMetadata.Files) + } +} + +func TestPathHelpersKeepSuppliedDirection(t *testing.T) { + root := t.TempDir() + path := writeFixture(t, root, "plain.txt", "abc") + info, err := os.Stat(path) + + if err != nil { + t.Fatal(err) + } + + for _, direction := range []string{"/", "|", "", root} { + want := root + direction + "plain.txt" + + if got := FullFilePath(root, direction, info); got != want { + t.Fatalf("FullFilePath direction=%q got=%q; want %q", direction, got, want) + } + } + + files, dirs := CollectFilesAndDirs(root, PathDirection()) + + if len(files) != 1 || len(dirs) != 0 || files[0].Name() != "plain.txt" { + t.Fatalf("collection files=%v dirs=%v", files, dirs) + } + + files, dirs = CollectFilesAndDirs(filepath.Join(root, "missing"), PathDirection()) + + if len(files)+len(dirs) != 0 { + t.Fatal("failed directory read should return no entries") + } +} + +func TestAdditiveWalkCallbackAndSymlinks(t *testing.T) { + root := t.TempDir() + target := writeFixture(t, root, "target.txt", "abc") + wantErr := errors.New("visitor failed") + + if err := walkFiles(root, nil, nil, func(FileData) error { return wantErr }); !errors.Is(err, wantErr) { + t.Fatalf("visitor error=%v", err) + } + + if err := os.Symlink(target, filepath.Join(root, "file-link")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + + if err := os.Symlink(root, filepath.Join(root, "dir-link")); err != nil { + t.Fatal(err) + } + + var paths []string + + if err := walkFiles(root, nil, nil, func(file FileData) error { + paths = append(paths, file.Path) + + return nil + }); err != nil { + t.Fatal(err) + } + + if !slices.Equal(paths, []string{filepath.Join(root, "file-link"), target}) { + t.Fatalf("additive traversal=%q", paths) + } +} diff --git a/pkg/fnf.go b/pkg/fnf.go index cc12b81..261d298 100644 --- a/pkg/fnf.go +++ b/pkg/fnf.go @@ -6,7 +6,7 @@ import ( "sync" ) -// FileNameFinder struct contains needed data to perform concurrent operations +// FileNameFinder finds filenames containing the supplied keywords. type FileNameFinder struct { mutex sync.Mutex Direction string @@ -14,58 +14,39 @@ type FileNameFinder struct { Keywords []string } -// NewFileNameFinder creates a pointer to FileNameFinder with default values func NewFileNameFinder(keywords []string) *FileNameFinder { - fnf := new(FileNameFinder) - - fnf.Direction = PathDirection() - - fnf.Keywords = keywords - - return fnf + return &FileNameFinder{Direction: PathDirection(), Keywords: keywords} } -// Scan is a concurrent/parallel directory walker +// Scan preserves the original void API and root-directory panic. Results append +// as files are visited, including a result for each matching keyword. func (f *FileNameFinder) Scan(directory string) { CheckDirOrPanic(directory) + f.findFiles(directory) } func (f *FileNameFinder) findFiles(directory string) { - files, dirs := CollectFilesAndDirs(directory, f.Direction) - - for _, file := range files { - if file != nil { - fileName := file.Name() - - fullFilePath := FullFilePath(directory, directory, file) - - for _, keyword := range f.Keywords { - if strings.Contains(fileName, keyword) { - f.mutex.Lock() - - f.Files = append(f.Files, fullFilePath) - - f.mutex.Unlock() - } + walkFinderFiles(directory, f.Direction, func(directory string, file os.FileInfo) { + // Keep the original name finder's path construction for compatibility. + path := FullFilePath(directory, directory, file) + + for _, keyword := range f.Keywords { + if strings.Contains(file.Name(), keyword) { + f.mutex.Lock() + f.Files = append(f.Files, path) + f.mutex.Unlock() } } - } - - dirLen := len(dirs) - if dirLen > 0 { - var dirGroup sync.WaitGroup - - dirGroup.Add(dirLen) - - for _, dir := range dirs { - go func(dirInfo os.FileInfo, dirName string, pathDirection string) { - f.findFiles(dirName + pathDirection + dirInfo.Name()) + }) +} - dirGroup.Done() - }(dir, directory, f.Direction) +func containsAny(value string, keywords []string) bool { + for _, keyword := range keywords { + if keyword != "" && strings.Contains(value, keyword) { + return true } - - dirGroup.Wait() } + + return false } diff --git a/pkg/fsf.go b/pkg/fsf.go index f5283cd..5f1639b 100644 --- a/pkg/fsf.go +++ b/pkg/fsf.go @@ -1,11 +1,12 @@ package scnnr import ( + "fmt" "os" "sync" ) -// FileSizeFinder struct contains needed data to perform concurrent operations +// FileSizeFinder finds files at least Size bytes long. type FileSizeFinder struct { mutex sync.Mutex Files []string @@ -13,35 +14,40 @@ type FileSizeFinder struct { Size int64 } -// NewFileSizeFinder creates a pointer to FileSizeFinder with default values -func NewFileSizeFinder(size string) *FileSizeFinder { - fsf := new(FileSizeFinder) - - fsf.Direction = PathDirection() - +// ParseSize converts one of the supported decimal size thresholds into bytes. +func ParseSize(size string) (int64, error) { switch size { case "1MB": - fsf.Size = 1000000 + return 1000000, nil case "10MB": - fsf.Size = 10000000 + return 10000000, nil case "100MB": - fsf.Size = 100000000 + return 100000000, nil case "1GB": - fsf.Size = 1000000000 + return 1000000000, nil case "10GB": - fsf.Size = 10000000000 + return 10000000000, nil case "100GB": - fsf.Size = 100000000000 + return 100000000000, nil case "1TB": - fsf.Size = 1000000000000 + return 1000000000000, nil default: + return 0, fmt.Errorf("invalid size %q: please provide a size 1MB 10MB 100MB 1GB 10GB 100GB 1TB", size) + } +} + +// NewFileSizeFinder preserves the original panic for unsupported thresholds. +func NewFileSizeFinder(size string) *FileSizeFinder { + threshold, err := ParseSize(size) + + if err != nil { panic("please provide a size 1MB 10MB 100MB 1GB 10GB 100GB 1TB") } - return fsf + return &FileSizeFinder{Direction: PathDirection(), Size: threshold} } -// Scan is a concurrent/parallel directory walker +// Scan preserves the original void API, root panic, and append behavior. func (f *FileSizeFinder) Scan(directory string) { CheckDirOrPanic(directory) @@ -49,36 +55,16 @@ func (f *FileSizeFinder) Scan(directory string) { } func (f *FileSizeFinder) findFiles(directory string) { - files, dirs := CollectFilesAndDirs(directory, f.Direction) - - for _, file := range files { - if file != nil { - fullFilePath := FullFilePath(directory, directory, file) + walkFinderFiles(directory, f.Direction, func(directory string, file os.FileInfo) { + if file.Size() >= f.Size { + // Keep the original size finder's path construction for compatibility. + path := FullFilePath(directory, directory, file) - if file.Size() >= f.Size { - f.mutex.Lock() + f.mutex.Lock() - f.Files = append(f.Files, fullFilePath) + f.Files = append(f.Files, path) - f.mutex.Unlock() - } + f.mutex.Unlock() } - } - - dirLen := len(dirs) - if dirLen > 0 { - var dirGroup sync.WaitGroup - - dirGroup.Add(dirLen) - - for _, dir := range dirs { - go func(dirInfo os.FileInfo, dirName string, pathDirection string) { - f.findFiles(dirName + pathDirection + dirInfo.Name()) - - dirGroup.Done() - }(dir, directory, f.Direction) - } - - dirGroup.Wait() - } + }) } diff --git a/pkg/helpers.go b/pkg/helpers.go index 6ec1b2a..9d5c771 100644 --- a/pkg/helpers.go +++ b/pkg/helpers.go @@ -2,23 +2,18 @@ package scnnr import ( "os" - "runtime" ) +// FullFilePath uses the supplied direction verbatim, as in the original API. func FullFilePath(directory, direction string, file os.FileInfo) string { - fileName := file.Name() - - return directory + direction + fileName + return directory + direction + file.Name() } func PathDirection() string { - if runtime.GOOS == "windows" { - return "\\" - } else { - return "/" - } + return string(os.PathSeparator) } +// CheckDirOrPanic panics when the root directory cannot be read. func CheckDirOrPanic(directory string) { _, err := os.ReadDir(directory) @@ -27,6 +22,7 @@ func CheckDirOrPanic(directory string) { } } +// CollectFilesAndDirs preserves the original best-effort directory collection. func CollectFilesAndDirs(directory string, direction string) ([]os.FileInfo, []os.FileInfo) { paths, _ := os.ReadDir(directory) diff --git a/pkg/matcher.go b/pkg/matcher.go new file mode 100644 index 0000000..0554827 --- /dev/null +++ b/pkg/matcher.go @@ -0,0 +1,147 @@ +package scnnr + +import ( + "bufio" + "errors" + "fmt" + "io" + "regexp" + "strings" +) + +type contentMatcher struct { + keywords []string + patterns []*regexp.Regexp + regex bool + positions bool +} + +func newContentMatcher(keywords []string, useRegex, positions bool) (*contentMatcher, error) { + matcher := &contentMatcher{keywords: nonEmpty(keywords), regex: useRegex, positions: positions} + if useRegex { + for _, keyword := range matcher.keywords { + pattern, err := regexp.Compile(keyword) + + if err != nil { + return nil, fmt.Errorf("invalid regex %q: %w", keyword, err) + } + + matcher.patterns = append(matcher.patterns, pattern) + } + } + + return matcher, nil +} + +// parseLegacy is the reader seam for the original Scan behavior. It reports the +// first occurrence of each keyword on each line, keeps reading after a match, +// retains empty keywords, and compiles regexes only when a line is examined. +func (m *contentMatcher) parseLegacy(r io.Reader, path string, size int64, record func(Match)) error { + scanner := bufio.NewScanner(r) + scanner.Buffer(make([]byte, 0, 1024), 2*int(size)) + found := false + lineNumber := 0 + + for scanner.Scan() { + lineNumber++ + line := scanner.Text() + for _, keyword := range m.keywords { + if !m.positions && found { + break + } + + column := -1 + + if m.regex { + if location := regexp.MustCompile(keyword).FindStringIndex(line); location != nil { + column = location[0] + } + } else { + column = strings.Index(line, keyword) + } + + if column >= 0 { + record(Match{Path: path, Line: lineNumber, Column: column + 1, Keyword: keyword}) + + if !m.positions { + found = true + break + } + } + } + } + return scanner.Err() +} + +// parse uses a buffered reader so line length does not depend on file metadata +// or bufio.Scanner's token limit. Without positions, the first match ends a read. +func (m *contentMatcher) parse(r io.Reader, path string) ([]Match, error) { + reader := bufio.NewReader(r) + + var matches []Match + + for lineNumber := 1; ; lineNumber++ { + line, err := reader.ReadString('\n') + if err != nil && !errors.Is(err, io.EOF) { + return nil, err + } + + if len(line) > 0 { + line = strings.TrimSuffix(strings.TrimSuffix(line, "\n"), "\r") + + for i, keyword := range m.keywords { + var columns []int + + if len(m.patterns) > 0 { + limit := 1 + + if m.positions { + limit = -1 + } + + for _, location := range m.patterns[i].FindAllStringIndex(line, limit) { + columns = append(columns, location[0]+1) + } + } else { + for offset := 0; offset < len(line); { + index := strings.Index(line[offset:], keyword) + + if index < 0 { + break + } + + columns = append(columns, offset+index+1) + offset += index + len(keyword) + + if !m.positions { + break + } + } + } + for _, column := range columns { + matches = append(matches, Match{Path: path, Line: lineNumber, Column: column, Keyword: keyword}) + + if !m.positions { + return matches, nil + } + } + } + } + + if errors.Is(err, io.EOF) { + return matches, nil + } + } +} + +func nonEmpty(values []string) []string { + var result []string + + for _, value := range values { + if value != "" { + result = append(result, value) + } + } + + return result +} diff --git a/pkg/matcher_test.go b/pkg/matcher_test.go new file mode 100644 index 0000000..7078f88 --- /dev/null +++ b/pkg/matcher_test.go @@ -0,0 +1,155 @@ +package scnnr + +import ( + "errors" + "io" + "reflect" + "strings" + "testing" +) + +type failingReader struct{ err error } + +func (r failingReader) Read([]byte) (int, error) { return 0, r.err } + +type failingWriter struct{ err error } + +func (w failingWriter) Write([]byte) (int, error) { return 0, w.err } + +func TestContentMatcher(t *testing.T) { + tests := []struct { + name, input string + keywords []string + regex, all bool + want []Match + }{ + { + name: "first match only", input: "hello needle needle\nneedle\n", + keywords: []string{"needle"}, + want: []Match{{"sample", 1, 7, "needle"}}, + }, + { + name: "every literal occurrence", input: "hello needle needle\nsecond token\n", + keywords: []string{"needle", "token"}, all: true, + want: []Match{{"sample", 1, 7, "needle"}, {"sample", 1, 14, "needle"}, {"sample", 2, 8, "token"}}, + }, + { + name: "every regex occurrence", input: "id12 id345\nid6", + keywords: []string{`id\d+`}, regex: true, all: true, + want: []Match{{"sample", 1, 1, `id\d+`}, {"sample", 1, 6, `id\d+`}, {"sample", 2, 1, `id\d+`}}, + }, + { + name: "regex first match", input: "id12 id345", + keywords: []string{`id\d+`}, regex: true, + want: []Match{{"sample", 1, 1, `id\d+`}}, + }, + { + name: "CRLF blank lines and final unterminated line", input: "\r\nneedle\r\n\r\nneedle", + keywords: []string{"needle"}, all: true, + want: []Match{{"sample", 2, 1, "needle"}, {"sample", 4, 1, "needle"}}, + }, + { + name: "byte columns", input: "é needle", + keywords: []string{"needle"}, all: true, + want: []Match{{"sample", 1, 4, "needle"}}, + }, + { + name: "literal metacharacters and spaces", input: "a+b a+b", + keywords: []string{"a+b", " "}, all: true, + want: []Match{{"sample", 1, 1, "a+b"}, {"sample", 1, 5, "a+b"}, {"sample", 1, 4, " "}}, + }, + { + name: "zero width regex", input: "abc", + keywords: []string{"^", "$"}, regex: true, all: true, + want: []Match{{"sample", 1, 1, "^"}, {"sample", 1, 4, "$"}}, + }, + { + name: "long line", input: strings.Repeat("x", 100000) + "needle", + keywords: []string{"needle"}, all: true, + want: []Match{{"sample", 1, 100001, "needle"}}, + }, + {name: "empty input", keywords: []string{"needle"}, all: true}, + {name: "no match", input: "unrelated\n", keywords: []string{"needle"}, all: true}, + {name: "empty keyword entries", input: "hello", keywords: []string{"", ""}, all: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + matcher, err := newContentMatcher(test.keywords, test.regex, test.all) + if err != nil { + t.Fatal(err) + } + got, err := matcher.parse(strings.NewReader(test.input), "sample") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, test.want) { + t.Fatalf("matches = %#v; want %#v", got, test.want) + } + }) + } +} + +func TestContentMatcherErrors(t *testing.T) { + if _, err := newContentMatcher([]string{"["}, true, true); err == nil { + t.Fatal("invalid regex accepted") + } + wantErr := errors.New("reader failed") + for _, all := range []bool{false, true} { + matcher, err := newContentMatcher([]string{"needle"}, false, all) + if err != nil { + t.Fatal(err) + } + if _, err := matcher.parse(failingReader{wantErr}, "sample"); !errors.Is(err, wantErr) { + t.Fatalf("read error = %v; want %v", err, wantErr) + } + input := io.MultiReader(strings.NewReader("needle\n"), failingReader{wantErr}) + matches, err := matcher.parse(input, "sample") + if all && !errors.Is(err, wantErr) { + t.Fatalf("position scan lost reader failure: %v", err) + } + if !all && (err != nil || len(matches) != 1) { + t.Fatalf("first-match scan kept reading: matches=%v, err=%v", matches, err) + } + } +} + +func TestLegacyReaderMatching(t *testing.T) { + for _, test := range []struct { + name, input string + keywords []string + regex, positions bool + want []Match + }{ + {"first occurrence per keyword per line", "needle needle\ntoken token\n", []string{"needle", "token"}, false, true, []Match{{"sample", 1, 1, "needle"}, {"sample", 2, 1, "token"}}}, + {"one result per file", "needle needle\nneedle\n", []string{"needle"}, false, false, []Match{{"sample", 1, 1, "needle"}}}, + {"regex first occurrence per line", "id1 id2\nid3", []string{"id[0-9]"}, true, true, []Match{{"sample", 1, 1, "id[0-9]"}, {"sample", 2, 1, "id[0-9]"}}}, + {"empty keyword retained", "nothing\n\n", []string{""}, false, true, []Match{{"sample", 1, 1, ""}, {"sample", 2, 1, ""}}}, + {"duplicate keywords retained", "needle", []string{"needle", "needle"}, false, true, []Match{{"sample", 1, 1, "needle"}, {"sample", 1, 1, "needle"}}}, + {"byte columns and CRLF", "é needle\r\n", []string{"needle"}, false, true, []Match{{"sample", 1, 4, "needle"}}}, + {"empty input does not compile invalid regex", "", []string{"["}, true, true, nil}, + } { + t.Run(test.name, func(t *testing.T) { + matcher := contentMatcher{keywords: test.keywords, regex: test.regex, positions: test.positions} + var got []Match + err := matcher.parseLegacy(strings.NewReader(test.input), "sample", int64(len(test.input)), func(match Match) { got = append(got, match) }) + if err != nil || !reflect.DeepEqual(got, test.want) { + t.Fatalf("legacy matches=%#v error=%v; want %#v", got, err, test.want) + } + }) + } + invalid := contentMatcher{keywords: []string{"["}, regex: true} + expectPanic(t, func() { invalid.parseLegacy(strings.NewReader("line"), "sample", 4, func(Match) {}) }) +} + +func TestLegacyReaderKeepsReadingAfterMatch(t *testing.T) { + wantErr := errors.New("failure after matching line") + for _, positions := range []bool{false, true} { + matcher := contentMatcher{keywords: []string{"needle"}, positions: positions} + var got []Match + input := io.MultiReader(strings.NewReader("needle\n"), failingReader{wantErr}) + err := matcher.parseLegacy(input, "sample", 100, func(match Match) { got = append(got, match) }) + if !errors.Is(err, wantErr) || len(got) != 1 { + t.Fatalf("original reader behavior positions=%v matches=%v error=%v", positions, got, err) + } + } +} diff --git a/pkg/scanner.go b/pkg/scanner.go index a40346b..6df6f09 100644 --- a/pkg/scanner.go +++ b/pkg/scanner.go @@ -1,18 +1,19 @@ package scnnr import ( - "bufio" + "errors" "fmt" + "io" "log" "os" "path/filepath" - "regexp" "slices" "strings" "sync" ) -// Scanner scans files and based on pattern stores in array for goroutine processing +// Scanner configures a content search. Search returns results without printing +// or changing the result fields; Scan also populates those fields and prints. type Scanner struct { sync.Mutex Regex bool @@ -28,13 +29,13 @@ type Scanner struct { MatchedFilePaths []FileData } -// FileData contains the path of the file as well as relevant metadata +// FileData contains a file's path and metadata. type FileData struct { Path string Info os.FileInfo } -// Match represents a single keyword match with position information +// Match identifies a keyword occurrence. Lines and byte columns are one-based. type Match struct { Path string Line int @@ -42,72 +43,144 @@ type Match struct { Keyword string } -// Scan walks the given directory tree and stores all matching files into a slice -func (s *Scanner) Scan() error { - err := filepath.Walk(s.Directory, s.scan) +// ScanResult contains candidate files, matching paths, and keyword occurrences. +// Without keywords, Paths contains every candidate and Matches is empty. +type ScanResult struct { + Files []FileData + Paths []string + Matches []Match +} + +// Search walks Directory and returns results in file-path order. Empty keyword +// and extension lists mean list files and include all extensions, respectively. +func (s *Scanner) Search() (ScanResult, error) { + keywords := nonEmpty(s.Keywords) + + positions := s.ShowLines || s.ShowCols + if positions && len(keywords) == 0 { + return ScanResult{}, errors.New("position tracking flags (-l, -c) require keywords (-k)") + } + + matcher, err := newContentMatcher(keywords, s.Regex, positions) if err != nil { - return err + return ScanResult{}, err } - if s.Keywords[0] == "" { - var foundFiles []string + extensions := nonEmpty(s.FileExtensions) + + var result ScanResult + + err = walkFiles(s.Directory, s.ExcludeDirs, func(path string) bool { + return includesExtension(filepath.Ext(path), extensions, s.ExcludeExts) + }, func(file FileData) error { + result.Files = append(result.Files, file) + + return nil + }) + + if err != nil { + return ScanResult{}, err + } - for _, fileInfo := range s.MatchedFilePaths { - foundFiles = append(foundFiles, fileInfo.Path) + if len(keywords) == 0 { + for _, file := range result.Files { + result.Paths = append(result.Paths, file.Path) } - fmt.Println(strings.Join(foundFiles, "\n")) - } else { + return result, nil + } + + // Keep file I/O bounded independently of the number of discovered files. + slots := make(chan struct{}, concurrentFiles) + + for _, chunk := range eachSlice(result.Files) { + matches := make([][]Match, len(chunk)) + errs := make([]error, len(chunk)) + var wg sync.WaitGroup - for _, chunk := range eachSlice(s.MatchedFilePaths) { - matchedCount := len(chunk) + wg.Add(len(chunk)) + + for i, file := range chunk { + slots <- struct{}{} - wg.Add(matchedCount) + go func() { + defer func() { <-slots; wg.Done() }() + matches[i], errs[i] = parseFile(file.Path, matcher) + }() + } - for _, match := range chunk { - go func(m FileData) { - s.parse(m) + wg.Wait() - wg.Done() - }(match) + for i, fileMatches := range matches { + if errs[i] != nil { + return ScanResult{}, errs[i] } - wg.Wait() + if len(fileMatches) > 0 { + result.Paths = append(result.Paths, chunk[i].Path) + result.Matches = append(result.Matches, fileMatches...) + } } + } - // Output results based on position tracking settings - if s.ShowLines || s.ShowCols { - s.outputPositionMatches() - } else { - // Original behavior - fmt.Println(strings.Join(s.KeywordMatches, "\n")) + return result, nil +} + +// Scan preserves the original API: result fields accumulate and output goes to +// stdout. Walk errors are returned; file-reading errors use the original fatal +// handling. Search remains an independent, additive result-oriented API. +func (s *Scanner) Scan() error { + return s.ScanTo(os.Stdout) +} + +// ScanTo supplies the output seam for testing Scan without changing its behavior. +// Like Scan's original fmt.Println calls, output errors are not returned. +func (s *Scanner) ScanTo(w io.Writer) error { + if err := filepath.Walk(s.Directory, s.scan); err != nil { + return err + } + + if s.Keywords[0] == "" { + var paths []string + + for _, file := range s.MatchedFilePaths { + paths = append(paths, file.Path) } + + fmt.Fprintln(w, strings.Join(paths, "\n")) + + return nil } + for _, chunk := range legacySlices(s.MatchedFilePaths) { + var wg sync.WaitGroup - return nil -} + wg.Add(len(chunk)) -// outputPositionMatches formats and outputs matches with position information -func (s *Scanner) outputPositionMatches() { - for _, match := range s.AllMatches { - output := match.Path - - if s.ShowCols { - // -c flag shows both line and column - output = fmt.Sprintf("%s:%d:%d", output, match.Line, match.Column) - } else if s.ShowLines { - // -l flag shows only line - output = fmt.Sprintf("%s:%d", output, match.Line) + for _, file := range chunk { + go func() { + defer wg.Done() + s.parse(file) + }() } - // If multiple keywords, append the matching keyword - if len(s.Keywords) > 1 { - output = fmt.Sprintf("%s:%s", output, match.Keyword) + wg.Wait() + } + + if s.ShowLines || s.ShowCols { + options := OutputOptions{ + ShowLines: s.ShowLines, ShowCols: s.ShowCols, + ShowKeywords: len(s.Keywords) > 1, } - fmt.Println(output) + for _, match := range s.AllMatches { + fmt.Fprintln(w, formatMatch(match, options)) + } + } else { + fmt.Fprintln(w, strings.Join(s.KeywordMatches, "\n")) } + + return nil } func (s *Scanner) scan(path string, info os.FileInfo, err error) error { @@ -115,26 +188,20 @@ func (s *Scanner) scan(path string, info os.FileInfo, err error) error { return err } - // Check if directory should be excluded - if info.IsDir() && s.shouldExcludeDir(filepath.Base(path)) { + if info.IsDir() && slices.Contains(s.ExcludeDirs, filepath.Base(path)) { return filepath.SkipDir } - if !info.IsDir() { - fileExtension := filepath.Ext(path) - - // Check if file extension should be excluded - if s.shouldExcludeExt(fileExtension) { - return nil - } + if info.IsDir() || slices.Contains(s.ExcludeExts, filepath.Ext(path)) { + return nil + } - if s.FileExtensions[0] == "" { - s.MatchedFilePaths = append(s.MatchedFilePaths, FileData{path, info}) - } else { - for _, pattern := range s.FileExtensions { - if fileExtension == pattern { - s.MatchedFilePaths = append(s.MatchedFilePaths, FileData{path, info}) - } + if s.FileExtensions[0] == "" { + s.MatchedFilePaths = append(s.MatchedFilePaths, FileData{path, info}) + } else { + for _, extension := range s.FileExtensions { + if filepath.Ext(path) == extension { + s.MatchedFilePaths = append(s.MatchedFilePaths, FileData{path, info}) } } } @@ -142,119 +209,139 @@ func (s *Scanner) scan(path string, info os.FileInfo, err error) error { return nil } -// If Regex is true the parser will switch to regex mode. -// Otherwise strings.Contains will be used. -func (s *Scanner) parse(match FileData) { - file, err := os.Open(match.Path) +func (s *Scanner) parse(fileData FileData) { + file, err := os.Open(fileData.Path) check(err) - scanner := bufio.NewScanner(file) - - buf := make([]byte, 0, 1024) + matcher := contentMatcher{ + keywords: s.Keywords, regex: s.Regex, + positions: s.ShowLines || s.ShowCols, + } - // only extend buffer to file size - scanner.Buffer(buf, 2*int(match.Info.Size())) + err = matcher.parseLegacy(file, fileData.Path, fileData.Info.Size(), func(match Match) { + s.Lock() + defer s.Unlock() + if matcher.positions { + s.AllMatches = append(s.AllMatches, match) + } else { + s.KeywordMatches = append(s.KeywordMatches, match.Path) + } + }) - found := false + check(err) - lineNumber := 0 + file.Close() +} - positionTracking := s.ShowLines || s.ShowCols +// legacySlices preserves Scan's existing chunk-boundary behavior. The additive +// Search API uses eachSlice, independently of this compatibility path. +func legacySlices(files []FileData) [][]FileData { + var chunks [][]FileData + var chunk []FileData - for scanner.Scan() { - line := scanner.Text() + for _, file := range files { + if len(chunk) == filesPerChunk { + chunks = append(chunks, chunk) + chunk = nil + } else { + chunk = append(chunk, file) + } + } - lineNumber++ + if len(chunk) < filesPerChunk { + chunks = append(chunks, chunk) + } - for i := 0; i < len(s.Keywords); i++ { - if !positionTracking && found { - break - } + return chunks +} - matchFound := false - var column int - - if s.Regex { - re := regexp.MustCompile(s.Keywords[i]) - - if loc := re.FindStringIndex(line); loc != nil { - matchFound = true - column = loc[0] + 1 - } - } else { - if idx := strings.Index(line, s.Keywords[i]); idx != -1 { - matchFound = true - column = idx + 1 - } - } +func check(err error) { + if err != nil { + log.Fatal(err) + } +} - if matchFound { - s.Lock() +func includesExtension(ext string, extensions, excluded []string) bool { + if slices.Contains(excluded, ext) { + return false + } - if positionTracking { - match := Match{ - Path: match.Path, - Line: lineNumber, - Column: column, - Keyword: s.Keywords[i], - } + return len(extensions) == 0 || slices.Contains(extensions, ext) +} - s.AllMatches = append(s.AllMatches, match) - } else { - s.KeywordMatches = append(s.KeywordMatches, match.Path) +func parseFile(path string, matcher *contentMatcher) (matches []Match, err error) { + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("open %q: %w", path, err) + } - found = true - } + defer func() { err = errors.Join(err, file.Close()) }() - s.Unlock() + matches, err = matcher.parse(file, path) - if !positionTracking { - break - } - } - } + if err != nil { + err = fmt.Errorf("read %q: %w", path, err) } - check(scanner.Err()) - - file.Close() -} - -// shouldExcludeDir checks if a directory name should be excluded -func (s *Scanner) shouldExcludeDir(dirName string) bool { - return slices.Contains(s.ExcludeDirs, dirName) + return matches, err } -// shouldExcludeExt checks if a file extension should be excluded -func (s *Scanner) shouldExcludeExt(ext string) bool { - return slices.Contains(s.ExcludeExts, ext) -} +const ( + filesPerChunk = 1024 + concurrentFiles = 128 +) func eachSlice(files []FileData) [][]FileData { var chunks [][]FileData - var chunk []FileData - for _, fileData := range files { - switch len(chunk) { - case 1024: - var newChunk []FileData - chunks = append(chunks, chunk) - chunk = newChunk - default: - chunk = append(chunk, fileData) + for start := 0; start < len(files); start += filesPerChunk { + chunks = append(chunks, files[start:min(start+filesPerChunk, len(files))]) + } + + return chunks +} + +// OutputOptions controls the CLI-compatible result format. +type OutputOptions struct { + ShowLines bool + ShowCols bool + ShowKeywords bool +} + +// WriteResults writes one path or occurrence per line and reports write errors. +func WriteResults(w io.Writer, result ScanResult, options OutputOptions) error { + if options.ShowLines || options.ShowCols { + for _, match := range result.Matches { + if _, err := fmt.Fprintln(w, formatMatch(match, options)); err != nil { + return err + } } + + return nil } - if len(chunk) < 1024 { - chunks = append(chunks, chunk) + for _, path := range result.Paths { + if _, err := fmt.Fprintln(w, path); err != nil { + return err + } } - return chunks + return nil } -func check(err error) { - if err != nil { - log.Fatal(err) +func formatMatch(match Match, options OutputOptions) string { + output := match.Path + + if options.ShowCols { + output = fmt.Sprintf("%s:%d:%d", output, match.Line, match.Column) + } else if options.ShowLines { + output = fmt.Sprintf("%s:%d", output, match.Line) } + + if options.ShowKeywords { + output += ":" + match.Keyword + } + + return output } diff --git a/pkg/scanner_test.go b/pkg/scanner_test.go new file mode 100644 index 0000000..fd4f12c --- /dev/null +++ b/pkg/scanner_test.go @@ -0,0 +1,358 @@ +package scnnr + +import ( + "bytes" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "slices" + "sync" + "testing" +) + +func fixturePath(parts ...string) string { + return filepath.Join(append([]string{"testdata", "tree"}, parts...)...) +} + +func TestSearchFilters(t *testing.T) { + all := []string{ + fixturePath("alpha.txt"), fixturePath("empty.txt"), fixturePath("ignored", "hidden.txt"), + fixturePath("nested", "beta.go"), fixturePath("nested", "quiet.txt"), fixturePath("skip.log"), + } + tests := []struct { + name string + extensions, excludeExts []string + excludeDirs []string + want []string + }{ + {name: "nil lists", want: all}, + {name: "legacy empty sentinel", extensions: []string{""}, want: all}, + { + name: "duplicate extension", extensions: []string{".txt", ".txt"}, + want: []string{all[0], all[1], all[2], all[4]}, + }, + { + name: "excluded directory and extension", + excludeDirs: []string{"ignored"}, excludeExts: []string{".log"}, + want: []string{all[0], all[1], all[3], all[4]}, + }, + {name: "exclusion wins", extensions: []string{".txt"}, excludeExts: []string{".txt"}}, + {name: "excluded root", excludeDirs: []string{"tree"}}, + {name: "unknown extension", extensions: []string{".missing"}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + scanner := Scanner{ + Directory: fixturePath(), FileExtensions: test.extensions, + ExcludeDirs: test.excludeDirs, ExcludeExts: test.excludeExts, + } + result, err := scanner.Search() + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(result.Paths, test.want) { + t.Fatalf("paths = %q; want %q", result.Paths, test.want) + } + if len(result.Matches) != 0 || len(result.Files) != len(test.want) { + t.Fatalf("unexpected listing result: %#v", result) + } + }) + } +} + +func TestSearchFiltersBeforeReadingMetadata(t *testing.T) { + root := t.TempDir() + want := writeFixture(t, root, "good.txt", "needle") + if err := os.Symlink(filepath.Join(root, "missing"), filepath.Join(root, "broken.log")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + for _, test := range []struct { + name string + extensions, excluded []string + }{ + {"excluded extension", nil, []string{".log"}}, + {"included extensions only", []string{".txt"}, nil}, + } { + t.Run(test.name, func(t *testing.T) { + scanner := Scanner{ + Directory: root, Keywords: []string{"needle"}, + FileExtensions: test.extensions, ExcludeExts: test.excluded, + } + result, err := scanner.Search() + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(result.Paths, []string{want}) { + t.Fatalf("paths=%q; want %q", result.Paths, want) + } + }) + } +} + +func TestSearchMatchesAreStableAndIndependent(t *testing.T) { + scanner := Scanner{ + Directory: fixturePath(), Keywords: []string{"needle", "token"}, ShowCols: true, + ExcludeDirs: []string{"ignored"}, ExcludeExts: []string{".log"}, + KeywordMatches: []string{"previous"}, AllMatches: []Match{{Path: "previous"}}, + } + want := []Match{ + {fixturePath("alpha.txt"), 1, 7, "needle"}, + {fixturePath("alpha.txt"), 1, 14, "needle"}, + {fixturePath("alpha.txt"), 2, 8, "token"}, + {fixturePath("nested", "beta.go"), 2, 4, "needle"}, + } + var wg sync.WaitGroup + for range 3 { + wg.Add(1) + go func() { + defer wg.Done() + result, err := scanner.Search() + if err != nil { + t.Error(err) + return + } + if !reflect.DeepEqual(result.Matches, want) { + t.Errorf("matches = %#v; want %#v", result.Matches, want) + } + if !reflect.DeepEqual(result.Paths, []string{fixturePath("alpha.txt"), fixturePath("nested", "beta.go")}) { + t.Errorf("paths = %q", result.Paths) + } + }() + } + wg.Wait() + if !reflect.DeepEqual(scanner.KeywordMatches, []string{"previous"}) || scanner.AllMatches[0].Path != "previous" { + t.Fatal("Search mutated legacy result fields") + } +} + +func TestSearchFirstMatchMode(t *testing.T) { + scanner := Scanner{ + Directory: fixturePath(), Keywords: []string{"needle"}, + ExcludeDirs: []string{"ignored"}, ExcludeExts: []string{".log"}, + } + result, err := scanner.Search() + if err != nil { + t.Fatal(err) + } + if len(result.Matches) != 2 || len(result.Paths) != 2 { + t.Fatalf("first-match results = %#v", result) + } +} + +func TestSearchErrors(t *testing.T) { + tests := []struct { + name string + scanner *Scanner + }{ + {"missing directory", &Scanner{Directory: filepath.Join(t.TempDir(), "missing")}}, + {"empty directory", &Scanner{}}, + {"invalid regex", &Scanner{Directory: fixturePath(), Keywords: []string{"["}, Regex: true}}, + {"lines require keywords", &Scanner{Directory: fixturePath(), ShowLines: true}}, + {"columns require keywords", &Scanner{Directory: fixturePath(), Keywords: []string{""}, ShowCols: true}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result, err := test.scanner.Search() + if err == nil { + t.Fatal("expected error") + } + if len(result.Files)+len(result.Paths)+len(result.Matches) != 0 { + t.Fatalf("failure returned partial results: %#v", result) + } + }) + } + matcher, err := newContentMatcher([]string{"needle"}, false, true) + if err != nil { + t.Fatal(err) + } + if _, err := parseFile(filepath.Join(t.TempDir(), "missing"), matcher); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("missing file error = %v", err) + } + if _, err := parseFile(t.TempDir(), matcher); err == nil { + t.Fatal("directory was accepted as readable content") + } +} + +func TestScanAccumulatesLegacyResults(t *testing.T) { + scanner := Scanner{Directory: fixturePath(), FileExtensions: []string{".go"}, Keywords: []string{"needle"}} + for scan := 1; scan <= 2; scan++ { + if err := scanner.ScanTo(&bytes.Buffer{}); err != nil { + t.Fatal(err) + } + if len(scanner.MatchedFilePaths) != scan || len(scanner.KeywordMatches) != scan*(scan+1)/2 { + t.Fatalf("original accumulation changed: files=%d paths=%q", len(scanner.MatchedFilePaths), scanner.KeywordMatches) + } + } + scanner.ShowCols = true + if err := scanner.ScanTo(&bytes.Buffer{}); err != nil { + t.Fatal(err) + } + if len(scanner.KeywordMatches) != 3 || len(scanner.AllMatches) != 3 { + t.Fatal("position scan did not retain original accumulated results") + } + scanner.ShowCols = false + scanner.Keywords = []string{""} + if err := scanner.ScanTo(&bytes.Buffer{}); err != nil { + t.Fatal(err) + } + if len(scanner.AllMatches) != 3 || len(scanner.KeywordMatches) != 3 || len(scanner.MatchedFilePaths) != 4 { + t.Fatal("listing scan cleared the original result fields") + } +} + +func TestEachSlicePreservesEveryFile(t *testing.T) { + for _, count := range []int{0, 1, 1023, 1024, 1025, 2048, 2049, 2050, 4096} { + t.Run(fmt.Sprint(count), func(t *testing.T) { + files := make([]FileData, count) + for i := range files { + files[i].Path = fmt.Sprint(i) + } + var got []FileData + for _, chunk := range eachSlice(files) { + if len(chunk) == 0 || len(chunk) > filesPerChunk { + t.Fatalf("invalid chunk length %d", len(chunk)) + } + got = append(got, chunk...) + } + if !slices.Equal(got, files) { + t.Fatalf("lost or reordered files: input=%d output=%d", len(files), len(got)) + } + }) + } +} + +func TestLegacyScanContracts(t *testing.T) { + t.Run("missing root returns walk error", func(t *testing.T) { + scanner := Scanner{Directory: filepath.Join(t.TempDir(), "missing"), Keywords: []string{""}, FileExtensions: []string{""}} + if err := scanner.ScanTo(&bytes.Buffer{}); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("walk error=%v", err) + } + }) + t.Run("nil keywords retain panic", func(t *testing.T) { + scanner := Scanner{Directory: t.TempDir(), FileExtensions: []string{""}} + expectPanic(t, func() { scanner.ScanTo(&bytes.Buffer{}) }) + }) + t.Run("nil extensions retain panic", func(t *testing.T) { + root := t.TempDir() + writeFixture(t, root, "sample", "needle") + scanner := Scanner{Directory: root, Keywords: []string{"needle"}} + expectPanic(t, func() { scanner.ScanTo(&bytes.Buffer{}) }) + }) + t.Run("duplicate extensions are retained", func(t *testing.T) { + root := t.TempDir() + path := writeFixture(t, root, "sample.txt", "needle") + scanner := Scanner{Directory: root, Keywords: []string{""}, FileExtensions: []string{".txt", ".txt"}} + var output bytes.Buffer + if err := scanner.ScanTo(&output); err != nil { + t.Fatal(err) + } + if output.String() != path+"\n"+path+"\n" || len(scanner.MatchedFilePaths) != 2 { + t.Fatalf("original duplicate extensions changed: %q", output.String()) + } + }) + t.Run("empty first keyword disables matching", func(t *testing.T) { + root := t.TempDir() + path := writeFixture(t, root, "sample", "unrelated") + scanner := Scanner{Directory: root, Keywords: []string{"", "needle"}, FileExtensions: []string{""}, ShowCols: true} + var output bytes.Buffer + if err := scanner.ScanTo(&output); err != nil || output.String() != path+"\n" { + t.Fatalf("empty sentinel output=%q error=%v", output.String(), err) + } + }) + t.Run("empty results retain newline", func(t *testing.T) { + scanner := Scanner{Directory: t.TempDir(), Keywords: []string{""}, FileExtensions: []string{""}} + var output bytes.Buffer + if err := scanner.ScanTo(&output); err != nil || output.String() != "\n" { + t.Fatalf("empty output=%q error=%v", output.String(), err) + } + }) + t.Run("full batch retains original boundary behavior", func(t *testing.T) { + scanner := Scanner{ + Directory: t.TempDir(), Keywords: []string{"needle"}, FileExtensions: []string{""}, + MatchedFilePaths: make([]FileData, 1024), + } + var output bytes.Buffer + if err := scanner.ScanTo(&output); err != nil || output.String() != "\n" || len(scanner.KeywordMatches) != 0 { + t.Fatalf("original 1024-file batch changed: output=%q error=%v", output.String(), err) + } + }) +} + +func TestLegacySlicesMatchOriginalBoundaryBehavior(t *testing.T) { + for _, test := range []struct{ input, output int }{ + {0, 0}, {1, 1}, {1023, 1023}, {1024, 0}, + {1025, 1024}, {2048, 2047}, {2049, 1024}, {2050, 2048}, + } { + count := 0 + for _, chunk := range legacySlices(make([]FileData, test.input)) { + count += len(chunk) + } + if count != test.output { + t.Errorf("legacy batch input=%d output=%d; want original %d", test.input, count, test.output) + } + } +} + +func TestSearchAtChunkBoundary(t *testing.T) { + root := t.TempDir() + for i := range 1025 { + if err := os.WriteFile(filepath.Join(root, fmt.Sprintf("%04d.txt", i)), []byte("needle\n"), 0600); err != nil { + t.Fatal(err) + } + } + scanner := Scanner{Directory: root, Keywords: []string{"needle"}} + result, err := scanner.Search() + if err != nil { + t.Fatal(err) + } + if len(result.Paths) != 1025 { + t.Fatalf("returned %d of 1025 matching files", len(result.Paths)) + } + for i, path := range result.Paths { + if path != filepath.Join(root, fmt.Sprintf("%04d.txt", i)) { + t.Fatalf("file %d missing or out of order: %q", i, path) + } + } +} + +func TestWriteResults(t *testing.T) { + result := ScanResult{ + Paths: []string{"a", "b"}, + Matches: []Match{{"a", 2, 3, "needle"}, {"b", 4, 5, "token"}}, + } + tests := []struct { + name string + options OutputOptions + want string + }{ + {"paths", OutputOptions{}, "a\nb\n"}, + {"lines", OutputOptions{ShowLines: true}, "a:2\nb:4\n"}, + {"columns imply lines", OutputOptions{ShowCols: true}, "a:2:3\nb:4:5\n"}, + {"columns take precedence", OutputOptions{ShowLines: true, ShowCols: true}, "a:2:3\nb:4:5\n"}, + {"keyword labels", OutputOptions{ShowLines: true, ShowKeywords: true}, "a:2:needle\nb:4:token\n"}, + {"columns and keywords", OutputOptions{ShowCols: true, ShowKeywords: true}, "a:2:3:needle\nb:4:5:token\n"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var out bytes.Buffer + if err := WriteResults(&out, result, test.options); err != nil { + t.Fatal(err) + } + if out.String() != test.want { + t.Fatalf("output = %q; want %q", out.String(), test.want) + } + }) + } + var empty bytes.Buffer + if err := WriteResults(&empty, ScanResult{}, OutputOptions{}); err != nil || empty.Len() != 0 { + t.Fatalf("empty result output=%q err=%v", empty.String(), err) + } + wantErr := errors.New("output failed") + for _, options := range []OutputOptions{{}, {ShowCols: true}} { + if err := WriteResults(failingWriter{wantErr}, result, options); !errors.Is(err, wantErr) { + t.Fatalf("write error = %v", err) + } + } +} diff --git a/pkg/testdata/tree/alpha.txt b/pkg/testdata/tree/alpha.txt new file mode 100644 index 0000000..e3635c3 --- /dev/null +++ b/pkg/testdata/tree/alpha.txt @@ -0,0 +1,2 @@ +hello needle needle +second token diff --git a/pkg/testdata/tree/empty.txt b/pkg/testdata/tree/empty.txt new file mode 100644 index 0000000..e69de29 diff --git a/pkg/testdata/tree/ignored/hidden.txt b/pkg/testdata/tree/ignored/hidden.txt new file mode 100644 index 0000000..a6b681b --- /dev/null +++ b/pkg/testdata/tree/ignored/hidden.txt @@ -0,0 +1 @@ +needle diff --git a/pkg/testdata/tree/nested/beta.go b/pkg/testdata/tree/nested/beta.go new file mode 100644 index 0000000..ade7142 --- /dev/null +++ b/pkg/testdata/tree/nested/beta.go @@ -0,0 +1,2 @@ +package sample +// needle diff --git a/pkg/testdata/tree/nested/quiet.txt b/pkg/testdata/tree/nested/quiet.txt new file mode 100644 index 0000000..40b29c3 --- /dev/null +++ b/pkg/testdata/tree/nested/quiet.txt @@ -0,0 +1 @@ +no matching words diff --git a/pkg/testdata/tree/skip.log b/pkg/testdata/tree/skip.log new file mode 100644 index 0000000..d69bf2b --- /dev/null +++ b/pkg/testdata/tree/skip.log @@ -0,0 +1 @@ +needle token diff --git a/pkg/walk.go b/pkg/walk.go new file mode 100644 index 0000000..1c9eb16 --- /dev/null +++ b/pkg/walk.go @@ -0,0 +1,75 @@ +package scnnr + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "slices" + "sync" +) + +// walkFiles shares traversal and error handling across scanner modes. Symlinked +// files are read through their targets; symlinked directories are not traversed. +func walkFiles(root string, excludeDirs []string, include func(string) bool, visit func(FileData) error) error { + if root == "" { + return errors.New("directory is required") + } + + return filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + + if entry.IsDir() { + if slices.Contains(excludeDirs, entry.Name()) { + return filepath.SkipDir + } + return nil + } + + // Apply path filters before stat so excluded entries require no access. + if include != nil && !include(path) { + return nil + } + + info, err := entry.Info() + if entry.Type()&os.ModeSymlink != 0 { + info, err = os.Stat(path) + } + + if err != nil { + return err + } + + if info.IsDir() { + return nil + } + + return visit(FileData{Path: path, Info: info}) + }) +} + +// walkFinderFiles retains the finders' original parallel traversal: failed +// directory reads and nil file metadata are skipped, and sibling walks continue. +func walkFinderFiles(directory, direction string, visit func(string, os.FileInfo)) { + files, dirs := CollectFilesAndDirs(directory, direction) + for _, file := range files { + if file != nil { + visit(directory, file) + } + } + + var wg sync.WaitGroup + + wg.Add(len(dirs)) + + for _, dir := range dirs { + go func() { + defer wg.Done() + walkFinderFiles(directory+direction+dir.Name(), direction, visit) + }() + } + + wg.Wait() +} diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 22ce0f5..be40b95 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -1,118 +1,70 @@ #!/usr/bin/env bash -set -eou pipefail - -e2e_dir=$HOME - -if [[ $ETE_DIR != "" ]] -then - e2e_dir=$ETE_DIR -fi - -################################################## -################################################## -################################################## - -echo "--- FILE SIZE FINDER DRY RUN: BEGIN ---" - -go run main.go -m fsf -s 1MB -d $e2e_dir - -echo "--- FILE SIZE FINDER DRY RUN: DONE ---" - -sleep 2 - -################################################## -################################################## -################################################## - -echo "--- FILE NAME FINDER DRY RUN: BEGIN---" - -go run main.go -m fnf -f main,DEFCON -p $e2e_dir - -echo "--- FILE NAME FINDER DRY RUN: DONE ---" - -sleep 2 - -################################################## -################################################## -################################################## - -echo "--- SCANNER DRY RUN: BEGIN---" - -go run main.go -k main,const,let,var,for -p $e2e_dir - -echo "--- SCANNER DRY RUN: DONE ---" - -sleep 2 - -################################################## -################################################## -################################################## - -echo "--- FILE FINGERPRINT FINDER DRY RUN: BEGIN---" - -go run main.go -m fff -k $(go run cmd/checksum/main.go) -d . - -echo "--- FILE FINGERPRINT FINDER DRY RUN: DONE---" - -sleep 2 - -################################################## -################################################## -################################################## - -echo "--- SCANNER LINE RUN: BEGIN---" - -go run main.go -k main,const,let,var,for -p $e2e_dir -l - -echo "--- SCANNER LINE RUN: DONE ---" - -sleep 2 - -################################################## -################################################## -################################################## - -echo "--- SCANNER LINE AND COL RUN: BEGIN---" - -go run main.go -k main,const,let,var,for -p $e2e_dir -c - -echo "--- SCANNER LINE AND COL RUN: DONE ---" - -sleep 2 - -################################################## -################################################## -################################################## - -echo "--- SCANNER DIR EXCLUDE RUN: BEGIN---" - -sleep 2 - -go run main.go -k main,const,let,var,for -p $e2e_dir -c -xd ".git" - -echo "--- SCANNER DIR EXCLUDE RUN: DONE ---" - -sleep 2 - -################################################## -################################################## -################################################## - -echo "--- SCANNER EXTENSION EXCLUDE RUN: BEGIN---" - -go run main.go -k main,const,let,var,for -p $e2e_dir -c -xe ".yml,.sh,.md" - -echo "--- SCANNER EXTENSION RUN: DONE ---" - -sleep 2 - -################################################## -################################################## -################################################## - -echo "--- SCANNER DIRECTORY AND EXTENSION EXCLUDE RUN: BEGIN---" - -go run main.go -k main,const,let,var,for -p $e2e_dir -c -xd ".git" -xe ".yml,.sh,.md" - -echo "--- SCANNER DIRECTORY AND EXTENSION RUN: DONE ---" +set -euo pipefail + +fixture_dir=$(mktemp -d "${ETE_DIR:-${TMPDIR:-/tmp}}/scnnr-e2e.XXXXXX") + +trap 'rm -rf "$fixture_dir"' EXIT + +mkdir -p "$fixture_dir/nested" "$fixture_dir/ignored" + +printf 'needle needle\ntoken\n' > "$fixture_dir/alpha.txt" +printf 'needle\n' > "$fixture_dir/nested/beta.txt" +printf 'needle\n' > "$fixture_dir/ignored/hidden.txt" +printf 'needle\n' > "$fixture_dir/skip.log" +printf 'abc' > "$fixture_dir/abc.bin" + +dd if=/dev/zero of="$fixture_dir/large.bin" bs=1000000 count=1 2>/dev/null + +assert_output() { + local expected=$1 + shift + local actual + + # The original scanner appends matches from concurrent file readers. + actual=$(go run main.go "$@" | LC_ALL=C sort) + expected=$(printf '%s' "$expected" | LC_ALL=C sort) + + if [[ "$actual" != "$expected" ]]; then + printf 'Unexpected output for %s\nExpected:\n%s\nActual:\n%s\n' "$*" "$expected" "$actual" >&2 + exit 1 + fi +} + +assert_output "$fixture_dir/alpha.txt"$'\n'"$fixture_dir/nested/beta.txt" \ + -d "$fixture_dir" -e .txt -k needle -xd ignored + +assert_output "$fixture_dir/alpha.txt:1:1"$'\n'"$fixture_dir/nested/beta.txt:1:1" \ + -d "$fixture_dir" -e .txt -k needle -c -xd ignored + +assert_output "$fixture_dir/alpha.txt:1:needle"$'\n'"$fixture_dir/alpha.txt:2:token"$'\n'"$fixture_dir/nested/beta.txt:1:needle" \ + -d "$fixture_dir" -e .txt -k needle,token -l -xd ignored + +assert_output "$fixture_dir/alpha.txt"$'\n'"$fixture_dir/nested/beta.txt" \ + -d "$fixture_dir" -k 'n.edle' -r -xd ignored -xe .log + +assert_output "$fixture_dir${fixture_dir}alpha.txt" \ + -m fnf -p "$fixture_dir" -f alpha + +assert_output "$fixture_dir${fixture_dir}large.bin" \ + -m fsf -d "$fixture_dir" -s 1MB + +assert_output "$fixture_dir/abc.bin" \ + -m fff -d "$fixture_dir" -k ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad + +assert_output "" -d "$fixture_dir" -k absent + +for failure in missing-directory invalid-regex missing-keywords; do + case "$failure" in + missing-directory) args=(-d "$fixture_dir/missing") ;; + invalid-regex) args=(-d "$fixture_dir" -k '[' -r) ;; + missing-keywords) args=(-c) ;; + esac + + if go run main.go "${args[@]}" >/dev/null 2>&1; then + printf 'Expected failure for %s\n' "$failure" >&2 + exit 1 + fi +done + +printf 'CLI fixtures passed\n'