From bc212e9b9a8c26fceb45e419ab027d018a41b696 Mon Sep 17 00:00:00 2001 From: Jeremy De La Cruz Date: Fri, 30 Dec 2022 20:29:04 +0000 Subject: [PATCH 1/4] Add disable-ignore-traversal flag to opt out of recursive wokeignore traversal --- .pre-commit-config.yaml | 2 +- cmd/root.go | 18 ++++++++++-------- cmd/root_test.go | 7 +++++-- pkg/ignore/ignore.go | 34 +++++++++++++++++++++++++++++----- pkg/ignore/ignore_test.go | 6 +++--- pkg/parser/parser_test.go | 6 +++--- 6 files changed, 51 insertions(+), 22 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ca55abcb..5f45a837 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ # First: Install pre-commit https://pre-commit.com/#install # Then, run `pre-commit install` repos: - - repo: git://github.com/caitlinelfring/pre-commit-golang + - repo: https://github.com/caitlinelfring/pre-commit-golang rev: v0.4.0 hooks: - id: go-fmt diff --git a/cmd/root.go b/cmd/root.go index e4822d03..dd947eb9 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -45,13 +45,14 @@ import ( var ( // flags - exitOneOnFailure bool - cfgFile string - debug bool - stdin bool - outputName string - noIgnore bool - disableDefaultRules bool + exitOneOnFailure bool + cfgFile string + debug bool + stdin bool + outputName string + noIgnore bool + disableDefaultRules bool + disableIgnoreTraversal bool // Version is populated by goreleaser during build // Version... @@ -109,7 +110,7 @@ func rootRunE(cmd *cobra.Command, args []string) error { if err != nil { return err } - ignorer, err = ignore.NewIgnore(fs, cfg.IgnoreFiles) + ignorer, err = ignore.NewIgnore(fs, cfg.IgnoreFiles, disableIgnoreTraversal) if err != nil { return err } @@ -155,6 +156,7 @@ func init() { rootCmd.PersistentFlags().BoolVar(&noIgnore, "no-ignore", false, "Ignored files in .gitignore, .ignore, .wokeignore, .git/info/exclude, and inline ignores are processed") rootCmd.PersistentFlags().StringVarP(&outputName, "output", "o", printer.OutFormatText, fmt.Sprintf("Output type [%s]", printer.OutFormatsString)) rootCmd.PersistentFlags().BoolVar(&disableDefaultRules, "disable-default-rules", false, "Disable the default ruleset") + rootCmd.PersistentFlags().BoolVar(&disableIgnoreTraversal, "disable-ignore-traversal", false, "Disable nested woke ignore traversal") } // GetRootCmd returns the rootCmd, which should only be used by the docs generator in cmd/docs/main.go diff --git a/cmd/root_test.go b/cmd/root_test.go index 87071e7e..d3dca57d 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -20,9 +20,12 @@ import ( // run profiling with // go test -v -cpuprofile cpu.prof -memprofile mem.prof -bench=. ./cmd // memory: -// go tool pprof mem.prof +// +// go tool pprof mem.prof +// // cpu: -// go tool pprof cpu.prof +// +// go tool pprof cpu.prof func BenchmarkRootRunE(b *testing.B) { zerolog.SetGlobalLevel(zerolog.NoLevel) output.Stdout = io.Discard diff --git a/pkg/ignore/ignore.go b/pkg/ignore/ignore.go index 6be035de..fa3f6066 100644 --- a/pkg/ignore/ignore.go +++ b/pkg/ignore/ignore.go @@ -1,6 +1,8 @@ package ignore import ( + "errors" + "io/ioutil" "os" "path/filepath" "strings" @@ -65,7 +67,7 @@ func GetRootGitDir(workingDir string) (filesystem billy.Filesystem, err error) { // NewIgnore produces an Ignore object, with compiled lines from defaultIgnoreFiles // which you can match files against -func NewIgnore(filesystem billy.Filesystem, lines []string) (ignore *Ignore, err error) { +func NewIgnore(filesystem billy.Filesystem, lines []string, disableIgnoreTraversal bool) (ignore *Ignore, err error) { start := time.Now() defer func() { log.Debug(). @@ -79,8 +81,16 @@ func NewIgnore(filesystem billy.Filesystem, lines []string) (ignore *Ignore, err } var ps []gitignore.Pattern - if ps, err = gitignore.ReadPatterns(filesystem, nil, defaultIgnoreFiles); err != nil { - return + + // if opted-out of nested wokeignore traversal, only use top-level ignore files + if disableIgnoreTraversal { + for _, filename := range defaultIgnoreFiles { + lines = append(lines, readIgnoreFile(filename)...) + } + } else { + if ps, err = gitignore.ReadPatterns(filesystem, nil, defaultIgnoreFiles); err != nil { + return + } } // get domain for git ignore rules supplied from the lines argument @@ -90,11 +100,25 @@ func NewIgnore(filesystem billy.Filesystem, lines []string) (ignore *Ignore, err ps = append(ps, pattern) } - ignore = &Ignore{ + return &Ignore{ matcher: gitignore.NewMatcher(ps), + }, nil +} + +func readIgnoreFile(file string) []string { + buffer, err := ioutil.ReadFile(file) + if err != nil { + _event := log.Warn() + if errors.Is(err, os.ErrNotExist) { + _event = log.Debug() + } + _event.Err(err).Str("file", file).Msg("skipping ignorefile") + return []string{} } - return + log.Debug().Str("file", file).Msg("adding ignorefile") + + return strings.Split(strings.TrimSpace(string(buffer)), "\n") } // Match returns true if the provided file matches any of the defined ignores diff --git a/pkg/ignore/ignore_test.go b/pkg/ignore/ignore_test.go index e7be706f..4773cc33 100644 --- a/pkg/ignore/ignore_test.go +++ b/pkg/ignore/ignore_test.go @@ -103,7 +103,7 @@ func BenchmarkIgnore(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - ignorer, err := NewIgnore(fs, []string{}) + ignorer, err := NewIgnore(fs, []string{}, false) assert.NoError(b, err) ignorer.Match(filepath.Join("not", "foo"), false) } @@ -136,7 +136,7 @@ func (suite *IgnoreTestSuite) TestGetRootGitDirNotExist() { suite.Equal(fs.Root(), rootFs.Root()) } func (suite *IgnoreTestSuite) TestIgnore_Match() { - i, err := NewIgnore(suite.GFS, []string{"my/files/*"}) + i, err := NewIgnore(suite.GFS, []string{"my/files/*"}, false) suite.NoError(err) suite.NotNil(i) @@ -148,7 +148,7 @@ func (suite *IgnoreTestSuite) TestIgnore_Match() { // Test all default ignore files, except for .git/info/exclude, since // that uses a .git directory that we cannot check in. func (suite *IgnoreTestSuite) TestIgnoreDefaultIgoreFiles_Match() { - i, err := NewIgnore(suite.GFS, []string{"*.FROMARGUMENT"}) + i, err := NewIgnore(suite.GFS, []string{"*.FROMARGUMENT"}, false) suite.NoError(err) suite.NotNil(i) diff --git a/pkg/parser/parser_test.go b/pkg/parser/parser_test.go index 639395e7..69e1a989 100644 --- a/pkg/parser/parser_test.go +++ b/pkg/parser/parser_test.go @@ -43,7 +43,7 @@ func testParser() (parser *Parser, err error) { return } fs := osfs.New(cwd) - ignorer, err := ignore.NewIgnore(fs, []string{}) + ignorer, err := ignore.NewIgnore(fs, []string{}, false) if err != nil { return } @@ -149,7 +149,7 @@ func parsePathTests(t *testing.T) { cwd, err := os.Getwd() assert.NoError(t, err) fs := osfs.New(cwd) - ignorer, err := ignore.NewIgnore(fs, []string{filepath.ToSlash(f.Name())}) + ignorer, err := ignore.NewIgnore(fs, []string{filepath.ToSlash(f.Name())}, false) assert.NoError(t, err) p.Ignorer = ignorer pr := new(testPrinter) @@ -193,7 +193,7 @@ func parsePathTests(t *testing.T) { cwd, err := os.Getwd() assert.NoError(t, err) fs := osfs.New(cwd) - ignorer, err := ignore.NewIgnore(fs, []string{"*_test.go"}) + ignorer, err := ignore.NewIgnore(fs, []string{"*_test.go"}, false) assert.NoError(t, err) p.Ignorer = ignorer pr := new(testPrinter) From 4351098f9f4c8155721d3643833db23ad39f28ed Mon Sep 17 00:00:00 2001 From: Jeremy De La Cruz Date: Wed, 4 Jan 2023 21:34:40 +0000 Subject: [PATCH 2/4] Add additional benchmark and tests for ignore traversal flag --- pkg/ignore/ignore.go | 21 ++++++++--- pkg/ignore/ignore_test.go | 76 ++++++++++++++++++++++++++++++++++----- 2 files changed, 83 insertions(+), 14 deletions(-) diff --git a/pkg/ignore/ignore.go b/pkg/ignore/ignore.go index fa3f6066..267fed88 100644 --- a/pkg/ignore/ignore.go +++ b/pkg/ignore/ignore.go @@ -1,8 +1,9 @@ package ignore import ( + "bufio" "errors" - "io/ioutil" + "io" "os" "path/filepath" "strings" @@ -85,7 +86,7 @@ func NewIgnore(filesystem billy.Filesystem, lines []string, disableIgnoreTravers // if opted-out of nested wokeignore traversal, only use top-level ignore files if disableIgnoreTraversal { for _, filename := range defaultIgnoreFiles { - lines = append(lines, readIgnoreFile(filename)...) + lines = append(lines, readIgnoreFile(filesystem, filename)...) } } else { if ps, err = gitignore.ReadPatterns(filesystem, nil, defaultIgnoreFiles); err != nil { @@ -105,8 +106,9 @@ func NewIgnore(filesystem billy.Filesystem, lines []string, disableIgnoreTravers }, nil } -func readIgnoreFile(file string) []string { - buffer, err := ioutil.ReadFile(file) +func readIgnoreFile(filesystem billy.Filesystem, file string) []string { + openFile, err := filesystem.Open(file) + if err != nil { _event := log.Warn() if errors.Is(err, os.ErrNotExist) { @@ -116,8 +118,17 @@ func readIgnoreFile(file string) []string { return []string{} } - log.Debug().Str("file", file).Msg("adding ignorefile") + defer openFile.Close() + var buffer []byte + buffer, err = io.ReadAll(bufio.NewReader(openFile)) + if err != nil { + _event := log.Warn() + _event.Err(err).Str("file", file).Msg("skipping ignorefile") + return []string{} + } + + log.Debug().Str("file", file).Msg("adding ignorefile") return strings.Split(strings.TrimSpace(string(buffer)), "\n") } diff --git a/pkg/ignore/ignore_test.go b/pkg/ignore/ignore_test.go index 4773cc33..c4087172 100644 --- a/pkg/ignore/ignore_test.go +++ b/pkg/ignore/ignore_test.go @@ -75,6 +75,16 @@ func (suite *IgnoreTestSuite) SetupTest() { err = f.Close() suite.NoError(err) + err = fs.MkdirAll("nestedIgnoreFolder", os.ModePerm) + suite.NoError(err) + + f, err = fs.Create(fs.Join("nestedIgnoreFolder", ".wokeignore")) + suite.NoError(err) + _, err = f.Write([]byte("*.NESTEDIGNORE\n")) + suite.NoError(err) + err = f.Close() + suite.NoError(err) + suite.GFS = fs } @@ -82,8 +92,8 @@ func BenchmarkIgnore(b *testing.B) { zerolog.SetGlobalLevel(zerolog.NoLevel) fs, clean := TempFileSystem() defer clean() - for i := 0; i < 10; i++ { - for j := 0; j < 10; j++ { + for i := 0; i < 50; i++ { + for j := 0; j < 50; j++ { err := fs.MkdirAll(fs.Join(fmt.Sprintf("%d", i), fmt.Sprintf("%d", j)), os.ModePerm) assert.NoError(b, err) f, err := fs.Create(".wokeignore") @@ -102,11 +112,20 @@ func BenchmarkIgnore(b *testing.B) { } b.ResetTimer() - for i := 0; i < b.N; i++ { - ignorer, err := NewIgnore(fs, []string{}, false) - assert.NoError(b, err) - ignorer.Match(filepath.Join("not", "foo"), false) - } + b.Run("ignore-traversal-enabled", func(b *testing.B) { + for i := 0; i < b.N; i++ { + ignorer, err := NewIgnore(fs, []string{}, false) + assert.NoError(b, err) + ignorer.Match(filepath.Join("not", "foo"), false) + } + }) + b.Run("ignore-traversal-disabled", func(b *testing.B) { + for i := 0; i < b.N; i++ { + ignorer, err := NewIgnore(fs, []string{}, true) + assert.NoError(b, err) + ignorer.Match(filepath.Join("not", "foo"), false) + } + }) } func (suite *IgnoreTestSuite) TestGetDomainFromWorkingDir() { @@ -135,7 +154,8 @@ func (suite *IgnoreTestSuite) TestGetRootGitDirNotExist() { suite.NoError(err) suite.Equal(fs.Root(), rootFs.Root()) } -func (suite *IgnoreTestSuite) TestIgnore_Match() { + +func (suite *IgnoreTestSuite) TestIgnoreLines_Match() { i, err := NewIgnore(suite.GFS, []string{"my/files/*"}, false) suite.NoError(err) suite.NotNil(i) @@ -145,13 +165,43 @@ func (suite *IgnoreTestSuite) TestIgnore_Match() { suite.False(i.Match(filepath.Join("my", "files"), false)) } +func (suite *IgnoreTestSuite) TestIgnoreLinesNoTraversal_Match() { + i, err := NewIgnore(suite.GFS, []string{"my/files/*"}, true) + suite.NoError(err) + suite.NotNil(i) + + suite.False(i.Match(filepath.Join("not", "foo"), false)) + suite.True(i.Match(filepath.Join("my", "files", "file1"), false)) + suite.False(i.Match(filepath.Join("my", "files"), false)) +} + // Test all default ignore files, except for .git/info/exclude, since // that uses a .git directory that we cannot check in. -func (suite *IgnoreTestSuite) TestIgnoreDefaultIgoreFiles_Match() { +func (suite *IgnoreTestSuite) TestIgnoreDefaultIgnoreFiles_Match() { i, err := NewIgnore(suite.GFS, []string{"*.FROMARGUMENT"}, false) suite.NoError(err) suite.NotNil(i) + // Test top-level ignore files all match + suite.testCommonIgnoreDefaultIgnoreFilesMatch(i) + + // Test match from the nested ./nestedIgnoreFolder/.wokeignore when ignore traversal is enabled + suite.True(i.Match(filepath.Join("nestedIgnoreFolder", "testdata", "test.NESTEDIGNORE"), false)) +} + +func (suite *IgnoreTestSuite) TestIgnoreDefaultIgnoreFilesNoTraversal_Match() { + i, err := NewIgnore(suite.GFS, []string{"*.FROMARGUMENT"}, true) + suite.NoError(err) + suite.NotNil(i) + + // Test top-level ignore files all match + suite.testCommonIgnoreDefaultIgnoreFilesMatch(i) + + // Test no match from the nested ./nestedIgnoreFolder/.wokeignore when ignore traversal is disabled + suite.False(i.Match(filepath.Join("nestedIgnoreFolder", "testdata", "test.NESTEDIGNORE"), false)) +} + +func (suite *IgnoreTestSuite) testCommonIgnoreDefaultIgnoreFilesMatch(i *Ignore) { suite.False(i.Match(filepath.Join("testdata", "notfoo"), false)) suite.True(i.Match(filepath.Join("testdata", "test.FROMARGUMENT"), false)) // From .gitignore suite.True(i.Match(filepath.Join("testdata", "test.DS_Store"), false)) // From .gitignore @@ -162,6 +212,14 @@ func (suite *IgnoreTestSuite) TestIgnoreDefaultIgoreFiles_Match() { suite.False(i.Match(filepath.Join("testdata", "test.NOTIGNORED"), false)) // From .notincluded - making sure only default are included } +func (suite *IgnoreTestSuite) TestReadIgnoreFile() { + ignoreLines := readIgnoreFile(suite.GFS, ".gitignore") + suite.Equal([]string{"*.DS_Store"}, ignoreLines) + + noIgnoreLines := readIgnoreFile(suite.GFS, "missing.gitignore") + suite.Equal([]string{}, noIgnoreLines) +} + // In order for 'go test' to run this suite, we need to create // a normal test function and pass our suite to suite.Run func TestIgnoreTestSuite(t *testing.T) { From 94b02056ca6a848b8b34226f6e3c3b448329340e Mon Sep 17 00:00:00 2001 From: Jeremy De La Cruz Date: Wed, 4 Jan 2023 21:48:34 +0000 Subject: [PATCH 3/4] Rename flag to --disable-nested-ignores --- cmd/root.go | 20 ++++++++++---------- pkg/ignore/ignore.go | 4 ++-- pkg/ignore/ignore_test.go | 8 ++++---- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index dd947eb9..d311ebae 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -45,14 +45,14 @@ import ( var ( // flags - exitOneOnFailure bool - cfgFile string - debug bool - stdin bool - outputName string - noIgnore bool - disableDefaultRules bool - disableIgnoreTraversal bool + exitOneOnFailure bool + cfgFile string + debug bool + stdin bool + outputName string + noIgnore bool + disableDefaultRules bool + disableNestedIgnores bool // Version is populated by goreleaser during build // Version... @@ -110,7 +110,7 @@ func rootRunE(cmd *cobra.Command, args []string) error { if err != nil { return err } - ignorer, err = ignore.NewIgnore(fs, cfg.IgnoreFiles, disableIgnoreTraversal) + ignorer, err = ignore.NewIgnore(fs, cfg.IgnoreFiles, disableNestedIgnores) if err != nil { return err } @@ -156,7 +156,7 @@ func init() { rootCmd.PersistentFlags().BoolVar(&noIgnore, "no-ignore", false, "Ignored files in .gitignore, .ignore, .wokeignore, .git/info/exclude, and inline ignores are processed") rootCmd.PersistentFlags().StringVarP(&outputName, "output", "o", printer.OutFormatText, fmt.Sprintf("Output type [%s]", printer.OutFormatsString)) rootCmd.PersistentFlags().BoolVar(&disableDefaultRules, "disable-default-rules", false, "Disable the default ruleset") - rootCmd.PersistentFlags().BoolVar(&disableIgnoreTraversal, "disable-ignore-traversal", false, "Disable nested woke ignore traversal") + rootCmd.PersistentFlags().BoolVar(&disableNestedIgnores, "disable-nested-ignores", false, "Disable nested woke ignore traversal") } // GetRootCmd returns the rootCmd, which should only be used by the docs generator in cmd/docs/main.go diff --git a/pkg/ignore/ignore.go b/pkg/ignore/ignore.go index 267fed88..0c6c5c29 100644 --- a/pkg/ignore/ignore.go +++ b/pkg/ignore/ignore.go @@ -68,7 +68,7 @@ func GetRootGitDir(workingDir string) (filesystem billy.Filesystem, err error) { // NewIgnore produces an Ignore object, with compiled lines from defaultIgnoreFiles // which you can match files against -func NewIgnore(filesystem billy.Filesystem, lines []string, disableIgnoreTraversal bool) (ignore *Ignore, err error) { +func NewIgnore(filesystem billy.Filesystem, lines []string, disableNestedIgnores bool) (ignore *Ignore, err error) { start := time.Now() defer func() { log.Debug(). @@ -84,7 +84,7 @@ func NewIgnore(filesystem billy.Filesystem, lines []string, disableIgnoreTravers var ps []gitignore.Pattern // if opted-out of nested wokeignore traversal, only use top-level ignore files - if disableIgnoreTraversal { + if disableNestedIgnores { for _, filename := range defaultIgnoreFiles { lines = append(lines, readIgnoreFile(filesystem, filename)...) } diff --git a/pkg/ignore/ignore_test.go b/pkg/ignore/ignore_test.go index c4087172..0423c49c 100644 --- a/pkg/ignore/ignore_test.go +++ b/pkg/ignore/ignore_test.go @@ -112,14 +112,14 @@ func BenchmarkIgnore(b *testing.B) { } b.ResetTimer() - b.Run("ignore-traversal-enabled", func(b *testing.B) { + b.Run("nested-ignores-enabled", func(b *testing.B) { for i := 0; i < b.N; i++ { ignorer, err := NewIgnore(fs, []string{}, false) assert.NoError(b, err) ignorer.Match(filepath.Join("not", "foo"), false) } }) - b.Run("ignore-traversal-disabled", func(b *testing.B) { + b.Run("nested-ignores-disabled", func(b *testing.B) { for i := 0; i < b.N; i++ { ignorer, err := NewIgnore(fs, []string{}, true) assert.NoError(b, err) @@ -185,7 +185,7 @@ func (suite *IgnoreTestSuite) TestIgnoreDefaultIgnoreFiles_Match() { // Test top-level ignore files all match suite.testCommonIgnoreDefaultIgnoreFilesMatch(i) - // Test match from the nested ./nestedIgnoreFolder/.wokeignore when ignore traversal is enabled + // Test match from the nested ./nestedIgnoreFolder/.wokeignore when nested ignores is enabled suite.True(i.Match(filepath.Join("nestedIgnoreFolder", "testdata", "test.NESTEDIGNORE"), false)) } @@ -197,7 +197,7 @@ func (suite *IgnoreTestSuite) TestIgnoreDefaultIgnoreFilesNoTraversal_Match() { // Test top-level ignore files all match suite.testCommonIgnoreDefaultIgnoreFilesMatch(i) - // Test no match from the nested ./nestedIgnoreFolder/.wokeignore when ignore traversal is disabled + // Test no match from the nested ./nestedIgnoreFolder/.wokeignore when nested ignores is disabled suite.False(i.Match(filepath.Join("nestedIgnoreFolder", "testdata", "test.NESTEDIGNORE"), false)) } From 495c36a3dab101b26201d11cf8813b4b160bdbc8 Mon Sep 17 00:00:00 2001 From: Jeremy De La Cruz Date: Wed, 4 Jan 2023 21:54:20 +0000 Subject: [PATCH 4/4] Add note in readme about new flag --- docs/ignore.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/ignore.md b/docs/ignore.md index bf5d6503..dca63221 100644 --- a/docs/ignore.md +++ b/docs/ignore.md @@ -79,6 +79,8 @@ func main() { `woke` will apply ignore rules from nested ignore files to any child files/folders, similar to a nested `.gitignore` file. Nested ignore files work for any ignore file type listed above. +>Note: To disable nested ignore file functionality, run `woke` with the `--disable-nested-ignores` flag. + ```txt project │ README.md