diff --git a/README.md b/README.md index 03a2add..de938af 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ To check license headers in GitHub Actions, add a step in your GitHub workflow. # log: debug # optional: set the log level. The default value is `info`. # config: .licenserc.yaml # optional: set the config file. The default value is `.licenserc.yaml`. # token: # optional: the token that license eye uses when it needs to comment on the pull request. Set to empty ("") to disable commenting on pull request. The default value is ${{ github.token }} - # mode: # optional: Which mode License-Eye should be run in. Choices are `check` or `fix`. The default value is `check`. + # mode: # optional: Which mode License-Eye should be run in. Choices are `check`, `fix` or `diff`. The default value is `check`. ``` #### Fix License Headers @@ -308,6 +308,34 @@ INFO Totally checked 20 files, valid: 10, invalid: 10, ignored: 0, fixed: 10 +#### Diff License Header + +This command shows where the license headers of the invalid files differ from the license configured in the config file, to help understand why `header check` fails, for example, to spot a typo in an existing license header. + +```bash +license-eye -c .licenserc.yaml header diff +``` + +
+Header Diff Result + +For a `test.go` whose license header has a typo `wwwhttp://www.apache.org/licenses/LICENSE-2.0` in the license URL, and a `missing.py` that doesn't have a license header at all: + +``` +INFO Loading configuration from file: .licenserc.yaml +missing.py: + [-licensed to the asf under one or more contributor license ... the specific language governing permissions and limitations under the license.-] ... +test.go: + ... copy of the license at [-http://www.apache.org/licenses/license-2.0-] {+wwwhttp://www.apache.org/licenses/license-2.0+} unless required by applicable law ... and limitations under the license. ... +INFO Totally checked 3 files, valid: 0, invalid: 2, ignored: 1, fixed: 0 +ERROR one or more files does not have a valid license header +exit status 1 +``` + +
+ +The texts are compared in their normalized forms (comment markers stripped, whitespace flattened, case-insensitive, etc., the same forms that `header check` compares), so every difference shown is a real cause of the check failure: `[-text-]` marks text that is expected by the configured license but missing in the file, `{+text+}` marks text that is in the file but not expected by the configured license, and long runs of unchanged or missing words are collapsed into `...`. + #### Resolve Dependencies' licenses This command assists human audits of the dependencies licenses. It's exit code is always 0. @@ -858,7 +886,7 @@ header: ## Supported File Types -The `header check` command theoretically supports all kinds of file types, while the supported file types of `header fix` command can be found [in this YAML file](assets/languages.yaml). In the YAML file, if the language has a non-empty property `comment_style_id`, and the comment style id is declared in [the comment styles file](assets/styles.yaml), then the language is supported by `fix` command. +The `header check` and `header diff` commands theoretically support all kinds of file types, while the supported file types of `header fix` command can be found [in this YAML file](assets/languages.yaml). In the YAML file, if the language has a non-empty property `comment_style_id`, and the comment style id is declared in [the comment styles file](assets/styles.yaml), then the language is supported by `fix` command. - [assets/languages.yaml](assets/languages.yaml) diff --git a/action.yml b/action.yml index ada8764..b551350 100644 --- a/action.yml +++ b/action.yml @@ -37,7 +37,7 @@ inputs: default: ${{ github.token }} mode: description: | - Which mode License Eye should be run in. Choices are `check` or `fix`. The + Which mode License Eye should be run in. Choices are `check`, `fix` or `diff`. The default value is `check`. required: false default: check diff --git a/commands/header.go b/commands/header.go index 062f873..fa95349 100644 --- a/commands/header.go +++ b/commands/header.go @@ -33,4 +33,5 @@ var Header = &cobra.Command{ func init() { Header.AddCommand(CheckCommand) Header.AddCommand(FixCommand) + Header.AddCommand(DiffCommand) } diff --git a/commands/header_diff.go b/commands/header_diff.go new file mode 100644 index 0000000..9ee4517 --- /dev/null +++ b/commands/header_diff.go @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package commands + +import ( + "fmt" + "sort" + "strings" + + "github.com/spf13/cobra" + + "github.com/apache/skywalking-eyes/pkg/header" + "github.com/apache/skywalking-eyes/pkg/logger" +) + +var DiffCommand = &cobra.Command{ + Use: "diff [paths...]", + Aliases: []string{"d"}, + Long: "diff command walks the specified paths recursively and shows where the " + + "license headers of the invalid files differ from the license header in the " + + "config file, to help understand why the check command fails. " + + "Accepts files, directories, and glob patterns. " + + "If no paths are specified, checks the current directory " + + "recursively as defined in the config file. " + + "The texts are compared in the same normalized forms that the check command " + + "compares (comment markers stripped, whitespace flattened, case-insensitive, etc.), " + + "so every difference shown is a real cause of the check failure: " + + "[-text-] is expected by the configured license but missing in the file, " + + "{+text+} is in the file but not expected by the configured license.", + RunE: func(_ *cobra.Command, args []string) error { + hasErrors := false + var errors []string + for _, h := range Config.Headers() { + var result header.Result + + if len(args) > 0 { + logger.Log.Debugln("Overriding paths with command line args.") + h.Paths = args + } + + if err := header.Check(h, &result); err != nil { + return err + } + + sort.Strings(result.Failure) + for _, file := range result.Failure { + diff, err := header.DiffFile(file, h) + if err != nil { + errors = append(errors, err.Error()) + continue + } + if diff == "" { + continue + } + fmt.Printf("%v:\n\t%v\n", file, diff) + } + + logger.Log.Infoln(result.String()) + + if result.HasFailure() { + hasErrors = true + } + } + if len(errors) > 0 { + return fmt.Errorf("%s", strings.Join(errors, "\n")) + } + if hasErrors { + return fmt.Errorf("one or more files does not have a valid license header") + } + return nil + }, +} diff --git a/go.mod b/go.mod index b27a801..3bfc3c7 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/go-git/go-git/v5 v5.19.1 github.com/google/go-github/v33 v33.0.0 github.com/google/licensecheck v0.3.1 + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.6.1 github.com/stretchr/testify v1.11.1 @@ -43,7 +44,6 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/pjbgf/sha1cd v0.6.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/shopspring/decimal v1.3.1 // indirect github.com/skeema/knownhosts v1.3.1 // indirect github.com/spf13/cast v1.5.0 // indirect diff --git a/header/action.yml b/header/action.yml index 54e72b1..ebd50cf 100644 --- a/header/action.yml +++ b/header/action.yml @@ -37,7 +37,7 @@ inputs: default: ${{ github.token }} mode: description: | - Which mode License Eye should be run in. Choices are `check` or `fix`. The + Which mode License Eye should be run in. Choices are `check`, `fix` or `diff`. The default value is `check`. required: false default: check diff --git a/pkg/header/diff.go b/pkg/header/diff.go new file mode 100644 index 0000000..51a91ea --- /dev/null +++ b/pkg/header/diff.go @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package header + +import ( + "fmt" + "net/http" + "os" + "strings" + "unicode/utf8" + + lcs "github.com/apache/skywalking-eyes/pkg/license" + + "github.com/sergi/go-diff/diffmatchpatch" +) + +// DiffFile compares the license header of the file with the license configured +// in the config file, and returns a word-level diff of the two normalized texts, +// the same texts that CheckFile compares, so every difference in the diff is a +// real cause of the check failure. +// +// In the diff, [-text-] marks text that is expected by the configured license +// but missing in the file, and {+text+} marks text that is in the file but not +// expected by the configured license. An empty diff is returned when the file's +// license header is valid. +func DiffFile(file string, config *ConfigHeader) (string, error) { + expected := config.NormalizedLicense() + if expected == "" { + return "", fmt.Errorf("no license content configured (spdx-id or content) to diff against") + } + + bs, err := os.ReadFile(file) + if err != nil { + return "", err + } + if t := http.DetectContentType(bs); !strings.HasPrefix(t, "text/") { + return "", fmt.Errorf("not a text file: %v (%v)", file, t) + } + + content := lcs.NormalizeHeader(string(bs)) + if satisfy(content, config, expected, config.NormalizedPattern()) { + return "", nil + } + + if index := strings.Index(content, expected); index >= 0 { + return fmt.Sprintf( + "license header is found at normalized offset %d, which exceeds the license-location-threshold %d, move it closer to the file start", + index, config.LicenseLocationThreshold, + ), nil + } + + // Only diff the region of the file where the license header is allowed to live, + // the content after that region cannot contribute to a successful match anyway. + end := len(expected) + config.LicenseLocationThreshold + if end >= len(content) { + end = len(content) + } else { + for end > 0 && !utf8.RuneStart(content[end]) { + end-- + } + } + + return renderDiff(wordDiff(expected, content[:end])), nil +} + +// wordDiff diffs the two texts word by word, by mapping every word to a "line" +// and reusing the line-mode diff of diffmatchpatch. +func wordDiff(expected, actual string) []diffmatchpatch.Diff { + dmp := diffmatchpatch.New() + + // The trailing "\n" makes the last word a complete "line" too, so that it can + // match its occurrences in the middle of the other text. + e, a, words := dmp.DiffLinesToChars( + strings.ReplaceAll(expected, " ", "\n")+"\n", + strings.ReplaceAll(actual, " ", "\n")+"\n", + ) + diffs := dmp.DiffMain(e, a, false) + + return dmp.DiffCharsToLines(diffs, words) +} + +func renderDiff(diffs []diffmatchpatch.Diff) string { + const ( + // contextWords is the number of words to keep on each side when a long run of words is collapsed. + contextWords = 5 + // maxEqualRun is the maximum number of words an unchanged run can have before being collapsed. + maxEqualRun = 12 + // maxChangedRun is the maximum number of words a changed run can have before being collapsed, + // it's larger than maxEqualRun because the changed words are what the user wants to see. + maxChangedRun = 40 + ) + + segments := make([]string, 0, len(diffs)) + for i, diff := range diffs { + words := strings.Fields(diff.Text) + if len(words) == 0 { + continue + } + last := i == len(diffs)-1 + switch diff.Type { + case diffmatchpatch.DiffEqual: + segments = append(segments, collapseWords(words, maxEqualRun, contextWords, i == 0, last)) + case diffmatchpatch.DiffDelete: + segments = append(segments, "[-"+collapseWords(words, maxChangedRun, contextWords*2, false, false)+"-]") + case diffmatchpatch.DiffInsert: + if last { + // The trailing inserted words are the file contents after the license header + // region, which are irrelevant to the diff. + segments = append(segments, "...") + continue + } + segments = append(segments, "{+"+collapseWords(words, maxChangedRun, contextWords*2, false, false)+"+}") + } + } + + return strings.Join(segments, " ") +} + +// collapseWords joins the words with spaces, eliding the middle of runs longer +// than maxRun. dropHead/dropTail elide one entire side instead, for runs at the +// beginning/end of the diff where only the words next to a change matter. +func collapseWords(words []string, maxRun, context int, dropHead, dropTail bool) string { + if len(words) <= maxRun { + return strings.Join(words, " ") + } + + head := strings.Join(words[:context], " ") + tail := strings.Join(words[len(words)-context:], " ") + switch { + case dropHead: + return "... " + tail + case dropTail: + return head + " ..." + default: + return head + " ... " + tail + } +} diff --git a/pkg/header/diff_test.go b/pkg/header/diff_test.go new file mode 100644 index 0000000..763849d --- /dev/null +++ b/pkg/header/diff_test.go @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package header + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +var diffConfig = &ConfigHeader{ + License: LicenseConfig{ + Content: `Apache License 2.0 + http://www.apache.org/licenses/LICENSE-2.0 +Apache License 2.0`, + }, + LicenseLocationThreshold: 80, +} + +func TestDiffFile(t *testing.T) { + tests := []struct { + name string + filename string + content string + config *ConfigHeader + expected string + }{ + { + name: "valid header", + filename: "valid.go", + content: `// Apache License 2.0 +// http://www.apache.org/licenses/LICENSE-2.0 +// Apache License 2.0 + +package main +`, + config: diffConfig, + expected: "", + }, + { + name: "typo in the header", + filename: "typo.go", + content: `// Apache License 2.0 +// wwwhttp://www.apache.org/licenses/LICENSE-2.0 +// Apache License 2.0 + +package main +`, + config: diffConfig, + expected: "apache license 2.0 " + + "[-http://www.apache.org/licenses/license-2.0-] " + + "{+wwwhttp://www.apache.org/licenses/license-2.0+} " + + "apache license 2.0 ...", + }, + { + name: "no header at all", + filename: "missing.go", + content: `package main + +func main() {} +`, + config: diffConfig, + expected: "[-apache license 2.0 http://www.apache.org/licenses/license-2.0 apache license 2.0-] " + + "...", + }, + { + name: "header too far from the file start", + filename: "far.go", + content: `// aaaa bbbb cccc dddd eeee ffff gggg hhhh +// Apache License 2.0 +// http://www.apache.org/licenses/LICENSE-2.0 +// Apache License 2.0 + +package main +`, + config: &ConfigHeader{ + License: diffConfig.License, + LicenseLocationThreshold: 10, + }, + expected: "license header is found at normalized offset 40, " + + "which exceeds the license-location-threshold 10, " + + "move it closer to the file start", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + file := filepath.Join(t.TempDir(), test.filename) + require.NoError(t, os.WriteFile(file, []byte(test.content), 0o600)) + + diff, err := DiffFile(file, test.config) + require.NoError(t, err) + require.Equal(t, test.expected, diff) + }) + } +} + +func TestDiffFileWithoutLicenseContent(t *testing.T) { + file := filepath.Join(t.TempDir(), "test.go") + require.NoError(t, os.WriteFile(file, []byte("package main\n"), 0o600)) + + _, err := DiffFile(file, &ConfigHeader{LicenseLocationThreshold: 80}) + require.Error(t, err) +}