Add repository exclusion from contribution counts

github/github.go

@@ -65,6 +65,14 @@ func (client HTTPGithubClient) SearchUsers(query UserSearchQuery) (GithubSearchR
 	users := []User{}
 	userLogins := map[string]bool{}
 
+	// Repositories whose commit contributions should be ignored when ranking
+	// users (e.g. dataset/archive repos that would otherwise inflate counts).
+	// Matching is case-insensitive on the "owner/name" form.
+	excludeRepos := map[string]bool{}
+	for _, repo := range query.ExcludeRepos {
+		excludeRepos[strings.ToLower(repo)] = true
+	}
+
 	totalCount := 0
 	minFollowerCount := -1
 	maxPerQuery := 1000
@@ -111,7 +119,16 @@ Pages:
                   },
                   totalCommitContributions,
                   totalPullRequestContributions,
-                  restrictedContributionsCount
+                  restrictedContributionsCount,
+                  commitContributionsByRepository(maxRepositories: 100) {
+                    repository {
+                      nameWithOwner,
+                      isPrivate
+                    },
+                    contributions {
+                      totalCount
+                    }
+                  }
                 }
               }
             },
@@ -205,18 +222,27 @@ Pages:
 				commitsCount := int(contributionsCollection["totalCommitContributions"].(float64))
 				pullRequestsCount := int(contributionsCollection["totalPullRequestContributions"].(float64))
 
+				repoContributions := parseRepoCommitContributions(contributionsCollection)
+				excludedPublic, excludedPrivate := repoExclusions(repoContributions, excludeRepos)
+				excludedTotal := excludedPublic + excludedPrivate
+
+				contributionCount = clampZero(contributionCount - excludedTotal)
+				privateContributionCount = clampZero(privateContributionCount - excludedPrivate)
+				commitsCount = clampZero(commitsCount - excludedTotal)
+
 				user := User{
-					Login:                    login,
-					AvatarURL:                avatarURL,
-					Name:                     name,
-					Company:                  company,
-					Organizations:            organizations,
-					FollowerCount:            followerCount,
-					ContributionCount:        contributionCount,
-					PublicContributionCount:  (contributionCount - privateContributionCount),
-					PrivateContributionCount: privateContributionCount,
-					CommitsCount:             commitsCount,
-					PullRequestsCount:        pullRequestsCount}
+					Login:                     login,
+					AvatarURL:                 avatarURL,
+					Name:                      name,
+					Company:                   company,
+					Organizations:             organizations,
+					FollowerCount:             followerCount,
+					ContributionCount:         contributionCount,
+					PublicContributionCount:   clampZero(contributionCount - privateContributionCount),
+					PrivateContributionCount:  privateContributionCount,
+					CommitsCount:              commitsCount,
+					PullRequestsCount:         pullRequestsCount,
+					ExcludedContributionCount: excludedTotal}
 
 				if !userLogins[login] {
 					userLogins[login] = true
@@ -235,6 +261,74 @@ Pages:
 		TotalUserCount:       totalUsersCount}, nil
 }
 
+// RepoCommitContribution holds the number of commit contributions a user made
+// to a single repository within the queried time window.
+type RepoCommitContribution struct {
+	NameWithOwner string
+	IsPrivate     bool
+	Commits       int
+}
+
+// parseRepoCommitContributions extracts the per-repository commit breakdown from
+// a parsed contributionsCollection node. Missing/malformed entries are skipped.
+func parseRepoCommitContributions(contributionsCollection map[string]interface{}) []RepoCommitContribution {
+	result := []RepoCommitContribution{}
+	rawRepos, ok := contributionsCollection["commitContributionsByRepository"].([]interface{})
+	if !ok {
+		return result
+	}
+	for _, raw := range rawRepos {
+		node, ok := raw.(map[string]interface{})
+		if !ok {
+			continue
+		}
+		repo, ok := node["repository"].(map[string]interface{})
+		if !ok {
+			continue
+		}
+		nameWithOwner, _ := repo["nameWithOwner"].(string)
+		isPrivate, _ := repo["isPrivate"].(bool)
+		commits := 0
+		if contribs, ok := node["contributions"].(map[string]interface{}); ok {
+			if total, ok := contribs["totalCount"].(float64); ok {
+				commits = int(total)
+			}
+		}
+		result = append(result, RepoCommitContribution{
+			NameWithOwner: nameWithOwner,
+			IsPrivate:     isPrivate,
+			Commits:       commits,
+		})
+	}
+	return result
+}
+
+// repoExclusions sums the commit contributions belonging to excluded repositories,
+// split by visibility so they can be subtracted from the right totals. The exclude
+// set keys are expected to be lower-cased "owner/name" strings.
+func repoExclusions(repos []RepoCommitContribution, exclude map[string]bool) (excludedPublic int, excludedPrivate int) {
+	if len(exclude) == 0 {
+		return 0, 0
+	}
+	for _, repo := range repos {
+		if exclude[strings.ToLower(repo.NameWithOwner)] {
+			if repo.IsPrivate {
+				excludedPrivate += repo.Commits
+			} else {
+				excludedPublic += repo.Commits
+			}
+		}
+	}
+	return excludedPublic, excludedPrivate
+}
+
+func clampZero(n int) int {
+	if n < 0 {
+		return 0
+	}
+	return n
+}
+
 func strPropOrEmpty(obj map[string]interface{}, prop string) string {
 	switch t := obj[prop].(type) {
 	case string:
@@ -276,24 +370,26 @@ func NewGithubClient(wrappers ...net.Wrapper) HTTPGithubClient {
 }
 
 type User struct {
-	Login                    string
-	AvatarURL                string
-	Name                     string
-	Company                  string
-	Organizations            []string
-	FollowerCount            int
-	ContributionCount        int
-	PublicContributionCount  int
-	PrivateContributionCount int
-	CommitsCount             int
-	PullRequestsCount        int
+	Login                     string
+	AvatarURL                 string
+	Name                      string
+	Company                   string
+	Organizations             []string
+	FollowerCount             int
+	ContributionCount         int
+	PublicContributionCount   int
+	PrivateContributionCount  int
+	CommitsCount              int
+	PullRequestsCount         int
+	ExcludedContributionCount int
 }
 
 type UserSearchQuery struct {
-	Q        string
-	Sort     string
-	Order    string
-	MaxUsers int
+	Q            string
+	Sort         string
+	Order        string
+	MaxUsers     int
+	ExcludeRepos []string
 }
 
 type GithubSearchResults struct {

github/github_test.go

@@ -0,0 +1,95 @@
+package github
+
+import "testing"
+
+func buildExcludeSet(repos ...string) map[string]bool {
+	set := map[string]bool{}
+	for _, r := range repos {
+		set[r] = true
+	}
+	return set
+}
+
+func TestRepoExclusionsEmptySet(t *testing.T) {
+	repos := []RepoCommitContribution{
+		{NameWithOwner: "owner/repo", IsPrivate: false, Commits: 100},
+	}
+	pub, priv := repoExclusions(repos, map[string]bool{})
+	if pub != 0 || priv != 0 {
+		t.Fatalf("expected no exclusions, got public=%d private=%d", pub, priv)
+	}
+}
+
+func TestRepoExclusionsPublicAndPrivate(t *testing.T) {
+	repos := []RepoCommitContribution{
+		{NameWithOwner: "domovinatv/dataset.domovina.tv", IsPrivate: false, Commits: 27604},
+		{NameWithOwner: "owner/private-archive", IsPrivate: true, Commits: 500},
+		{NameWithOwner: "owner/real-project", IsPrivate: false, Commits: 120},
+	}
+	exclude := buildExcludeSet("domovinatv/dataset.domovina.tv", "owner/private-archive")
+
+	pub, priv := repoExclusions(repos, exclude)
+	if pub != 27604 {
+		t.Errorf("expected excludedPublic=27604, got %d", pub)
+	}
+	if priv != 500 {
+		t.Errorf("expected excludedPrivate=500, got %d", priv)
+	}
+}
+
+func TestRepoExclusionsCaseInsensitive(t *testing.T) {
+	repos := []RepoCommitContribution{
+		{NameWithOwner: "DomovinaTV/Dataset.Domovina.TV", IsPrivate: false, Commits: 42},
+	}
+	// The exclude set is lower-cased by the caller; repoExclusions lower-cases the
+	// repository name before matching.
+	exclude := buildExcludeSet("domovinatv/dataset.domovina.tv")
+
+	pub, priv := repoExclusions(repos, exclude)
+	if pub != 42 || priv != 0 {
+		t.Errorf("expected case-insensitive match (public=42), got public=%d private=%d", pub, priv)
+	}
+}
+
+func TestParseRepoCommitContributions(t *testing.T) {
+	collection := map[string]interface{}{
+		"commitContributionsByRepository": []interface{}{
+			map[string]interface{}{
+				"repository": map[string]interface{}{
+					"nameWithOwner": "owner/repo",
+					"isPrivate":     true,
+				},
+				"contributions": map[string]interface{}{
+					"totalCount": float64(13),
+				},
+			},
+			// malformed entry should be skipped, not panic
+			map[string]interface{}{"unexpected": "shape"},
+		},
+	}
+
+	parsed := parseRepoCommitContributions(collection)
+	if len(parsed) != 1 {
+		t.Fatalf("expected 1 parsed repo, got %d", len(parsed))
+	}
+	got := parsed[0]
+	if got.NameWithOwner != "owner/repo" || !got.IsPrivate || got.Commits != 13 {
+		t.Errorf("unexpected parsed contribution: %+v", got)
+	}
+}
+
+func TestParseRepoCommitContributionsMissingKey(t *testing.T) {
+	parsed := parseRepoCommitContributions(map[string]interface{}{})
+	if len(parsed) != 0 {
+		t.Fatalf("expected no contributions for missing key, got %d", len(parsed))
+	}
+}
+
+func TestClampZero(t *testing.T) {
+	if got := clampZero(-5); got != 0 {
+		t.Errorf("expected clampZero(-5)=0, got %d", got)
+	}
+	if got := clampZero(7); got != 7 {
+		t.Errorf("expected clampZero(7)=7, got %d", got)
+	}
+}

main.go

@@ -24,6 +24,7 @@ func (i *arrayFlags) Set(value string) error {
 
 var locations arrayFlags
 var excludeLocations arrayFlags
+var excludeRepos arrayFlags
 var presetTitle string
 var presetChecksum string
 
@@ -37,8 +38,12 @@ func main() {
 	listPresets := flag.Bool("list-presets", false, "List all available presets as CSV and exit immediately")
 
 	flag.Var(&locations, "location", "Location to query")
+	flag.Var(&excludeRepos, "exclude-repo", "Repository (owner/name) whose commits are excluded from counts; repeatable")
 	flag.Parse()
 
+	// The global ExcludedRepos list always applies; --exclude-repo adds to it.
+	excludeRepos = append(excludeRepos, ExcludedRepos...)
+
 	if *listPresets {
 		fmt.Println("preset,title,definition_checksum")
 		for name, _ := range PRESETS {
@@ -67,7 +72,7 @@ func main() {
 		log.Fatal("Unrecognized output format: ", *outputOpt)
 	}
 
-	opts := top.Options{Token: *token, Locations: locations, ExcludeLocations: excludeLocations, Amount: *amount, ConsiderNum: *considerNum, PresetTitle: presetTitle, PresetChecksum: presetChecksum}
+	opts := top.Options{Token: *token, Locations: locations, ExcludeLocations: excludeLocations, ExcludeRepos: excludeRepos, Amount: *amount, ConsiderNum: *considerNum, PresetTitle: presetTitle, PresetChecksum: presetChecksum}
 	data, err := top.GithubTop(opts)
 
 	if err != nil {

output/output.go

@@ -78,6 +78,7 @@ func YamlOutput(results github.GithubSearchResults, writer io.Writer, options to
     login: %+v
     avatarUrl: %+v
     contributions: %+v
+    excluded: %+v
     company: %+v
     organizations: %+v
 `,
@@ -86,6 +87,7 @@ func YamlOutput(results github.GithubSearchResults, writer io.Writer, options to
 				strconv.QuoteToASCII(u.Login),
 				u.AvatarURL,
 				contributionCount,
+				u.ExcludedContributionCount,
 				strconv.QuoteToASCII(u.Company),
 				strconv.QuoteToASCII(strings.Join(u.Organizations, ",")))
 		}
@@ -134,6 +136,13 @@ func YamlOutput(results github.GithubSearchResults, writer io.Writer, options to
 		fmt.Fprintf(writer, "definition_checksum: %+v\n", options.PresetChecksum)
 	}
 
+	if len(options.ExcludeRepos) > 0 {
+		fmt.Fprintln(writer, "excluded_repos:")
+		for _, repo := range options.ExcludeRepos {
+			fmt.Fprintf(writer, "  - %+v\n", strconv.QuoteToASCII(repo))
+		}
+	}
+
 	return nil
 }
 

presets.go

@@ -13,6 +13,17 @@ type QueryPreset struct {
 	exclude []string
 }
 
+// ExcludedRepos lists repositories whose commit contributions are ignored when
+// ranking users, applied across every preset. This keeps the rankings
+// representative by excluding dataset/archive repositories whose automated commit
+// volume would otherwise dominate a user's contribution count.
+//
+// Entries are "owner/name" and matched case-insensitively. This list is published
+// on every region page so the exclusions are transparent.
+var ExcludedRepos = []string{
+	"domovinatv/dataset.domovina.tv",
+}
+
 var PRESETS = map[string]QueryPreset{
 	"panama": QueryPreset{
 		title:   "Panama",
@@ -638,5 +649,8 @@ func PresetTitle(name string) string {
 func PresetChecksum(name string) string {
 	hash := sha256.New()
 	io.WriteString(hash, fmt.Sprintf("%+v", Preset(name)))
+	// Fold in the global repo-exclusion list so that changing it invalidates every
+	// preset's checksum, triggering a regeneration of all region pages.
+	io.WriteString(hash, fmt.Sprintf("excluded_repos:%+v", ExcludedRepos))
 	return fmt.Sprintf("%x", hash.Sum(nil))
 }

top/top.go

@@ -24,7 +24,7 @@ func GithubTop(options Options) (github.GithubSearchResults, error) {
 	}
 
 	var client = github.NewGithubClient(net.TokenAuth(token))
-	users, err := client.SearchUsers(github.UserSearchQuery{Q: query, Sort: "followers", Order: "desc", MaxUsers: options.ConsiderNum})
+	users, err := client.SearchUsers(github.UserSearchQuery{Q: query, Sort: "followers", Order: "desc", MaxUsers: options.ConsiderNum, ExcludeRepos: options.ExcludeRepos})
 	if err != nil {
 		return github.GithubSearchResults{}, err
 	}
@@ -35,6 +35,7 @@ type Options struct {
 	Token            string
 	Locations        []string
 	ExcludeLocations []string
+	ExcludeRepos     []string
 	Amount           int
 	ConsiderNum      int
 	PresetTitle      string