From 381e5b837913b3f2848000b217056400fce39609 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Tue, 16 Jun 2026 03:46:50 -0600 Subject: [PATCH 01/26] Handle transient GraphQL response failures Amp-Thread-ID: https://ampcode.com/threads/T-019ecf95-ee6d-7426-b1e7-d724bdf73e0f Co-authored-by: Amp --- repo-troubleshooting/list-repos/list-repos.py | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/repo-troubleshooting/list-repos/list-repos.py b/repo-troubleshooting/list-repos/list-repos.py index 7130619..92692c6 100755 --- a/repo-troubleshooting/list-repos/list-repos.py +++ b/repo-troubleshooting/list-repos/list-repos.py @@ -60,6 +60,7 @@ "dial tcp", "eof", "gateway timeout", + "index:only failed since indexed search is not available yet", "no such host", "temporarily unavailable", "timeout", @@ -605,7 +606,7 @@ def fetch_commit_count( elapsed = time.monotonic() - start logger.warning("commit-count query failed for %s: %s", repo_name, exc) return None, None, elapsed, empty_extras - except OSError as exc: + except (OSError, http.client.HTTPException, json.JSONDecodeError) as exc: elapsed = time.monotonic() - start logger.warning( "commit-count network error for %s: %s", @@ -656,7 +657,7 @@ def fetch_run_search( elapsed = time.monotonic() - start logger.warning("run-search query failed for %s: %s", repo_name, exc) return None, elapsed, False, None - except OSError as exc: + except (OSError, http.client.HTTPException, json.JSONDecodeError) as exc: elapsed = time.monotonic() - start logger.warning("run-search network error for %s: %s", repo_name, exc) return None, elapsed, False, None @@ -1710,7 +1711,7 @@ def graphql_request( max_retries, ) continue - except OSError as error: + except (OSError, http.client.HTTPException, json.JSONDecodeError) as error: if retry_count >= max_retries: raise sleep_before_retry( @@ -1737,7 +1738,8 @@ def graphql_request( # log the errors and keep going; only abort if no data was returned. if response.get("data"): logger.warning( - "GraphQL returned %d partial error(s): %s", + "%sGraphQL returned %d partial error(s): %s", + retry_prefix, len(errors) if isinstance(errors, list) else 1, json.dumps(errors, indent=2), ) @@ -2088,7 +2090,14 @@ def fetch_skipped_file_reason_query( request_description=f"Skipped files for {name}@{rev}", ) elapsed = time.monotonic() - start - results_block: dict[str, Any] = data.get("search", {}).get("results", {}) + search_block = data.get("search") + if not isinstance(search_block, dict): + msg = f"skipped-file reason search returned no search data for {name}@{rev}" + raise GraphQLError(msg) + results_block = search_block.get("results") + if not isinstance(results_block, dict): + msg = f"skipped-file reason search returned no results data for {name}@{rev}" + raise GraphQLError(msg) raw_results: list[dict[str, Any] | None] = results_block.get("results") or [] # Non-FileMatch results come back as empty objects; drop them matches = [result for result in raw_results if result and result.get("file")] @@ -2569,7 +2578,13 @@ def collect_skipped_file_reason_search_results( skipped_indexed_query, max_retries=max_retries, ) - except (GraphQLError, HTTPRequestError, OSError) as error: + except ( + GraphQLError, + HTTPRequestError, + OSError, + http.client.HTTPException, + json.JSONDecodeError, + ) as error: results.append( SkippedFileReasonSearchResult( repository_name=repo_name, @@ -3721,7 +3736,7 @@ def main() -> None: except HTTPRequestError as exc: log_http_error(exc) sys.exit(1) - except OSError: + except (OSError, http.client.HTTPException, json.JSONDecodeError): logger.exception( "Could not connect to the server. Check your network and SRC_ENDPOINT", ) From 0c9f2930d1681f2b2eb4b50ad571f3a44b9399c5 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Tue, 16 Jun 2026 19:08:26 -0600 Subject: [PATCH 02/26] Handle requested query failures explicitly Amp-Thread-ID: https://ampcode.com/threads/T-019ecf95-ee6d-7426-b1e7-d724bdf73e0f Co-authored-by: Amp --- repo-troubleshooting/list-repos/list-repos.py | 473 +++++++++++++----- 1 file changed, 335 insertions(+), 138 deletions(-) diff --git a/repo-troubleshooting/list-repos/list-repos.py b/repo-troubleshooting/list-repos/list-repos.py index 92692c6..a620439 100755 --- a/repo-troubleshooting/list-repos/list-repos.py +++ b/repo-troubleshooting/list-repos/list-repos.py @@ -16,8 +16,9 @@ import shlex import sys import textwrap +import threading import time -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path from typing import TYPE_CHECKING, Any, NoReturn, TextIO, cast @@ -29,6 +30,51 @@ logger = logging.getLogger(__name__) +@dataclass +class RunIssueCounts: + """Count warnings, errors, and retries emitted during this run""" + + warnings: int = 0 + errors: int = 0 + retries: int = 0 + _lock: Any = field(default_factory=threading.Lock, init=False, repr=False) + + def reset(self) -> None: + """Reset counters before configuring a new run""" + with self._lock: + self.warnings = 0 + self.errors = 0 + self.retries = 0 + + def increment_retry(self) -> None: + """Count one retry sleep""" + with self._lock: + self.retries += 1 + + def increment_log_record(self, record: logging.LogRecord) -> None: + """Count a warning or error log record""" + with self._lock: + if record.levelno >= logging.ERROR: + self.errors += 1 + elif record.levelno >= logging.WARNING: + self.warnings += 1 + + def snapshot(self) -> tuple[int, int, int]: + """Return stable (errors, warnings, retries) counts""" + with self._lock: + return self.errors, self.warnings, self.retries + + +RUN_ISSUE_COUNTS = RunIssueCounts() + + +class IssueCountingHandler(logging.Handler): + """Logging handler that counts warning and error records without output""" + + def emit(self, record: logging.LogRecord) -> None: + RUN_ISSUE_COUNTS.increment_log_record(record) + + # --- Tune-ables ----------------------------------------------------------------- DEFAULT_CLONING_ERRORS_FILE = "repos-with-cloning-errors.csv" @@ -586,47 +632,66 @@ def fetch_commit_count( max_retries: int = DEFAULT_MAX_RETRIES, ) -> tuple[int | None, int | None, float, list[Any]]: """Return exact rev count, approximate all-refs count, elapsed time, extras""" - empty_extras: list[Any] = [None] * len(COMMIT_COUNT_OPTIMIZATION_COLUMNS) start = time.monotonic() - try: - data = graphql_request( - endpoint, - token, - COMMIT_COUNT_QUERY, - { - "name": repo_name, - "rev": rev, - "allRefsSearch": build_all_refs_search(repo_name), - }, - timeout=REQUEST_TIMEOUT_SECONDS_WITH_COMMIT_COUNT, - max_retries=max_retries, - request_description=f"Commit count for {repo_name}", + + def validate(data: dict[str, Any]) -> None: + repo_block = require_dict(data.get("repository"), "commit-count repository") + commit_block = require_dict( + repo_block.get("commit"), + "commit-count commit", + retryable=False, ) - except (GraphQLError, HTTPRequestError) as exc: - elapsed = time.monotonic() - start - logger.warning("commit-count query failed for %s: %s", repo_name, exc) - return None, None, elapsed, empty_extras - except (OSError, http.client.HTTPException, json.JSONDecodeError) as exc: - elapsed = time.monotonic() - start - logger.warning( - "commit-count network error for %s: %s", - repo_name, - exc, + ancestors_block = require_dict( + commit_block.get("ancestors"), + "commit-count ancestors", ) - return None, None, elapsed, empty_extras + require_int( + ancestors_block.get("totalCount"), + "commit-count default branch totalCount", + ) + search_block = require_dict(data.get("search"), "commit-count search") + search_results = require_dict( + search_block.get("results"), + "commit-count search results", + ) + require_int( + search_results.get("matchCount"), "commit-count all-refs matchCount" + ) + + data = graphql_request_with_validation( + endpoint, + token, + COMMIT_COUNT_QUERY, + { + "name": repo_name, + "rev": rev, + "allRefsSearch": build_all_refs_search(repo_name), + }, + timeout=REQUEST_TIMEOUT_SECONDS_WITH_COMMIT_COUNT, + max_retries=max_retries, + request_description=f"Commit count for {repo_name}", + validate=validate, + ) elapsed = time.monotonic() - start - repo: dict[str, Any] = data.get("repository") or {} - commit: dict[str, Any] = repo.get("commit") or {} - ancestors: dict[str, Any] = commit.get("ancestors") or {} - default_count_raw = ancestors.get("totalCount") - default_count: int | None = ( - default_count_raw if isinstance(default_count_raw, int) else None - ) - search_block: dict[str, Any] = data.get("search") or {} - search_results: dict[str, Any] = search_block.get("results") or {} - all_refs_count_raw = search_results.get("matchCount") - all_refs_count: int | None = ( - all_refs_count_raw if isinstance(all_refs_count_raw, int) else None + repo = require_dict(data.get("repository"), "commit-count repository") + commit = require_dict( + repo.get("commit"), + "commit-count commit", + retryable=False, + ) + ancestors = require_dict(commit.get("ancestors"), "commit-count ancestors") + default_count = require_int( + ancestors.get("totalCount"), + "commit-count default branch totalCount", + ) + search_block = require_dict(data.get("search"), "commit-count search") + search_results = require_dict( + search_block.get("results"), + "commit-count search results", + ) + all_refs_count = require_int( + search_results.get("matchCount"), + "commit-count all-refs matchCount", ) optimization_values = [ extract(repo) for _, extract, _, _, _ in COMMIT_COUNT_OPTIMIZATION_COLUMNS @@ -644,26 +709,23 @@ def fetch_run_search( """Return --run-search match count, elapsed time, limit flag, and alert""" start = time.monotonic() query = build_run_search_query(repo_name, pattern) - try: - data = graphql_request( - endpoint, - token, - RUN_SEARCH_GRAPHQL, - {"query": query}, - max_retries=max_retries, - request_description=f"Search {pattern} in {repo_name}", - ) - except (GraphQLError, HTTPRequestError) as exc: - elapsed = time.monotonic() - start - logger.warning("run-search query failed for %s: %s", repo_name, exc) - return None, elapsed, False, None - except (OSError, http.client.HTTPException, json.JSONDecodeError) as exc: - elapsed = time.monotonic() - start - logger.warning("run-search network error for %s: %s", repo_name, exc) - return None, elapsed, False, None + + def validate(data: dict[str, Any]) -> None: + search_block = require_dict(data.get("search"), "run-search search") + require_dict(search_block.get("results"), "run-search results") + + data = graphql_request_with_validation( + endpoint, + token, + RUN_SEARCH_GRAPHQL, + {"query": query}, + max_retries=max_retries, + request_description=f"Search {pattern} in {repo_name}", + validate=validate, + ) elapsed = time.monotonic() - start - search_block: dict[str, Any] = data.get("search") or {} - results: dict[str, Any] = search_block.get("results") or {} + search_block = require_dict(data.get("search"), "run-search search") + results = require_dict(search_block.get("results"), "run-search results") raw_count = results.get("matchCount") match_count: int | None = raw_count if isinstance(raw_count, int) else None limit_hit = bool(results.get("limitHit")) @@ -1482,6 +1544,10 @@ class GraphQLError(RuntimeError): """Raised when the Sourcegraph GraphQL API returns errors""" +class NonRetryableGraphQLError(GraphQLError): + """Raised when GraphQL data is missing for a deterministic reason""" + + class HTTPRequestError(RuntimeError): """Raised when the server returns a definitive 4xx/5xx HTTP response""" @@ -1502,6 +1568,15 @@ def __init__( self.body = body +REQUEST_FAILURE_TYPES = ( + GraphQLError, + HTTPRequestError, + OSError, + http.client.HTTPException, + json.JSONDecodeError, +) + + @dataclass(frozen=True) class GraphQLFieldCountViolation: """Sourcegraph GraphQL field-count limit details from an error response""" @@ -1637,6 +1712,7 @@ def retry_delay_seconds(retry_number: int) -> int: def sleep_before_retry(reason: str, retry_number: int, max_retries: int) -> None: """Log and sleep before the next retry attempt for this request""" delay = retry_delay_seconds(retry_number) + RUN_ISSUE_COUNTS.increment_retry() logger.warning( "%s; retrying (%d/%d) in %ds...", reason, @@ -1680,6 +1756,97 @@ def summarize_graphql_errors(errors: object) -> str: return "; ".join(messages) +def request_error_summary(error: Exception) -> str: + """Return a compact request failure string for logs""" + if isinstance(error, HTTPRequestError): + body = error.body.decode(errors="replace").strip() + if body: + return f"HTTP {error.status} {error.reason}: {body}" + return f"HTTP {error.status} {error.reason}" + return str(error) + + +def request_failure_can_retry(error: Exception) -> bool: + """Return True if the request failure should consume retry budget""" + if isinstance(error, NonRetryableGraphQLError): + return False + if isinstance(error, HTTPRequestError): + return retryable_http_error(error) + return True + + +def require_dict( + value: object, + description: str, + *, + retryable: bool = True, +) -> dict[str, Any]: + """Return value as a dict, or raise GraphQLError for validation""" + if isinstance(value, dict): + return value + msg = f"{description} missing or not an object" + if retryable: + raise GraphQLError(msg) + raise NonRetryableGraphQLError(msg) + + +def require_int(value: object, description: str) -> int: + """Return value as an int, or raise GraphQLError for validation""" + if isinstance(value, bool): + msg = f"{description} missing or not an integer" + raise GraphQLError(msg) + if isinstance(value, int): + return value + msg = f"{description} missing or not an integer" + raise GraphQLError(msg) + + +def graphql_request_with_validation( + endpoint: str, + token: str, + query: str, + variables: dict[str, Any], + *, + timeout: int = REQUEST_TIMEOUT_SECONDS, + max_retries: int = DEFAULT_MAX_RETRIES, + request_description: str, + validate: Callable[[dict[str, Any]], None] | None = None, +) -> dict[str, Any]: + """Run a GraphQL request and retry transport, GraphQL, and shape errors""" + for retry_count in range(max_retries + 1): + retry_number = retry_count + 1 + try: + data = graphql_request( + endpoint, + token, + query, + variables, + timeout=timeout, + max_retries=0, + request_description=request_description, + ) + if validate is not None: + validate(data) + return data + except REQUEST_FAILURE_TYPES as error: + error_summary = request_error_summary(error) + if retry_count >= max_retries or not request_failure_can_retry(error): + logger.error( + "%s failed after %d attempt(s): %s", + request_description, + retry_number, + error_summary, + ) + raise + sleep_before_retry( + f"{request_description} failed: {error_summary}", + retry_number, + max_retries, + ) + msg = f"{request_description} retry loop exhausted unexpectedly" + raise RuntimeError(msg) + + def graphql_request( endpoint: str, token: str, @@ -1723,27 +1890,27 @@ def graphql_request( errors = response.get("errors") if not errors: - return response["data"] + data = response.get("data") + if isinstance(data, dict): + return data + if retry_count < max_retries: + sleep_before_retry( + f"{retry_prefix}GraphQL response data missing or not an object", + retry_number, + max_retries, + ) + continue + raise GraphQLError("GraphQL response data missing or not an object") - if has_retryable_graphql_error(errors) and retry_count < max_retries: + if retry_count < max_retries: + retry_label = "retryable " if has_retryable_graphql_error(errors) else "" sleep_before_retry( - f"{retry_prefix}GraphQL returned retryable error(s): " - + summarize_graphql_errors(errors), + f"{retry_prefix}GraphQL returned {retry_label}error(s): " + f"{summarize_graphql_errors(errors)}", retry_number, max_retries, ) continue - - # GraphQL can return both `errors` and partial `data`. If we have data, - # log the errors and keep going; only abort if no data was returned. - if response.get("data"): - logger.warning( - "%sGraphQL returned %d partial error(s): %s", - retry_prefix, - len(errors) if isinstance(errors, list) else 1, - json.dumps(errors, indent=2), - ) - return response["data"] msg = f"GraphQL errors: {json.dumps(errors, indent=2)}" raise GraphQLError(msg) msg = "graphql_request retry loop exhausted unexpectedly" @@ -1803,11 +1970,11 @@ def supports_text_search_index_failure_fields( token, max_retries=max_retries, ) - except (GraphQLError, HTTPRequestError, OSError) as error: - logger.warning( + except REQUEST_FAILURE_TYPES as error: + logger.error( "Could not inspect Sourcegraph schema for new text-search index " "failure fields (requires v7.5.0); leaving those CSV columns blank: %s", - error, + request_error_summary(error), ) return False @@ -1854,7 +2021,7 @@ def trigger_reclone( repo_id: str, max_retries: int = DEFAULT_MAX_RETRIES, ) -> bool: - """Send recloneRepository mutation. Returns True on success, False on GraphQL error""" + """Send recloneRepository mutation. Returns True on success, False on request error""" try: graphql_request( endpoint, @@ -1864,8 +2031,12 @@ def trigger_reclone( max_retries=max_retries, request_description=f"Reclone repository {repo_id}", ) - except (GraphQLError, HTTPRequestError) as exc: - logger.warning("recloneRepository failed for %s: %s", repo_id, exc) + except REQUEST_FAILURE_TYPES as error: + logger.error( + "recloneRepository failed for %s: %s", + repo_id, + request_error_summary(error), + ) return False return True @@ -1876,7 +2047,7 @@ def trigger_reindex( repo_id: str, max_retries: int = DEFAULT_MAX_RETRIES, ) -> bool: - """Send reindexRepository mutation. Returns True on success, False on GraphQL error""" + """Send reindexRepository mutation. Returns True on success, False on request error""" try: graphql_request( endpoint, @@ -1886,8 +2057,12 @@ def trigger_reindex( max_retries=max_retries, request_description=f"Reindex repository {repo_id}", ) - except (GraphQLError, HTTPRequestError) as exc: - logger.warning("reindexRepository failed for %s: %s", repo_id, exc) + except REQUEST_FAILURE_TYPES as error: + logger.error( + "reindexRepository failed for %s: %s", + repo_id, + request_error_summary(error), + ) return False return True @@ -2080,7 +2255,12 @@ def fetch_skipped_file_reason_query( """Return NOT-INDEXED matches and search metadata for one indexed repo ref""" start = time.monotonic() search_query = skipped_file_reason_search_query(skipped_indexed_query, name, rev) - data = graphql_request( + + def validate(data: dict[str, Any]) -> None: + search_block = require_dict(data.get("search"), "skipped-file reason search") + require_dict(search_block.get("results"), "skipped-file reason results") + + data = graphql_request_with_validation( endpoint, token, SKIPPED_FILES_REASON_QUERY, @@ -2088,16 +2268,14 @@ def fetch_skipped_file_reason_query( timeout=REQUEST_TIMEOUT_SECONDS_WITH_COMMIT_COUNT, max_retries=max_retries, request_description=f"Skipped files for {name}@{rev}", + validate=validate, ) elapsed = time.monotonic() - start - search_block = data.get("search") - if not isinstance(search_block, dict): - msg = f"skipped-file reason search returned no search data for {name}@{rev}" - raise GraphQLError(msg) - results_block = search_block.get("results") - if not isinstance(results_block, dict): - msg = f"skipped-file reason search returned no results data for {name}@{rev}" - raise GraphQLError(msg) + search_block = require_dict(data.get("search"), "skipped-file reason search") + results_block = require_dict( + search_block.get("results"), + "skipped-file reason results", + ) raw_results: list[dict[str, Any] | None] = results_block.get("results") or [] # Non-FileMatch results come back as empty objects; drop them matches = [result for result in raw_results if result and result.get("file")] @@ -2578,13 +2756,7 @@ def collect_skipped_file_reason_search_results( skipped_indexed_query, max_retries=max_retries, ) - except ( - GraphQLError, - HTTPRequestError, - OSError, - http.client.HTTPException, - json.JSONDecodeError, - ) as error: + except REQUEST_FAILURE_TYPES as error: results.append( SkippedFileReasonSearchResult( repository_name=repo_name, @@ -2595,7 +2767,7 @@ def collect_skipped_file_reason_search_results( limit_hit=False, alert_title=None, alert_description=None, - error=str(error), + error=request_error_summary(error), ), ) continue @@ -2623,12 +2795,6 @@ def write_skipped_file_reason_rows( """Append skipped-file detail rows from per-ref search results""" for search_result in search_results: if search_result.error is not None: - logger.warning( - "Skipped-file reason search failed for %s@%s: %s", - search_result.repository_name, - search_result.ref_name, - search_result.error, - ) continue alert_parts = [ part @@ -2706,11 +2872,13 @@ class RepoProcessingResult: commit_count: int | None all_refs_count: int | None commit_elapsed_seconds: float | None + commit_error: str | None optimization_values: list[Any] | None search_match_count: int | None search_elapsed_seconds: float | None search_limit_hit: bool search_alert_title: str | None + search_error: str | None skipped_file_reason_search_results: list[SkippedFileReasonSearchResult] @@ -2732,39 +2900,47 @@ def collect_repo_processing_result( commit_count: int | None = None all_refs_count: int | None = None commit_elapsed_seconds: float | None = None + commit_error: str | None = None optimization_values: list[Any] | None = None search_match_count: int | None = None search_elapsed_seconds: float | None = None search_limit_hit = False search_alert_title: str | None = None + search_error: str | None = None skipped_file_reason_search_results: list[SkippedFileReasonSearchResult] = [] repo_name = str(repo.get("name") or "") if count_commits: - ( - commit_count, - all_refs_count, - commit_elapsed_seconds, - optimization_values, - ) = fetch_commit_count( - endpoint, - token, - repo_name, - count_commits_rev, - max_retries=max_retries, - ) + try: + ( + commit_count, + all_refs_count, + commit_elapsed_seconds, + optimization_values, + ) = fetch_commit_count( + endpoint, + token, + repo_name, + count_commits_rev, + max_retries=max_retries, + ) + except REQUEST_FAILURE_TYPES as error: + commit_error = request_error_summary(error) if run_search_pattern is not None: - ( - search_match_count, - search_elapsed_seconds, - search_limit_hit, - search_alert_title, - ) = fetch_run_search( - endpoint, - token, - repo_name, - run_search_pattern, - max_retries=max_retries, - ) + try: + ( + search_match_count, + search_elapsed_seconds, + search_limit_hit, + search_alert_title, + ) = fetch_run_search( + endpoint, + token, + repo_name, + run_search_pattern, + max_retries=max_retries, + ) + except REQUEST_FAILURE_TYPES as error: + search_error = request_error_summary(error) if skipped_file_reasons and has_skipped_files(repo): skipped_file_reason_search_results = collect_skipped_file_reason_search_results( endpoint, @@ -2780,11 +2956,13 @@ def collect_repo_processing_result( commit_count=commit_count, all_refs_count=all_refs_count, commit_elapsed_seconds=commit_elapsed_seconds, + commit_error=commit_error, optimization_values=optimization_values, search_match_count=search_match_count, search_elapsed_seconds=search_elapsed_seconds, search_limit_hit=search_limit_hit, search_alert_title=search_alert_title, + search_error=search_error, skipped_file_reason_search_results=skipped_file_reason_search_results, ) @@ -2826,7 +3004,7 @@ def log_processing_result( repo_label = ( result.repo.get("name") or result.repo.get("url") or result.repo.get("id") ) - if count_commits: + if count_commits and result.commit_error is None: default_str = "?" if result.commit_count is None else f"{result.commit_count}" all_refs_str = ( "?" if result.all_refs_count is None else f"{result.all_refs_count}" @@ -2850,7 +3028,7 @@ def log_processing_result( all_refs_str, elapsed, ) - if run_search_pattern is not None: + if run_search_pattern is not None and result.search_error is None: count_str = ( "?" if result.search_match_count is None else f"{result.search_match_count}" ) @@ -3675,12 +3853,15 @@ def timestamped_log_path() -> Path: def configure_logging(log_path: Path) -> None: """Send INFO-level logs to both stderr (live feedback) and log_path""" + RUN_ISSUE_COUNTS.reset() root = logging.getLogger() root.setLevel(logging.INFO) # Clear existing handlers (e.g. on re-entry from tests) for handler in list(root.handlers): root.removeHandler(handler) + root.addHandler(IssueCountingHandler(level=logging.WARNING)) + stderr_handler = logging.StreamHandler(sys.stderr) stderr_handler.setFormatter(logging.Formatter("%(message)s")) root.addHandler(stderr_handler) @@ -3712,6 +3893,17 @@ def _log_uncaught_exception( ) +def log_run_issue_summary() -> None: + """Log final warning, error, and retry counts""" + errors, warnings, retries = RUN_ISSUE_COUNTS.snapshot() + logger.info( + "Run issue summary: errors=%d warnings=%d retries=%d", + errors, + warnings, + retries, + ) + + def main() -> None: """Entry point: configure logging, load env, parse args, run, handle errors""" configure_logging(timestamped_log_path()) @@ -3721,17 +3913,20 @@ def main() -> None: args = parse_args(sys.argv[1:]) # Schema generation is offline and credential-free if args.write_csv_schema: - write_csv_schema(Path(DEFAULT_CSV_SCHEMA_FILE)) + try: + write_csv_schema(Path(DEFAULT_CSV_SCHEMA_FILE)) + finally: + log_run_issue_summary() return - load_dotenv() - endpoint, token = require_credentials(args) - logger.info( - "Running: %s (SRC_ENDPOINT=%s)", - redact_argv_for_log(sys.argv), - endpoint, - ) try: + load_dotenv() + endpoint, token = require_credentials(args) + logger.info( + "Running: %s (SRC_ENDPOINT=%s)", + redact_argv_for_log(sys.argv), + endpoint, + ) run(args, endpoint, token) except HTTPRequestError as exc: log_http_error(exc) @@ -3752,6 +3947,8 @@ def main() -> None: else: logger.exception("GraphQL request failed") sys.exit(1) + finally: + log_run_issue_summary() if __name__ == "__main__": From 6dd49964c081ff8abd862e1519e7d0094bd47d4a Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Wed, 17 Jun 2026 14:54:40 -0600 Subject: [PATCH 03/26] Add optional skipped file trigram metrics Amp-Thread-ID: https://ampcode.com/threads/T-019ecf95-ee6d-7426-b1e7-d724bdf73e0f Co-authored-by: Amp --- repo-troubleshooting/list-repos/CSV_SCHEMA.md | 14 +- repo-troubleshooting/list-repos/list-repos.py | 225 ++++++++++++++++-- 2 files changed, 221 insertions(+), 18 deletions(-) diff --git a/repo-troubleshooting/list-repos/CSV_SCHEMA.md b/repo-troubleshooting/list-repos/CSV_SCHEMA.md index 2549f08..7729d0e 100644 --- a/repo-troubleshooting/list-repos/CSV_SCHEMA.md +++ b/repo-troubleshooting/list-repos/CSV_SCHEMA.md @@ -21,7 +21,7 @@ so the script can run against multiple instances without overwriting files | `-repos-with-cloning-errors.csv` | at least one repo has a cloning error | main columns + cloning-error extras | | `-repos-with-indexing-errors.csv` | at least one repo is cloned but is missing a search index | main columns | | `-repos-with-skipped-files.csv` | `--skipped-files` is set and the last index excluded some files | main columns + skipped-files extras | -| `-skipped-file-reasons.csv` | `--skipped-files-reason` is set without `REPO[@REV]` | skipped-file reason columns | +| `-skipped-file-reasons.csv` | `--skipped-files-reason` is set without `REPO[@REV]` | skipped-file reason columns, plus skipped-file metrics columns when `--skipped-file-metrics` is set | | `-stats-*.csv` | `--statistics` is set | `bucket,count` (see Statistics section) | The optional `--count-commits` and `--run-search` flags append extra @@ -98,10 +98,20 @@ Written to `-skipped-file-reasons.csv` when | `reason` | string | | NOT-INDEXED reason parsed from the indexed placeholder content | | `file.extension` | string | | File extension derived from file.path | | `file.byteSize` | integer | | Sourcegraph-reported file byte size | -| `skippedIndexed.count` | integer | | Count Sourcegraph reported for this repo/ref before running the details search | +| `repoRevSkippedIndexed.count` | integer | | Count Sourcegraph reported for this repo/ref before running the details search | | `file.path` | string | | Path of the skipped file within the repository | | `file_url` | string | | Sourcegraph blob URL for the skipped file at the indexed ref | +## Skipped-file metrics columns + +Appended to skipped-file detail CSVs only when `--skipped-file-metrics` +is used. These columns may require extra GitBlob content requests, and are +therefore not fetched by default + +| Column | Type | Requires admin | Description | +| --- | --- | --- | --- | +| `file.distinctTrigramCount` | integer | | Distinct three-rune trigrams computed from GitBlob.content. Only populated for skipped files whose reason is `contains too many trigrams` | + ## `--count-commits` columns Appended to CSV files when `--count-commits` is used diff --git a/repo-troubleshooting/list-repos/list-repos.py b/repo-troubleshooting/list-repos/list-repos.py index a620439..902bf75 100755 --- a/repo-troubleshooting/list-repos/list-repos.py +++ b/repo-troubleshooting/list-repos/list-repos.py @@ -112,6 +112,7 @@ def emit(self, record: logging.LogRecord) -> None: "timeout", "transport:", ) +TOO_MANY_TRIGRAMS_REASON = "contains too many trigrams" # --- GraphQL queries ---------------------------------------------------------- @@ -382,6 +383,18 @@ def build_run_search_query(repo_name: str, pattern: str) -> str: } """ +SKIPPED_FILE_BLOB_CONTENT_QUERY = """ +query SkippedFileBlobContent($repo: String!, $rev: String!, $path: String!) { + repository(name: $repo) { + commit(rev: $rev) { + blob(path: $path) { + content + } + } + } +} +""" + REPO_REV_VALIDATION_QUERY = """ query ValidateRepoRev($name: String!, $rev: String!) { repository(name: $name) { @@ -1184,7 +1197,7 @@ def validate(data: dict[str, Any]) -> None: "integer", ), ( - "skippedIndexed.count", + "repoRevSkippedIndexed.count", "Count Sourcegraph reported for this repo/ref before running the details search", False, "integer", @@ -1203,6 +1216,24 @@ def validate(data: dict[str, Any]) -> None: ), ] +SKIPPED_FILE_METRIC_COLUMNS: list[tuple[str, str, bool, str]] = [ + ( + "file.distinctTrigramCount", + "Distinct three-rune trigrams computed from GitBlob.content. " + "Only populated for skipped files whose reason is `contains too many trigrams`", + False, + "integer", + ), +] + + +def skipped_file_reason_column_names(skipped_file_metrics: bool) -> list[str]: + """Return skipped-file detail CSV columns for the enabled modes""" + columns = [name for name, _, _, _ in SKIPPED_FILE_REASON_COLUMNS] + if skipped_file_metrics: + columns.extend(name for name, _, _, _ in SKIPPED_FILE_METRIC_COLUMNS) + return columns + # --- Statistics --------------------------------------------------------------- @@ -1449,6 +1480,7 @@ def write_csv_schema(path: Path) -> None: cloning_list = format_columns_list(name_desc(CLONING_ERROR_EXTRA_COLUMNS)) skipped_list = format_columns_list(name_desc(SKIPPED_FILES_EXTRA_COLUMNS)) skipped_reason_list = format_columns_list(SKIPPED_FILE_REASON_COLUMNS) + skipped_metrics_list = format_columns_list(SKIPPED_FILE_METRIC_COLUMNS) commit_count_list = format_columns_list(COMMIT_COUNT_COLUMNS) run_search_list = format_columns_list(RUN_SEARCH_COLUMNS) stats_files_list = format_stats_files_list() @@ -1476,7 +1508,7 @@ def write_csv_schema(path: Path) -> None: | `-{DEFAULT_CLONING_ERRORS_FILE}` | at least one repo has a cloning error | main columns + cloning-error extras | | `-{DEFAULT_INDEXING_ERRORS_FILE}` | at least one repo is cloned but is missing a search index | main columns | | `-{DEFAULT_SKIPPED_FILES_FILE}` | `--skipped-files` is set and the last index excluded some files | main columns + skipped-files extras | -| `-{DEFAULT_SKIPPED_FILE_REASONS_FILE}` | `--skipped-files-reason` is set without `REPO[@REV]` | skipped-file reason columns | +| `-{DEFAULT_SKIPPED_FILE_REASONS_FILE}` | `--skipped-files-reason` is set without `REPO[@REV]` | skipped-file reason columns, plus skipped-file metrics columns when `--skipped-file-metrics` is set | | `-{DEFAULT_STATS_FILE_PREFIX}-*.csv` | `--statistics` is set | `bucket,count` (see Statistics section) | The optional `--count-commits` and `--run-search` flags append extra @@ -1509,6 +1541,14 @@ def write_csv_schema(path: Path) -> None: {skipped_reason_list} +## Skipped-file metrics columns + +Appended to skipped-file detail CSVs only when `--skipped-file-metrics` +is used. These columns may require extra GitBlob content requests, and are +therefore not fetched by default + +{skipped_metrics_list} + ## `--count-commits` columns Appended to CSV files when `--count-commits` is used @@ -2212,6 +2252,97 @@ def skipped_file_reason(match: dict[str, Any]) -> str: return "" +def skipped_file_needs_metrics(match: dict[str, Any]) -> bool: + """Return True when extra GitBlob metrics are useful for this skipped file""" + return skipped_file_reason(match) == TOO_MANY_TRIGRAMS_REASON + + +def distinct_trigram_count(content: str) -> int: + """Return the count of distinct three-rune trigrams in content""" + if len(content) < 3: + return 0 + return len({content[index : index + 3] for index in range(len(content) - 2)}) + + +def fetch_blob_distinct_trigram_count( + endpoint: str, + token: str, + repo_name: str, + rev: str, + file_path: str, + max_retries: int, +) -> int: + """Fetch GitBlob.content, compute trigram count, and retain only the count""" + + def validate(data: dict[str, Any]) -> None: + repository = require_dict(data.get("repository"), "blob-content repository") + commit = require_dict(repository.get("commit"), "blob-content commit") + blob = require_dict(commit.get("blob"), "blob-content blob") + content = blob.get("content") + if not isinstance(content, str): + msg = "blob-content content missing or not a string" + raise GraphQLError(msg) + + data = graphql_request_with_validation( + endpoint, + token, + SKIPPED_FILE_BLOB_CONTENT_QUERY, + {"repo": repo_name, "rev": rev, "path": file_path}, + timeout=REQUEST_TIMEOUT_SECONDS_WITH_COMMIT_COUNT, + max_retries=max_retries, + request_description=f"Blob content for {repo_name}@{rev}:{file_path}", + validate=validate, + ) + repository = require_dict(data.get("repository"), "blob-content repository") + commit = require_dict(repository.get("commit"), "blob-content commit") + blob = require_dict(commit.get("blob"), "blob-content blob") + content = blob.get("content") + if not isinstance(content, str): + msg = "blob-content content missing or not a string" + raise GraphQLError(msg) + count = distinct_trigram_count(content) + # Do not retain GitBlob.content longer than needed. The count is the only + # metric stored for CSV output. + del content, blob, commit, repository, data + return count + + +def collect_distinct_trigram_counts( + endpoint: str, + token: str, + repo_name: str, + rev: str, + matches: list[dict[str, Any]], + max_retries: int, +) -> dict[str, int]: + """Fetch blob content and compute distinct trigram counts where needed""" + counts: dict[str, int] = {} + for match in matches: + if not skipped_file_needs_metrics(match): + continue + file_obj: dict[str, Any] = match.get("file") or {} + file_path = str(file_obj.get("path") or "") + if not file_path: + logger.error( + "Cannot fetch skipped-file metrics for %s@%s: search result has no file path", + repo_name, + rev, + ) + continue + try: + counts[file_path] = fetch_blob_distinct_trigram_count( + endpoint, + token, + repo_name, + rev, + file_path, + max_retries, + ) + except REQUEST_FAILURE_TYPES: + continue + return counts + + def skipped_file_query_revision(query: str, fallback: str) -> str: """Return the @rev term from a skippedIndexed query, or fallback""" match = re.search(r"\br:\S+@([^\s]+)", query) @@ -2321,6 +2452,7 @@ def write_skipped_files_reason( token: str, repo_rev: str, max_retries: int = DEFAULT_MAX_RETRIES, + skipped_file_metrics: bool = False, ) -> None: """Fetch skipped-file matches for repo_rev and write the per-file and stats CSVs""" endpoint_sanitized = sanitize_endpoint_for_filename(endpoint) @@ -2372,12 +2504,21 @@ def match_file_url(m: dict[str, Any]) -> str: str(file_obj.get("path") or ""), ) + distinct_trigram_counts_by_path: dict[str, int] = {} + + def match_distinct_trigram_count(m: dict[str, Any]) -> int | str: + file_obj: dict[str, Any] = m.get("file") or {} + file_path = str(file_obj.get("path") or "") + return distinct_trigram_counts_by_path.get(file_path, "") + file_columns: list[tuple[str, Callable[[dict[str, Any]], Any]]] = [ ("chunkMatches.content", chunk_matches_content), ("file.byteSize", match_file_byte_size), ("file.extension", match_file_extension), ("file_url", match_file_url), ] + if skipped_file_metrics: + file_columns.append(("file.distinctTrigramCount", match_distinct_trigram_count)) stats_columns: list[tuple[str, Callable[[tuple[str, int]], Any]]] = [ ("reason", lambda r: r[0]), ("count", lambda r: r[1]), @@ -2392,6 +2533,15 @@ def match_file_url(m: dict[str, Any]) -> str: max_retries=max_retries, ) matches = query_result.matches + if skipped_file_metrics: + distinct_trigram_counts_by_path = collect_distinct_trigram_counts( + endpoint, + token, + name, + rev, + matches, + max_retries, + ) reason_counts: collections.Counter[str] = collections.Counter() rows: list[list[Any]] = [] @@ -2493,6 +2643,7 @@ class SkippedFileReasonSearchResult: limit_hit: bool alert_title: str | None alert_description: str | None + distinct_trigram_counts_by_path: dict[str, int] error: str | None @@ -2735,6 +2886,7 @@ def collect_skipped_file_reason_search_results( token: str, repo: dict[str, Any], max_retries: int, + skipped_file_metrics: bool, ) -> list[SkippedFileReasonSearchResult]: """Run skipped-file searches for every skipped indexed ref in one repo""" repo_name = str(repo.get("name") or "") @@ -2767,10 +2919,23 @@ def collect_skipped_file_reason_search_results( limit_hit=False, alert_title=None, alert_description=None, + distinct_trigram_counts_by_path={}, error=request_error_summary(error), ), ) continue + distinct_trigram_counts_by_path = ( + collect_distinct_trigram_counts( + endpoint, + token, + repo_name, + revision, + query_result.matches, + max_retries, + ) + if skipped_file_metrics + else {} + ) results.append( SkippedFileReasonSearchResult( repository_name=repo_name, @@ -2781,6 +2946,7 @@ def collect_skipped_file_reason_search_results( limit_hit=query_result.limit_hit, alert_title=query_result.alert_title, alert_description=query_result.alert_description, + distinct_trigram_counts_by_path=distinct_trigram_counts_by_path, error=None, ), ) @@ -2791,6 +2957,7 @@ def write_skipped_file_reason_rows( writer: LazyCSVWriter, endpoint: str, search_results: list[SkippedFileReasonSearchResult], + skipped_file_metrics: bool, ) -> None: """Append skipped-file detail rows from per-ref search results""" for search_result in search_results: @@ -2842,22 +3009,27 @@ def write_skipped_file_reason_rows( file_path = str(file_obj.get("path") or "") byte_size = file_obj.get("byteSize") file_extension = Path(file_path).suffix.lstrip(".") - writer.writerow( - [ + row = [ + search_result.repository_name, + search_result.ref_name, + skipped_file_reason(match), + file_extension, + int(byte_size) if byte_size is not None else "", + search_result.skipped_count, + file_path, + file_url( + endpoint, search_result.repository_name, search_result.ref_name, - skipped_file_reason(match), - file_extension, - int(byte_size) if byte_size is not None else "", - search_result.skipped_count, file_path, - file_url( - endpoint, - search_result.repository_name, - search_result.ref_name, - file_path, - ), - ], + ), + ] + if skipped_file_metrics: + row.append( + search_result.distinct_trigram_counts_by_path.get(file_path, ""), + ) + writer.writerow( + row, ) @@ -2893,6 +3065,7 @@ def collect_repo_processing_result( count_commits_rev: str, run_search_pattern: str | None, skipped_file_reasons: bool, + skipped_file_metrics: bool, max_retries: int, ) -> RepoProcessingResult: """Build the row and run optional per-repo network queries""" @@ -2947,6 +3120,7 @@ def collect_repo_processing_result( token, repo, max_retries, + skipped_file_metrics, ) return RepoProcessingResult( index=index, @@ -3061,6 +3235,7 @@ def iter_repo_processing_results( count_commits_rev: str, run_search_pattern: str | None, skipped_file_reasons: bool, + skipped_file_metrics: bool, concurrency: int, max_retries: int, ) -> Iterator[RepoProcessingResult]: @@ -3090,6 +3265,7 @@ def iter_repo_processing_results( count_commits_rev=count_commits_rev, run_search_pattern=run_search_pattern, skipped_file_reasons=skipped_file_reasons, + skipped_file_metrics=skipped_file_metrics, max_retries=max_retries, ) return @@ -3116,6 +3292,7 @@ def submit_repo( count_commits_rev=count_commits_rev, run_search_pattern=run_search_pattern, skipped_file_reasons=skipped_file_reasons, + skipped_file_metrics=skipped_file_metrics, max_retries=max_retries, ) pending_results[future] = index @@ -3158,6 +3335,7 @@ def write_csv( scope_repo: str | None = None, count_commits_rev: str = "HEAD", run_search_pattern: str | None = None, + skipped_file_metrics: bool = False, page_size: int = PAGE_SIZE, concurrency: int = DEFAULT_CONCURRENCY, max_retries: int = DEFAULT_MAX_RETRIES, @@ -3192,6 +3370,7 @@ def write_csv( count_commits_rev=count_commits_rev, run_search_pattern=run_search_pattern, skipped_file_reasons=skipped_file_reasons_enabled, + skipped_file_metrics=skipped_file_metrics, concurrency=concurrency, max_retries=max_retries, ): @@ -3265,6 +3444,7 @@ def write_csv( skipped_file_reason_writer, endpoint, result.skipped_file_reason_search_results, + skipped_file_metrics, ) return (total, reclone_total, reindex_total) @@ -3459,6 +3639,15 @@ def parse_args(argv: list[str]) -> argparse.Namespace: "all repos with skipped files" ), ) + parser.add_argument( + "--skipped-file-metrics", + action="store_true", + help=( + "With --skipped-files-reason, fetch GitBlob.content only for files " + "skipped because they contain too many trigrams, then append " + "file.distinctTrigramCount" + ), + ) parser.add_argument( "--run-search", metavar="PATTERN", @@ -3582,6 +3771,8 @@ def run(args: argparse.Namespace, endpoint: str, token: str) -> None: "Retry policy: %d retries per GraphQL request (backoff: 1s, 2s, 4s, ...)", args.max_retries, ) + if args.skipped_file_metrics and args.skipped_files_reason is None: + die("--skipped-file-metrics requires --skipped-files-reason") if args.count_commits: # Announce the longer per-repo timeout because this mode can be slow logger.info( @@ -3668,6 +3859,7 @@ def run(args: argparse.Namespace, endpoint: str, token: str) -> None: token, args.skipped_files_reason, max_retries=args.max_retries, + skipped_file_metrics=args.skipped_file_metrics, ) return @@ -3746,7 +3938,7 @@ def run(args: argparse.Namespace, endpoint: str, token: str) -> None: skipped_file_reason_writer = ( LazyCSVWriter( skipped_file_reasons_path, - [name for name, _, _, _ in SKIPPED_FILE_REASON_COLUMNS], + skipped_file_reason_column_names(args.skipped_file_metrics), ) if skipped_file_reasons_path is not None else None @@ -3782,6 +3974,7 @@ def run(args: argparse.Namespace, endpoint: str, token: str) -> None: scope_repo=scope_repo, count_commits_rev=scope_rev, run_search_pattern=run_search_pattern, + skipped_file_metrics=args.skipped_file_metrics, page_size=args.page_size, concurrency=args.concurrency, max_retries=args.max_retries, From d1ad0208593836a77c35572fc45b93c3d8629e90 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Wed, 17 Jun 2026 15:20:22 -0600 Subject: [PATCH 04/26] Use indexed commits for skipped file details Amp-Thread-ID: https://ampcode.com/threads/T-019ecf95-ee6d-7426-b1e7-d724bdf73e0f Co-authored-by: Amp --- repo-troubleshooting/list-repos/list-repos.py | 181 ++++++++++++------ 1 file changed, 123 insertions(+), 58 deletions(-) diff --git a/repo-troubleshooting/list-repos/list-repos.py b/repo-troubleshooting/list-repos/list-repos.py index 902bf75..61ac2e0 100755 --- a/repo-troubleshooting/list-repos/list-repos.py +++ b/repo-troubleshooting/list-repos/list-repos.py @@ -176,6 +176,10 @@ def emit(self, record: logging.LogRecord) -> None: ref { displayName } + indexed + indexedCommit { + oid + } skippedIndexed { count query @@ -586,10 +590,14 @@ def refs_with_skips(repo: dict[str, Any]) -> str: ) -def refs_with_skipped_file_queries(repo: dict[str, Any]) -> list[tuple[str, int, str]]: - """Return (ref name, skipped count, API search query) for refs with skips""" - refs: list[tuple[str, int, str]] = [] +def refs_with_skipped_file_queries( + repo: dict[str, Any], +) -> list[tuple[str, int, str, str]]: + """Return (ref name, skipped count, API search query, indexed commit) tuples""" + refs: list[tuple[str, int, str, str]] = [] for ref in _index_refs(repo): + if ref.get("indexed") is False: + continue skipped: dict[str, Any] = ref.get("skippedIndexed") or {} count = skipped.get("count") if count is None: @@ -600,7 +608,11 @@ def refs_with_skipped_file_queries(repo: dict[str, Any]) -> list[tuple[str, int, ref_node: dict[str, Any] = ref.get("ref") or {} name = str(ref_node.get("displayName") or "") if name: - refs.append((name, skipped_count, str(skipped.get("query") or ""))) + indexed_commit: dict[str, Any] = ref.get("indexedCommit") or {} + indexed_oid = str(indexed_commit.get("oid") or "") + refs.append( + (name, skipped_count, str(skipped.get("query") or ""), indexed_oid) + ) return refs @@ -608,7 +620,9 @@ def refs_with_skipped_file_counts(repo: dict[str, Any]) -> list[tuple[str, int]] """Return (ref name, skipped count) pairs for refs with skipped files""" return [ (name, skipped_count) - for name, skipped_count, _query in refs_with_skipped_file_queries(repo) + for name, skipped_count, _query, _indexed_oid in refs_with_skipped_file_queries( + repo + ) ] @@ -2163,8 +2177,8 @@ def verify_repo_rev( token: str, repo_rev: str, max_retries: int = DEFAULT_MAX_RETRIES, -) -> tuple[str, str]: - """Require repo/rev to resolve to an index; return output rev and skip query""" +) -> tuple[str, str, str]: + """Resolve repo/rev to an indexed ref; return display rev, indexed rev, and skip query""" name = parse_repo_name(repo_rev) rev = parse_repo_rev(repo_rev) data = graphql_request( @@ -2184,52 +2198,72 @@ def verify_repo_rev( text_index: dict[str, Any] | None = repository.get("textSearchIndex") refs: list[dict[str, Any]] = (text_index or {}).get("refs") or [] - target_oid = commit.get("oid") - indexed_oids: set[str] = set() + target_oid = str(commit.get("oid") or "") indexed_names: list[str] = [] default_branch: dict[str, Any] = repository.get("defaultBranch") or {} default_branch_name = str(default_branch.get("displayName") or "") + requested_indexed_ref = default_branch_name if rev == "HEAD" else rev + if not requested_indexed_ref: + requested_indexed_ref = "HEAD" + selected_ref_name = "" + selected_indexed_oid = "" selected_skipped_query = "" - fallback_skipped_query = "" + target_match_ref_name = "" + target_match_indexed_oid = "" + target_match_skipped_query = "" for ref in refs: if not ref.get("indexed"): continue indexed_commit: dict[str, Any] = ref.get("indexedCommit") or {} - oid = indexed_commit.get("oid") - if oid: - indexed_oids.add(str(oid)) + oid = str(indexed_commit.get("oid") or "") ref_node: dict[str, Any] = ref.get("ref") or {} ref_name = str(ref_node.get("displayName") or "?") indexed_names.append(ref_name) - if oid != target_oid: - continue skipped: dict[str, Any] = ref.get("skippedIndexed") or {} skipped_count = int(skipped.get("count") or 0) skipped_query = str(skipped.get("query") or "") - if skipped_count <= 0 or not skipped_query: - continue - if not fallback_skipped_query: - fallback_skipped_query = skipped_query - if ref_name == rev or (rev == "HEAD" and ref_name == default_branch_name): - selected_skipped_query = skipped_query - if target_oid not in indexed_oids: + if ref_name == requested_indexed_ref: + selected_ref_name = ref_name + selected_indexed_oid = oid + if skipped_count > 0: + selected_skipped_query = skipped_query + if oid == target_oid: + target_match_ref_name = ref_name + target_match_indexed_oid = oid + if skipped_count > 0: + target_match_skipped_query = skipped_query + + if not selected_ref_name and target_match_ref_name: + selected_ref_name = target_match_ref_name + selected_indexed_oid = target_match_indexed_oid + selected_skipped_query = target_match_skipped_query + + if not selected_ref_name: if indexed_names: indexed_summary = "\n".join(f" - {n}" for n in indexed_names) else: indexed_summary = " (none)" die( - f"revision {rev!r} (commit {target_oid}) is not currently indexed " - f"in repository {name!r}.\nIndexed refs:\n{indexed_summary}", + f"revision {rev!r} did not match an indexed ref in repository " + f"{name!r}.\nIndexed refs:\n{indexed_summary}", ) - - # When the user didn't specify a rev (or explicitly used "HEAD"), substitute - # the actual default branch name so filenames and URLs read naturally if rev == "HEAD": - return ( - default_branch_name or "HEAD", - selected_skipped_query or fallback_skipped_query, + display_rev = default_branch_name or selected_ref_name or "HEAD" + else: + display_rev = rev + indexed_rev = selected_indexed_oid or display_rev + if selected_indexed_oid and selected_indexed_oid != target_oid: + logger.warning( + "revision %r resolves to commit %s in repository %s, but the text " + "search index has ref %s at commit %s; using the indexed commit for " + "skipped-file details", + rev, + target_oid, + name, + selected_ref_name, + selected_indexed_oid, ) - return rev, selected_skipped_query or fallback_skipped_query + return display_rev, indexed_rev, selected_skipped_query def file_url(endpoint: str, repo_name: str, rev: str, file_path: str) -> str: @@ -2351,6 +2385,17 @@ def skipped_file_query_revision(query: str, fallback: str) -> str: return fallback +def repo_filter_term_at_revision(term: str, revision: str) -> str: + """Return a repo-filter query term pinned to revision""" + for prefix in ("r:", "repo:"): + if term.startswith(prefix): + repo_filter = term[len(prefix) :] + if "@" in repo_filter: + repo_filter = repo_filter.rsplit("@", 1)[0] + return f"{prefix}{repo_filter}@{revision}" + return term + + def skipped_file_reason_search_query( skipped_indexed_query: str, repo_name: str, @@ -2363,13 +2408,22 @@ def skipped_file_reason_search_query( f"r:{repo_filter}@{revision} type:file index:only " f"patternType:regexp ^NOT-INDEXED:" ) - terms = [ - term - for term in skipped_indexed_query.split() - if term != "select:file" - and not term.startswith("count:") - and not term.startswith("timeout:") - ] + terms: list[str] = [] + has_repo_filter = False + for term in skipped_indexed_query.split(): + if ( + term == "select:file" + or term.startswith("count:") + or term.startswith("timeout:") + ): + continue + revised_term = repo_filter_term_at_revision(term, revision) + if term.startswith(("r:", "repo:")): + has_repo_filter = True + terms.append(revised_term) + if not has_repo_filter: + repo_filter = f"^{re.escape(repo_name)}$" + terms.insert(0, f"r:{repo_filter}@{revision}") terms.append("count:all") terms.append(SKIPPED_FILE_REASON_SEARCH_TIMEOUT_PARAMETER) return " ".join(terms) @@ -2463,7 +2517,7 @@ def write_skipped_files_reason( Path(f"{input_prefix}-skipped-files.csv").unlink(missing_ok=True) Path(f"{input_prefix}-skipped-stats.csv").unlink(missing_ok=True) - rev, skipped_indexed_query = verify_repo_rev( + display_rev, indexed_rev, skipped_indexed_query = verify_repo_rev( endpoint, token, repo_rev, @@ -2471,7 +2525,7 @@ def write_skipped_files_reason( ) name = parse_repo_name(repo_rev) name_sanitized = sanitize_for_filename(name) - rev_sanitized = sanitize_for_filename(rev) + rev_sanitized = sanitize_for_filename(display_rev) prefix = f"{endpoint_sanitized}-{name_sanitized}-{rev_sanitized}" files_path = Path(f"{prefix}-skipped-files.csv") stats_path = Path(f"{prefix}-skipped-stats.csv") @@ -2500,7 +2554,7 @@ def match_file_url(m: dict[str, Any]) -> str: return file_url( endpoint, str(repo_obj.get("name") or ""), - rev, + indexed_rev, str(file_obj.get("path") or ""), ) @@ -2513,12 +2567,12 @@ def match_distinct_trigram_count(m: dict[str, Any]) -> int | str: file_columns: list[tuple[str, Callable[[dict[str, Any]], Any]]] = [ ("chunkMatches.content", chunk_matches_content), - ("file.byteSize", match_file_byte_size), ("file.extension", match_file_extension), - ("file_url", match_file_url), + ("file.byteSize", match_file_byte_size), ] if skipped_file_metrics: file_columns.append(("file.distinctTrigramCount", match_distinct_trigram_count)) + file_columns.append(("file_url", match_file_url)) stats_columns: list[tuple[str, Callable[[tuple[str, int]], Any]]] = [ ("reason", lambda r: r[0]), ("count", lambda r: r[1]), @@ -2528,7 +2582,7 @@ def match_distinct_trigram_count(m: dict[str, Any]) -> int | str: endpoint, token, name, - rev, + indexed_rev, skipped_indexed_query, max_retries=max_retries, ) @@ -2538,7 +2592,7 @@ def match_distinct_trigram_count(m: dict[str, Any]) -> int | str: endpoint, token, name, - rev, + indexed_rev, matches, max_retries, ) @@ -2551,23 +2605,27 @@ def match_distinct_trigram_count(m: dict[str, Any]) -> int | str: if reason: reason_counts[reason] += 1 - # Sort by chunkMatches.content so files with the same NOT-INDEXED reason - # are grouped together; ties broken by byteSize, extension, then file_url - # Coerce byteSize to int (treating missing values as -1) so an int/str - # union can't blow up the comparator + # Sort by chunkMatches.content so files with the same NOT-INDEXED reason are + # grouped together; ties broken by extension, byteSize, metric, then file_url. + # Coerce numeric cells to int so an int/str union can't blow up the comparator. + def row_sort_key(row: list[Any]) -> tuple[Any, ...]: + byte_size = row[2] if isinstance(row[2], int) else -1 + metric = row[3] if skipped_file_metrics and isinstance(row[3], int) else -1 + return (row[0], row[1], byte_size, metric, row[-1]) + rows.sort( - key=lambda r: (r[0], r[1] if isinstance(r[1], int) else -1, r[2], r[3]), + key=row_sort_key, ) with files_path.open("w", newline="") as out: writer = csv.writer(out) - writer.writerow([n for n, _ in file_columns]) + writer.writerow([name for name, _ in file_columns]) writer.writerows(rows) files_written = len(rows) with stats_path.open("w", newline="") as out: writer = csv.writer(out) - writer.writerow([n for n, _ in stats_columns]) + writer.writerow([name for name, _ in stats_columns]) for record in reason_counts.most_common(): writer.writerow([extract(record) for _, extract in stats_columns]) @@ -2637,6 +2695,7 @@ class SkippedFileReasonSearchResult: repository_name: str ref_name: str + indexed_rev: str skipped_count: int matches: list[dict[str, Any]] match_count: int | None @@ -2895,16 +2954,20 @@ def collect_skipped_file_reason_search_results( display_ref_name, skipped_count, skipped_indexed_query, + indexed_commit_oid, ) in refs_with_skipped_file_queries( repo, ): - revision = skipped_file_query_revision(skipped_indexed_query, display_ref_name) + indexed_rev = indexed_commit_oid or skipped_file_query_revision( + skipped_indexed_query, + display_ref_name, + ) try: query_result = fetch_skipped_file_reason_query( endpoint, token, repo_name, - revision, + indexed_rev, skipped_indexed_query, max_retries=max_retries, ) @@ -2912,7 +2975,8 @@ def collect_skipped_file_reason_search_results( results.append( SkippedFileReasonSearchResult( repository_name=repo_name, - ref_name=revision, + ref_name=display_ref_name, + indexed_rev=indexed_rev, skipped_count=skipped_count, matches=[], match_count=None, @@ -2929,7 +2993,7 @@ def collect_skipped_file_reason_search_results( endpoint, token, repo_name, - revision, + indexed_rev, query_result.matches, max_retries, ) @@ -2939,7 +3003,8 @@ def collect_skipped_file_reason_search_results( results.append( SkippedFileReasonSearchResult( repository_name=repo_name, - ref_name=revision, + ref_name=display_ref_name, + indexed_rev=indexed_rev, skipped_count=skipped_count, matches=query_result.matches, match_count=query_result.match_count, @@ -3020,7 +3085,7 @@ def write_skipped_file_reason_rows( file_url( endpoint, search_result.repository_name, - search_result.ref_name, + search_result.indexed_rev, file_path, ), ] From 8a3035d8b4a745103122a0422866ab3f27abd818 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Wed, 17 Jun 2026 15:25:33 -0600 Subject: [PATCH 05/26] Keep generated CSV rows line-oriented Amp-Thread-ID: https://ampcode.com/threads/T-019ed76a-a9e6-753c-b3da-a8fc221c3a64 Co-authored-by: Amp --- repo-troubleshooting/list-repos/list-repos.py | 58 ++++++++++++++----- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/repo-troubleshooting/list-repos/list-repos.py b/repo-troubleshooting/list-repos/list-repos.py index 61ac2e0..3c05c5d 100755 --- a/repo-troubleshooting/list-repos/list-repos.py +++ b/repo-troubleshooting/list-repos/list-repos.py @@ -86,6 +86,7 @@ def emit(self, record: logging.LogRecord) -> None: DEFAULT_SKIPPED_FILES_FILE = "repos-with-skipped-files.csv" DEFAULT_SKIPPED_FILE_REASONS_FILE = "skipped-file-reasons.csv" DEFAULT_STATS_FILE_PREFIX = "stats" +CSV_RECORD_LINE_TERMINATOR = "\n" DEFAULT_MAX_RETRIES = 5 GRAPHQL_FIELD_COUNT_RETRY_HEADROOM_PERCENT = 95 PAGE_SIZE = 500 @@ -115,6 +116,28 @@ def emit(self, record: logging.LogRecord) -> None: TOO_MANY_TRIGRAMS_REASON = "contains too many trigrams" +def normalize_csv_value(value: Any) -> Any: + """Keep CSV records one physical line while preserving text separators""" + if not isinstance(value, str): + return value + return value.replace("\r\n", "\\n").replace("\r", "\\n").replace("\n", "\\n") + + +def normalize_csv_row(row: list[Any]) -> list[Any]: + """Return a row safe for line-oriented CSV consumers""" + return [normalize_csv_value(value) for value in row] + + +def make_csv_writer(out: TextIO) -> Any: + """Return the repo's CSV writer with Unix record endings""" + return csv.writer(out, lineterminator=CSV_RECORD_LINE_TERMINATOR) + + +def write_csv_row(writer: Any, row: list[Any]) -> None: + """Write one normalized CSV row""" + writer.writerow(normalize_csv_row(row)) + + # --- GraphQL queries ---------------------------------------------------------- # Shared fields for full-listing and single-repo queries. The text-search @@ -1425,12 +1448,12 @@ def write_stats(prefix: str, stats: StatsCollector) -> list[Path]: path = Path(f"{prefix}-{DEFAULT_STATS_FILE_PREFIX}-{suffix}.csv") counter: collections.Counter[str] = getattr(stats, attr) with path.open("w", newline="") as out: - writer = csv.writer(out) - writer.writerow(["bucket", "count"]) + writer = make_csv_writer(out) + write_csv_row(writer, ["bucket", "count"]) for label, _lo, _hi in buckets: - writer.writerow([label, counter.get(label, 0)]) + write_csv_row(writer, [label, counter.get(label, 0)]) for metric, value in summary_builder(stats): - writer.writerow([metric, value]) + write_csv_row(writer, [metric, value]) written.append(path) return written @@ -2618,16 +2641,17 @@ def row_sort_key(row: list[Any]) -> tuple[Any, ...]: ) with files_path.open("w", newline="") as out: - writer = csv.writer(out) - writer.writerow([name for name, _ in file_columns]) - writer.writerows(rows) + writer = make_csv_writer(out) + write_csv_row(writer, [name for name, _ in file_columns]) + for row in rows: + write_csv_row(writer, row) files_written = len(rows) with stats_path.open("w", newline="") as out: - writer = csv.writer(out) - writer.writerow([name for name, _ in stats_columns]) + writer = make_csv_writer(out) + write_csv_row(writer, [name for name, _ in stats_columns]) for record in reason_counts.most_common(): - writer.writerow([extract(record) for _, extract in stats_columns]) + write_csv_row(writer, [extract(record) for _, extract in stats_columns]) logger.info( "Wrote %d skipped-file match(es) to %s", @@ -2657,9 +2681,9 @@ def __init__(self, path: Path, columns: list[str]) -> None: def writerow(self, row: list[Any]) -> None: if self._writer is None: self._file = self.path.open("w", newline="") - self._writer = csv.writer(self._file) - self._writer.writerow(self.columns) - self._writer.writerow(row) + self._writer = make_csv_writer(self._file) + write_csv_row(self._writer, self.columns) + write_csv_row(self._writer, row) self.count += 1 def __enter__(self) -> LazyCSVWriter: @@ -3411,8 +3435,9 @@ def write_csv( """Stream repos to CSVs and optionally trigger reclone/reindex mutations""" run_search_enabled = run_search_pattern is not None skipped_file_reasons_enabled = skipped_file_reason_writer is not None - writer = csv.writer(out) - writer.writerow( + writer = make_csv_writer(out) + write_csv_row( + writer, csv_columns_for( CSV_COLUMNS, count_commits=count_commits, @@ -3446,7 +3471,8 @@ def write_csv( count_commits=count_commits, run_search_pattern=run_search_pattern, ) - writer.writerow( + write_csv_row( + writer, append_processing_result_columns( row, result, From e8588b2d45270d9ef4c117e8f2a6d9c032c0d92a Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Wed, 17 Jun 2026 15:29:13 -0600 Subject: [PATCH 06/26] Use standard CSV record terminators Amp-Thread-ID: https://ampcode.com/threads/T-019ed76a-a9e6-753c-b3da-a8fc221c3a64 Co-authored-by: Amp --- repo-troubleshooting/list-repos/list-repos.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repo-troubleshooting/list-repos/list-repos.py b/repo-troubleshooting/list-repos/list-repos.py index 3c05c5d..e345725 100755 --- a/repo-troubleshooting/list-repos/list-repos.py +++ b/repo-troubleshooting/list-repos/list-repos.py @@ -86,7 +86,7 @@ def emit(self, record: logging.LogRecord) -> None: DEFAULT_SKIPPED_FILES_FILE = "repos-with-skipped-files.csv" DEFAULT_SKIPPED_FILE_REASONS_FILE = "skipped-file-reasons.csv" DEFAULT_STATS_FILE_PREFIX = "stats" -CSV_RECORD_LINE_TERMINATOR = "\n" +CSV_RECORD_LINE_TERMINATOR = "\r\n" DEFAULT_MAX_RETRIES = 5 GRAPHQL_FIELD_COUNT_RETRY_HEADROOM_PERCENT = 95 PAGE_SIZE = 500 @@ -129,7 +129,7 @@ def normalize_csv_row(row: list[Any]) -> list[Any]: def make_csv_writer(out: TextIO) -> Any: - """Return the repo's CSV writer with Unix record endings""" + """Return the repo's CSV writer with standard CSV record endings""" return csv.writer(out, lineterminator=CSV_RECORD_LINE_TERMINATOR) From dffff908f440681b06c80b3c262623a750421f36 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Wed, 17 Jun 2026 15:36:50 -0600 Subject: [PATCH 07/26] Avoid log creation for help output Amp-Thread-ID: https://ampcode.com/threads/T-019ed782-1e1b-760a-ae57-6cc141bb14f5 Co-authored-by: Amp --- repo-troubleshooting/list-repos/list-repos.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/repo-troubleshooting/list-repos/list-repos.py b/repo-troubleshooting/list-repos/list-repos.py index e345725..9340978 100755 --- a/repo-troubleshooting/list-repos/list-repos.py +++ b/repo-troubleshooting/list-repos/list-repos.py @@ -4189,12 +4189,15 @@ def log_run_issue_summary() -> None: def main() -> None: - """Entry point: configure logging, load env, parse args, run, handle errors""" + """Entry point: parse args, configure logging, load env, run, handle errors""" + args = parse_args(sys.argv[1:]) + configure_logging(timestamped_log_path()) - # Include pre-run failures in the timestamped log file + # Include credential, network, and run failures in the timestamped log file. + # ArgumentParser handles --help and parse errors before logging is configured + # so informational CLI exits do not create log files. sys.excepthook = _log_uncaught_exception - args = parse_args(sys.argv[1:]) # Schema generation is offline and credential-free if args.write_csv_schema: try: From 7a061c7dbc7cfeefd8258d378198d7de7ad133fb Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Wed, 17 Jun 2026 15:41:39 -0600 Subject: [PATCH 08/26] Place skipped file metric before skipped count Amp-Thread-ID: https://ampcode.com/threads/T-019ecf95-ee6d-7426-b1e7-d724bdf73e0f Co-authored-by: Amp --- repo-troubleshooting/list-repos/list-repos.py | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/repo-troubleshooting/list-repos/list-repos.py b/repo-troubleshooting/list-repos/list-repos.py index 9340978..ce47573 100755 --- a/repo-troubleshooting/list-repos/list-repos.py +++ b/repo-troubleshooting/list-repos/list-repos.py @@ -1266,9 +1266,11 @@ def validate(data: dict[str, Any]) -> None: def skipped_file_reason_column_names(skipped_file_metrics: bool) -> list[str]: """Return skipped-file detail CSV columns for the enabled modes""" - columns = [name for name, _, _, _ in SKIPPED_FILE_REASON_COLUMNS] - if skipped_file_metrics: - columns.extend(name for name, _, _, _ in SKIPPED_FILE_METRIC_COLUMNS) + columns: list[str] = [] + for name, _, _, _ in SKIPPED_FILE_REASON_COLUMNS: + columns.append(name) + if skipped_file_metrics and name == "file.byteSize": + columns.extend(name for name, _, _, _ in SKIPPED_FILE_METRIC_COLUMNS) return columns @@ -3104,19 +3106,23 @@ def write_skipped_file_reason_rows( skipped_file_reason(match), file_extension, int(byte_size) if byte_size is not None else "", - search_result.skipped_count, - file_path, - file_url( - endpoint, - search_result.repository_name, - search_result.indexed_rev, - file_path, - ), ] if skipped_file_metrics: row.append( search_result.distinct_trigram_counts_by_path.get(file_path, ""), ) + row.extend( + [ + search_result.skipped_count, + file_path, + file_url( + endpoint, + search_result.repository_name, + search_result.indexed_rev, + file_path, + ), + ], + ) writer.writerow( row, ) From 09f9c050b1a4b536668ea9b4f6cb761314aca9b1 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Wed, 17 Jun 2026 15:55:59 -0600 Subject: [PATCH 09/26] Update helper text --- repo-troubleshooting/list-repos/list-repos.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/repo-troubleshooting/list-repos/list-repos.py b/repo-troubleshooting/list-repos/list-repos.py index ce47573..7a39442 100755 --- a/repo-troubleshooting/list-repos/list-repos.py +++ b/repo-troubleshooting/list-repos/list-repos.py @@ -3740,9 +3740,8 @@ def parse_args(argv: list[str]) -> argparse.Namespace: "--skipped-file-metrics", action="store_true", help=( - "With --skipped-files-reason, fetch GitBlob.content only for files " - "skipped because they contain too many trigrams, then append " - "file.distinctTrigramCount" + "With --skipped-files-reason, pulls file contents for local metrics analysis\n" + "eg. calculates the number of trigrams in a file for files with too many trigrams" ), ) parser.add_argument( @@ -3812,15 +3811,18 @@ def parse_args(argv: list[str]) -> argparse.Namespace: "--src-endpoint", default=None, metavar="URL", - help="Sourcegraph endpoint URL (e.g. https://sourcegraph.example.com)", + help=( + "Sourcegraph endpoint URL (e.g. https://sourcegraph.example.com)\n" + "Prefer to use the SRC_ENDPOINT environment variable for security" + ), ) parser.add_argument( "--src-access-token", default=None, metavar="TOKEN", help=( - "Sourcegraph access token (must start with 'sgp_'); prefer the " - "SRC_ACCESS_TOKEN environment variable" + "Sourcegraph access token (must start with 'sgp_')\n" + "Prefer to use the SRC_ACCESS_TOKEN environment variable for security" ), ) return parser.parse_args(argv) From 093396771148f2538bb75b5658196f4bc79ed572 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Wed, 17 Jun 2026 16:13:53 -0600 Subject: [PATCH 10/26] Write list repos outputs to lazy run directories Amp-Thread-ID: https://ampcode.com/threads/T-019ecf95-ee6d-7426-b1e7-d724bdf73e0f Co-authored-by: Amp --- repo-troubleshooting/list-repos/CSV_SCHEMA.md | 25 +- repo-troubleshooting/list-repos/list-repos.py | 231 +++++++++--------- 2 files changed, 126 insertions(+), 130 deletions(-) diff --git a/repo-troubleshooting/list-repos/CSV_SCHEMA.md b/repo-troubleshooting/list-repos/CSV_SCHEMA.md index 7729d0e..e789e4c 100644 --- a/repo-troubleshooting/list-repos/CSV_SCHEMA.md +++ b/repo-troubleshooting/list-repos/CSV_SCHEMA.md @@ -11,18 +11,19 @@ to the repository ## Output files -The script prefixes output file names with the sanitized Sourcegraph endpoint -(e.g. `sourcegraph.example.com-repos.csv`), -so the script can run against multiple instances without overwriting files +Each run writes outputs under +`list-repos-runs///`, so each run has its +own directory. Files are created lazily: a CSV is absent when that run had no +rows for it | File | Written when | Columns | | --- | --- | --- | -| `-repos.csv` | always | main columns | -| `-repos-with-cloning-errors.csv` | at least one repo has a cloning error | main columns + cloning-error extras | -| `-repos-with-indexing-errors.csv` | at least one repo is cloned but is missing a search index | main columns | -| `-repos-with-skipped-files.csv` | `--skipped-files` is set and the last index excluded some files | main columns + skipped-files extras | -| `-skipped-file-reasons.csv` | `--skipped-files-reason` is set without `REPO[@REV]` | skipped-file reason columns, plus skipped-file metrics columns when `--skipped-file-metrics` is set | -| `-stats-*.csv` | `--statistics` is set | `bucket,count` (see Statistics section) | +| `repos.csv` | at least one repo row is written | main columns | +| `repos-with-cloning-errors.csv` | at least one repo has a cloning error | main columns + cloning-error extras | +| `repos-with-indexing-errors.csv` | at least one repo is cloned but is missing a search index | main columns | +| `repos-with-skipped-files.csv` | `--skipped-files` is set and the last index excluded files in at least one repo | main columns + skipped-files extras | +| `skipped-file-reasons.csv` | `--skipped-files-reason` is set without `REPO[@REV]`, and at least one skipped-file detail row is found | skipped-file reason columns, plus skipped-file metrics columns when `--skipped-file-metrics` is set | +| `stats-*.csv` | `--statistics` is set and repo rows were processed | `bucket,count` (see Statistics section) | The optional `--count-commits` and `--run-search` flags append extra columns to the repo-listing CSVs above, excluding the `--statistics` @@ -67,7 +68,7 @@ These are written to every repo-listing CSV file ## Cloning-error extras -Appended to `-repos-with-cloning-errors.csv` +Appended to `repos-with-cloning-errors.csv` | Column | Type | Requires admin | Description | | --- | --- | --- | --- | @@ -78,7 +79,7 @@ Appended to `-repos-with-cloning-errors.csv` ## Skipped-files extras -Appended to `-repos-with-skipped-files.csv` +Appended to `repos-with-skipped-files.csv` | Column | Type | Requires admin | Description | | --- | --- | --- | --- | @@ -88,7 +89,7 @@ Appended to `-repos-with-skipped-files.csv` ## Skipped-file reason columns -Written to `-skipped-file-reasons.csv` when +Written to `skipped-file-reasons.csv` when `--skipped-files-reason` is used without `REPO[@REV]` | Column | Type | Requires admin | Description | diff --git a/repo-troubleshooting/list-repos/list-repos.py b/repo-troubleshooting/list-repos/list-repos.py index 7a39442..150ec6a 100755 --- a/repo-troubleshooting/list-repos/list-repos.py +++ b/repo-troubleshooting/list-repos/list-repos.py @@ -83,6 +83,7 @@ def emit(self, record: logging.LogRecord) -> None: DEFAULT_INDEXING_ERRORS_FILE = "repos-with-indexing-errors.csv" DEFAULT_LOG_FILE_STEM = "list-repos" DEFAULT_OUTPUT_FILE = "repos.csv" +DEFAULT_RUNS_DIR = "list-repos-runs" DEFAULT_SKIPPED_FILES_FILE = "repos-with-skipped-files.csv" DEFAULT_SKIPPED_FILE_REASONS_FILE = "skipped-file-reasons.csv" DEFAULT_STATS_FILE_PREFIX = "stats" @@ -1443,20 +1444,21 @@ def add(self, repo: dict[str, Any]) -> None: ] -def write_stats(prefix: str, stats: StatsCollector) -> list[Path]: +def write_stats(output_dir: Path, stats: StatsCollector) -> list[Path]: """Write one bucket/count CSV per stat and return the paths written""" written: list[Path] = [] for suffix, _desc, buckets, attr, summary_builder in STATS_FILES: - path = Path(f"{prefix}-{DEFAULT_STATS_FILE_PREFIX}-{suffix}.csv") counter: collections.Counter[str] = getattr(stats, attr) - with path.open("w", newline="") as out: - writer = make_csv_writer(out) - write_csv_row(writer, ["bucket", "count"]) + with LazyCSVWriter( + output_dir / f"{DEFAULT_STATS_FILE_PREFIX}-{suffix}.csv", + ["bucket", "count"], + ) as writer: for label, _lo, _hi in buckets: - write_csv_row(writer, [label, counter.get(label, 0)]) + writer.writerow([label, counter.get(label, 0)]) for metric, value in summary_builder(stats): - write_csv_row(writer, [metric, value]) - written.append(path) + writer.writerow([metric, value]) + if writer.count: + written.append(writer.path) return written @@ -1537,18 +1539,19 @@ def write_csv_schema(path: Path) -> None: ## Output files -The script prefixes output file names with the sanitized Sourcegraph endpoint -(e.g. `sourcegraph.example.com-repos.csv`), -so the script can run against multiple instances without overwriting files +Each run writes outputs under +`{DEFAULT_RUNS_DIR}///`, so each run has its +own directory. Files are created lazily: a CSV is absent when that run had no +rows for it | File | Written when | Columns | | --- | --- | --- | -| `-{DEFAULT_OUTPUT_FILE}` | always | main columns | -| `-{DEFAULT_CLONING_ERRORS_FILE}` | at least one repo has a cloning error | main columns + cloning-error extras | -| `-{DEFAULT_INDEXING_ERRORS_FILE}` | at least one repo is cloned but is missing a search index | main columns | -| `-{DEFAULT_SKIPPED_FILES_FILE}` | `--skipped-files` is set and the last index excluded some files | main columns + skipped-files extras | -| `-{DEFAULT_SKIPPED_FILE_REASONS_FILE}` | `--skipped-files-reason` is set without `REPO[@REV]` | skipped-file reason columns, plus skipped-file metrics columns when `--skipped-file-metrics` is set | -| `-{DEFAULT_STATS_FILE_PREFIX}-*.csv` | `--statistics` is set | `bucket,count` (see Statistics section) | +| `{DEFAULT_OUTPUT_FILE}` | at least one repo row is written | main columns | +| `{DEFAULT_CLONING_ERRORS_FILE}` | at least one repo has a cloning error | main columns + cloning-error extras | +| `{DEFAULT_INDEXING_ERRORS_FILE}` | at least one repo is cloned but is missing a search index | main columns | +| `{DEFAULT_SKIPPED_FILES_FILE}` | `--skipped-files` is set and the last index excluded files in at least one repo | main columns + skipped-files extras | +| `{DEFAULT_SKIPPED_FILE_REASONS_FILE}` | `--skipped-files-reason` is set without `REPO[@REV]`, and at least one skipped-file detail row is found | skipped-file reason columns, plus skipped-file metrics columns when `--skipped-file-metrics` is set | +| `{DEFAULT_STATS_FILE_PREFIX}-*.csv` | `--statistics` is set and repo rows were processed | `bucket,count` (see Statistics section) | The optional `--count-commits` and `--run-search` flags append extra columns to the repo-listing CSVs above, excluding the `--statistics` @@ -1563,19 +1566,19 @@ def write_csv_schema(path: Path) -> None: ## Cloning-error extras -Appended to `-{DEFAULT_CLONING_ERRORS_FILE}` +Appended to `{DEFAULT_CLONING_ERRORS_FILE}` {cloning_list} ## Skipped-files extras -Appended to `-{DEFAULT_SKIPPED_FILES_FILE}` +Appended to `{DEFAULT_SKIPPED_FILES_FILE}` {skipped_list} ## Skipped-file reason columns -Written to `-{DEFAULT_SKIPPED_FILE_REASONS_FILE}` when +Written to `{DEFAULT_SKIPPED_FILE_REASONS_FILE}` when `--skipped-files-reason` is used without `REPO[@REV]` {skipped_reason_list} @@ -2530,18 +2533,11 @@ def write_skipped_files_reason( endpoint: str, token: str, repo_rev: str, + output_dir: Path, max_retries: int = DEFAULT_MAX_RETRIES, skipped_file_metrics: bool = False, ) -> None: """Fetch skipped-file matches for repo_rev and write the per-file and stats CSVs""" - endpoint_sanitized = sanitize_endpoint_for_filename(endpoint) - # Remove raw-input outputs before validation so failures cannot leave stale CSVs - input_name_sanitized = sanitize_for_filename(parse_repo_name(repo_rev)) - input_rev_sanitized = sanitize_for_filename(parse_repo_rev(repo_rev)) - input_prefix = f"{endpoint_sanitized}-{input_name_sanitized}-{input_rev_sanitized}" - Path(f"{input_prefix}-skipped-files.csv").unlink(missing_ok=True) - Path(f"{input_prefix}-skipped-stats.csv").unlink(missing_ok=True) - display_rev, indexed_rev, skipped_indexed_query = verify_repo_rev( endpoint, token, @@ -2549,15 +2545,6 @@ def write_skipped_files_reason( max_retries=max_retries, ) name = parse_repo_name(repo_rev) - name_sanitized = sanitize_for_filename(name) - rev_sanitized = sanitize_for_filename(display_rev) - prefix = f"{endpoint_sanitized}-{name_sanitized}-{rev_sanitized}" - files_path = Path(f"{prefix}-skipped-files.csv") - stats_path = Path(f"{prefix}-skipped-stats.csv") - # Also remove resolved-rev outputs when they differ from the raw input names - if prefix != input_prefix: - files_path.unlink(missing_ok=True) - stats_path.unlink(missing_ok=True) # Keep each local CSV header beside the extractor that writes its value def chunk_matches_content(m: dict[str, Any]) -> str: @@ -2642,29 +2629,40 @@ def row_sort_key(row: list[Any]) -> tuple[Any, ...]: key=row_sort_key, ) - with files_path.open("w", newline="") as out: - writer = make_csv_writer(out) - write_csv_row(writer, [name for name, _ in file_columns]) + files_writer = LazyCSVWriter( + output_dir / "skipped-files.csv", + [name for name, _ in file_columns], + ) + with files_writer as writer: for row in rows: - write_csv_row(writer, row) - files_written = len(rows) + writer.writerow(row) - with stats_path.open("w", newline="") as out: - writer = make_csv_writer(out) - write_csv_row(writer, [name for name, _ in stats_columns]) + stats_writer = LazyCSVWriter( + output_dir / "skipped-stats.csv", + [name for name, _ in stats_columns], + ) + with stats_writer as writer: for record in reason_counts.most_common(): - write_csv_row(writer, [extract(record) for _, extract in stats_columns]) + writer.writerow([extract(record) for _, extract in stats_columns]) - logger.info( - "Wrote %d skipped-file match(es) to %s", - files_written, - files_path.name, - ) - logger.info( - "Wrote %d NOT-INDEXED reason categor(ies) to %s", - len(reason_counts), - stats_path.name, - ) + if files_writer.count: + logger.info( + "Wrote %d skipped-file match(es) to %s", + files_writer.count, + files_writer.path.name, + ) + else: + logger.info( + "No skipped-file matches found for %s@%s; skipped-files.csv not written", + name, + display_rev, + ) + if stats_writer.count: + logger.info( + "Wrote %d NOT-INDEXED reason categor(ies) to %s", + stats_writer.count, + stats_writer.path.name, + ) # --- Repo CSV pipeline -------------------------------------------------------- @@ -2682,6 +2680,7 @@ def __init__(self, path: Path, columns: list[str]) -> None: def writerow(self, row: list[Any]) -> None: if self._writer is None: + self.path.parent.mkdir(parents=True, exist_ok=True) self._file = self.path.open("w", newline="") self._writer = make_csv_writer(self._file) write_csv_row(self._writer, self.columns) @@ -3415,7 +3414,7 @@ def fill_pending(executor: concurrent.futures.ThreadPoolExecutor) -> None: def write_csv( - out: TextIO, + output_writer: LazyCSVWriter, cloning_writer: LazyCSVWriter, indexing_writer: LazyCSVWriter, skipped_writer: LazyCSVWriter | None, @@ -3441,15 +3440,6 @@ def write_csv( """Stream repos to CSVs and optionally trigger reclone/reindex mutations""" run_search_enabled = run_search_pattern is not None skipped_file_reasons_enabled = skipped_file_reason_writer is not None - writer = make_csv_writer(out) - write_csv_row( - writer, - csv_columns_for( - CSV_COLUMNS, - count_commits=count_commits, - run_search=run_search_enabled, - ), - ) total = 0 reclone_total = 0 @@ -3477,8 +3467,7 @@ def write_csv( count_commits=count_commits, run_search_pattern=run_search_pattern, ) - write_csv_row( - writer, + output_writer.writerow( append_processing_result_columns( row, result, @@ -3864,7 +3853,7 @@ def collect_scope(args: argparse.Namespace) -> tuple[str, str] | None: return repo_name, rev -def run(args: argparse.Namespace, endpoint: str, token: str) -> None: +def run(args: argparse.Namespace, endpoint: str, token: str, output_dir: Path) -> None: """Confirm the connection, then stream every repo to the CSV file""" logger.info( "Retry policy: %d retries per GraphQL request (backoff: 1s, 2s, 4s, ...)", @@ -3957,6 +3946,7 @@ def run(args: argparse.Namespace, endpoint: str, token: str) -> None: endpoint, token, args.skipped_files_reason, + output_dir, max_retries=args.max_retries, skipped_file_metrics=args.skipped_file_metrics, ) @@ -3968,44 +3958,30 @@ def run(args: argparse.Namespace, endpoint: str, token: str) -> None: max_retries=args.max_retries, ) - # Prefix outputs with endpoint, plus scoped repo/rev when applicable - endpoint_sanitized = sanitize_endpoint_for_filename(endpoint) - if scope_repo is not None: - scope_suffix = sanitize_for_filename(scope_repo) - # Only --count-commits uses rev; reclone/reindex filenames stay repo-only - if args.count_commits and scope_rev != "HEAD": - scope_suffix = f"{scope_suffix}-{sanitize_for_filename(scope_rev)}" - prefix = f"{endpoint_sanitized}-{scope_suffix}" - else: - prefix = endpoint_sanitized - output_path = Path(f"{prefix}-{DEFAULT_OUTPUT_FILE}") - cloning_errors_path = Path(f"{prefix}-{DEFAULT_CLONING_ERRORS_FILE}") - indexing_errors_path = Path(f"{prefix}-{DEFAULT_INDEXING_ERRORS_FILE}") + output_path = output_dir / DEFAULT_OUTPUT_FILE + cloning_errors_path = output_dir / DEFAULT_CLONING_ERRORS_FILE + indexing_errors_path = output_dir / DEFAULT_INDEXING_ERRORS_FILE skipped_files_path = ( - Path(f"{prefix}-{DEFAULT_SKIPPED_FILES_FILE}") if args.skipped_files else None + output_dir / DEFAULT_SKIPPED_FILES_FILE if args.skipped_files else None ) skipped_file_reasons_path = ( - Path(f"{prefix}-{DEFAULT_SKIPPED_FILE_REASONS_FILE}") + output_dir / DEFAULT_SKIPPED_FILE_REASONS_FILE if args.skipped_files_reason is True else None ) - # Remove stale optional outputs; LazyCSVWriter recreates only non-empty ones - cloning_errors_path.unlink(missing_ok=True) - indexing_errors_path.unlink(missing_ok=True) - if skipped_files_path is not None: - skipped_files_path.unlink(missing_ok=True) - if skipped_file_reasons_path is not None: - skipped_file_reasons_path.unlink(missing_ok=True) - # Clear stale stats outputs even when --statistics is not enabled this run - for suffix, *_ in STATS_FILES: - Path(f"{prefix}-{DEFAULT_STATS_FILE_PREFIX}-{suffix}.csv").unlink( - missing_ok=True, - ) stats = StatsCollector() if args.statistics else None count_commits_enabled = bool(args.count_commits) run_search_pattern: str | None = args.run_search run_search_enabled = run_search_pattern is not None + output_writer = LazyCSVWriter( + output_path, + csv_columns_for( + CSV_COLUMNS, + count_commits=count_commits_enabled, + run_search=run_search_enabled, + ), + ) cloning_writer = LazyCSVWriter( cloning_errors_path, csv_columns_for( @@ -4052,14 +4028,14 @@ def run(args: argparse.Namespace, endpoint: str, token: str) -> None: else contextlib.nullcontext() ) with ( - output_path.open("w", newline="") as out, + output_writer, cloning_writer, indexing_writer, skipped_cm, skipped_file_reason_cm, ): total, reclone_total, reindex_total = write_csv( - out, + output_writer, cloning_writer, indexing_writer, skipped_writer, @@ -4082,12 +4058,17 @@ def run(args: argparse.Namespace, endpoint: str, token: str) -> None: include_index_failure_fields=include_index_failure_fields, ) - if stats is not None: - stats_paths = write_stats(prefix, stats) + if stats is not None and total: + stats_paths = write_stats(output_dir, stats) for stats_path in stats_paths: logger.info("Wrote statistics to %s", stats_path.name) + elif stats is not None: + logger.info("No repo rows processed; statistics files not written") - logger.info("Wrote %d repos to %s", total, output_path.name) + if output_writer.count: + logger.info("Wrote %d repos to %s", output_writer.count, output_path.name) + else: + logger.info("No repo rows written; %s not written", output_path.name) if cloning_writer.count: logger.info( "Wrote %d repos with cloning errors to %s", @@ -4137,10 +4118,23 @@ def redact_argv_for_log(argv: list[str]) -> str: return " ".join(shlex.quote(a) for a in redacted) -def timestamped_log_path() -> Path: - """Return list-repos-YYYY-MM-DD-HH-MM-SS.log for this run""" - timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") - return Path(f"{DEFAULT_LOG_FILE_STEM}-{timestamp}.log") +def run_timestamp() -> str: + """Return a filesystem-safe timestamp unique enough for one run directory""" + return datetime.now().strftime("%Y-%m-%d-%H-%M-%S-%f") + + +def run_output_dir(endpoint: str, timestamp: str) -> Path: + """Return the per-run output directory path without creating it""" + endpoint_name = sanitize_endpoint_for_filename(endpoint) or "unknown-endpoint" + return Path(DEFAULT_RUNS_DIR) / endpoint_name / timestamp + + +class LazyDirectoryFileHandler(logging.FileHandler): + """FileHandler that creates its parent directory only when first opened""" + + def _open(self): + Path(self.baseFilename).parent.mkdir(parents=True, exist_ok=True) + return super()._open() def configure_logging(log_path: Path) -> None: @@ -4158,7 +4152,7 @@ def configure_logging(log_path: Path) -> None: stderr_handler.setFormatter(logging.Formatter("%(message)s")) root.addHandler(stderr_handler) - file_handler = logging.FileHandler( + file_handler = LazyDirectoryFileHandler( log_path, mode="w", encoding="utf-8", @@ -4200,29 +4194,30 @@ def main() -> None: """Entry point: parse args, configure logging, load env, run, handle errors""" args = parse_args(sys.argv[1:]) - configure_logging(timestamped_log_path()) - # Include credential, network, and run failures in the timestamped log file. - # ArgumentParser handles --help and parse errors before logging is configured - # so informational CLI exits do not create log files. - sys.excepthook = _log_uncaught_exception - # Schema generation is offline and credential-free if args.write_csv_schema: - try: - write_csv_schema(Path(DEFAULT_CSV_SCHEMA_FILE)) - finally: - log_run_issue_summary() + write_csv_schema(Path(DEFAULT_CSV_SCHEMA_FILE)) return + load_dotenv() + raw_endpoint = args.src_endpoint or os.environ.get("SRC_ENDPOINT", "") + timestamp = run_timestamp() + output_dir = run_output_dir(raw_endpoint, timestamp) + configure_logging(output_dir / f"{DEFAULT_LOG_FILE_STEM}.log") + # Include credential, network, and run failures in the per-run log file. + # ArgumentParser handles --help and parse errors before logging is configured + # so informational CLI exits do not create run directories or log files. + sys.excepthook = _log_uncaught_exception + try: - load_dotenv() endpoint, token = require_credentials(args) logger.info( "Running: %s (SRC_ENDPOINT=%s)", redact_argv_for_log(sys.argv), endpoint, ) - run(args, endpoint, token) + logger.info("Output directory: %s", output_dir) + run(args, endpoint, token, output_dir) except HTTPRequestError as exc: log_http_error(exc) sys.exit(1) From 8819d97d3f0a1b8c9349aa1881fd61770074e956 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Wed, 17 Jun 2026 16:36:10 -0600 Subject: [PATCH 11/26] Rename statistics output flag to stats Amp-Thread-ID: https://ampcode.com/threads/T-019ecf95-ee6d-7426-b1e7-d724bdf73e0f Co-authored-by: Amp --- repo-troubleshooting/list-repos/CSV_SCHEMA.md | 10 +++---- repo-troubleshooting/list-repos/list-repos.py | 28 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/repo-troubleshooting/list-repos/CSV_SCHEMA.md b/repo-troubleshooting/list-repos/CSV_SCHEMA.md index e789e4c..60cce90 100644 --- a/repo-troubleshooting/list-repos/CSV_SCHEMA.md +++ b/repo-troubleshooting/list-repos/CSV_SCHEMA.md @@ -23,10 +23,10 @@ rows for it | `repos-with-indexing-errors.csv` | at least one repo is cloned but is missing a search index | main columns | | `repos-with-skipped-files.csv` | `--skipped-files` is set and the last index excluded files in at least one repo | main columns + skipped-files extras | | `skipped-file-reasons.csv` | `--skipped-files-reason` is set without `REPO[@REV]`, and at least one skipped-file detail row is found | skipped-file reason columns, plus skipped-file metrics columns when `--skipped-file-metrics` is set | -| `stats-*.csv` | `--statistics` is set and repo rows were processed | `bucket,count` (see Statistics section) | +| `stats-*.csv` | `--stats` is set and repo rows were processed | `bucket,count` (see Stats section) | The optional `--count-commits` and `--run-search` flags append extra -columns to the repo-listing CSVs above, excluding the `--statistics` +columns to the repo-listing CSVs above, excluding the `--stats` files and the skipped-file reason detail CSV, in this order: main columns → per-CSV extras → commit-count columns → run-search columns @@ -140,15 +140,15 @@ Appended to CSV files when `--run-search PATTERN` is used | `runSearch.limitHit` | boolean | | `True` when the search hit a limit, so the results are incomplete | | `runSearch.alertTitle` | string | | Title of the search-API alert when the server's `timeout:` budget was exceeded or the query was malformed | -## `--statistics` files +## `--stats` files -- Written when `--statistics` is used +- Written when `--stats` is used - One CSV file per dimension - Each file has two columns listing every bucket in declaration order, followed by per-stat summary rows (totals) appended below the bucket rows - Counts come from the same listing pass that produces the -main CSV, so enabling `--statistics` adds no extra GraphQL requests +main CSV, so enabling `--stats` adds no extra GraphQL requests | File suffix | Buckets | Description | | --- | --- | --- | diff --git a/repo-troubleshooting/list-repos/list-repos.py b/repo-troubleshooting/list-repos/list-repos.py index 150ec6a..50ea3ff 100755 --- a/repo-troubleshooting/list-repos/list-repos.py +++ b/repo-troubleshooting/list-repos/list-repos.py @@ -1275,9 +1275,9 @@ def skipped_file_reason_column_names(skipped_file_metrics: bool) -> list[str]: return columns -# --- Statistics --------------------------------------------------------------- +# --- Stats -------------------------------------------------------------------- -# --statistics buckets repo/content/index sizes and size ratios during listing +# --stats buckets repo/content/index sizes and size ratios during listing # (label, lo_inclusive_mb, hi_exclusive_mb_or_None) — used for the repo and # indexed-content size distributions, which span many orders of magnitude @@ -1325,7 +1325,7 @@ def bucket_label( class StatsCollector: - """Accumulate per-repo size and ratio counts for --statistics""" + """Accumulate per-repo size and ratio counts for --stats""" def __init__(self) -> None: self.mirror_buckets: collections.Counter[str] = collections.Counter() @@ -1551,10 +1551,10 @@ def write_csv_schema(path: Path) -> None: | `{DEFAULT_INDEXING_ERRORS_FILE}` | at least one repo is cloned but is missing a search index | main columns | | `{DEFAULT_SKIPPED_FILES_FILE}` | `--skipped-files` is set and the last index excluded files in at least one repo | main columns + skipped-files extras | | `{DEFAULT_SKIPPED_FILE_REASONS_FILE}` | `--skipped-files-reason` is set without `REPO[@REV]`, and at least one skipped-file detail row is found | skipped-file reason columns, plus skipped-file metrics columns when `--skipped-file-metrics` is set | -| `{DEFAULT_STATS_FILE_PREFIX}-*.csv` | `--statistics` is set and repo rows were processed | `bucket,count` (see Statistics section) | +| `{DEFAULT_STATS_FILE_PREFIX}-*.csv` | `--stats` is set and repo rows were processed | `bucket,count` (see Stats section) | The optional `--count-commits` and `--run-search` flags append extra -columns to the repo-listing CSVs above, excluding the `--statistics` +columns to the repo-listing CSVs above, excluding the `--stats` files and the skipped-file reason detail CSV, in this order: main columns → per-CSV extras → commit-count columns → run-search columns @@ -1603,15 +1603,15 @@ def write_csv_schema(path: Path) -> None: {run_search_list} -## `--statistics` files +## `--stats` files -- Written when `--statistics` is used +- Written when `--stats` is used - One CSV file per dimension - Each file has two columns listing every bucket in declaration order, followed by per-stat summary rows (totals) appended below the bucket rows - Counts come from the same listing pass that produces the -main CSV, so enabling `--statistics` adds no extra GraphQL requests +main CSV, so enabling `--stats` adds no extra GraphQL requests {stats_files_list} @@ -3692,9 +3692,9 @@ def parse_args(argv: list[str]) -> argparse.Namespace: help="Fetch at most repos (>=1)", ) parser.add_argument( - "--statistics", + "--stats", action="store_true", - help="Write statistics CSV files", + help="Write stats CSV files", ) parser.add_argument( "--count-commits", @@ -3932,7 +3932,7 @@ def run(args: argparse.Namespace, endpoint: str, token: str, output_dir: Path) - ("--skipped-files", args.skipped_files), ("--count-commits", args.count_commits), ("--run-search", args.run_search is not None), - ("--statistics", args.statistics), + ("--stats", args.stats), ) if set_ ] @@ -3970,7 +3970,7 @@ def run(args: argparse.Namespace, endpoint: str, token: str, output_dir: Path) - else None ) - stats = StatsCollector() if args.statistics else None + stats = StatsCollector() if args.stats else None count_commits_enabled = bool(args.count_commits) run_search_pattern: str | None = args.run_search run_search_enabled = run_search_pattern is not None @@ -4061,9 +4061,9 @@ def run(args: argparse.Namespace, endpoint: str, token: str, output_dir: Path) - if stats is not None and total: stats_paths = write_stats(output_dir, stats) for stats_path in stats_paths: - logger.info("Wrote statistics to %s", stats_path.name) + logger.info("Wrote stats to %s", stats_path.name) elif stats is not None: - logger.info("No repo rows processed; statistics files not written") + logger.info("No repo rows processed; stats files not written") if output_writer.count: logger.info("Wrote %d repos to %s", output_writer.count, output_path.name) From 0b839dd72191670b4b477e4d4b9b5e2a1338e344 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Wed, 17 Jun 2026 16:54:20 -0600 Subject: [PATCH 12/26] Align skipped file output filenames with args Amp-Thread-ID: https://ampcode.com/threads/T-019ecf95-ee6d-7426-b1e7-d724bdf73e0f Co-authored-by: Amp --- repo-troubleshooting/list-repos/CSV_SCHEMA.md | 11 ++++++----- repo-troubleshooting/list-repos/list-repos.py | 17 ++++++++++------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/repo-troubleshooting/list-repos/CSV_SCHEMA.md b/repo-troubleshooting/list-repos/CSV_SCHEMA.md index 60cce90..ca80f11 100644 --- a/repo-troubleshooting/list-repos/CSV_SCHEMA.md +++ b/repo-troubleshooting/list-repos/CSV_SCHEMA.md @@ -21,8 +21,9 @@ rows for it | `repos.csv` | at least one repo row is written | main columns | | `repos-with-cloning-errors.csv` | at least one repo has a cloning error | main columns + cloning-error extras | | `repos-with-indexing-errors.csv` | at least one repo is cloned but is missing a search index | main columns | -| `repos-with-skipped-files.csv` | `--skipped-files` is set and the last index excluded files in at least one repo | main columns + skipped-files extras | -| `skipped-file-reasons.csv` | `--skipped-files-reason` is set without `REPO[@REV]`, and at least one skipped-file detail row is found | skipped-file reason columns, plus skipped-file metrics columns when `--skipped-file-metrics` is set | +| `skipped-files-repos.csv` | `--skipped-files` is set and the last index excluded files in at least one repo | main columns + skipped-files extras | +| `skipped-files-reason-details.csv` | `--skipped-files-reason` is set, and at least one skipped-file detail row is found | skipped-file reason columns, plus skipped-file metrics columns when `--skipped-file-metrics` is set | +| `skipped-files-reason-stats.csv` | `--skipped-files-reason REPO[@REV]` is set, and at least one NOT-INDEXED reason category is found | `reason,count` | | `stats-*.csv` | `--stats` is set and repo rows were processed | `bucket,count` (see Stats section) | The optional `--count-commits` and `--run-search` flags append extra @@ -79,7 +80,7 @@ Appended to `repos-with-cloning-errors.csv` ## Skipped-files extras -Appended to `repos-with-skipped-files.csv` +Appended to `skipped-files-repos.csv` | Column | Type | Requires admin | Description | | --- | --- | --- | --- | @@ -89,8 +90,8 @@ Appended to `repos-with-skipped-files.csv` ## Skipped-file reason columns -Written to `skipped-file-reasons.csv` when -`--skipped-files-reason` is used without `REPO[@REV]` +Written to `skipped-files-reason-details.csv` when +`--skipped-files-reason` finds detail rows | Column | Type | Requires admin | Description | | --- | --- | --- | --- | diff --git a/repo-troubleshooting/list-repos/list-repos.py b/repo-troubleshooting/list-repos/list-repos.py index 50ea3ff..490fa69 100755 --- a/repo-troubleshooting/list-repos/list-repos.py +++ b/repo-troubleshooting/list-repos/list-repos.py @@ -84,8 +84,9 @@ def emit(self, record: logging.LogRecord) -> None: DEFAULT_LOG_FILE_STEM = "list-repos" DEFAULT_OUTPUT_FILE = "repos.csv" DEFAULT_RUNS_DIR = "list-repos-runs" -DEFAULT_SKIPPED_FILES_FILE = "repos-with-skipped-files.csv" -DEFAULT_SKIPPED_FILE_REASONS_FILE = "skipped-file-reasons.csv" +DEFAULT_SKIPPED_FILES_FILE = "skipped-files-repos.csv" +DEFAULT_SKIPPED_FILE_REASONS_FILE = "skipped-files-reason-details.csv" +DEFAULT_SKIPPED_FILE_REASON_STATS_FILE = "skipped-files-reason-stats.csv" DEFAULT_STATS_FILE_PREFIX = "stats" CSV_RECORD_LINE_TERMINATOR = "\r\n" DEFAULT_MAX_RETRIES = 5 @@ -1550,7 +1551,8 @@ def write_csv_schema(path: Path) -> None: | `{DEFAULT_CLONING_ERRORS_FILE}` | at least one repo has a cloning error | main columns + cloning-error extras | | `{DEFAULT_INDEXING_ERRORS_FILE}` | at least one repo is cloned but is missing a search index | main columns | | `{DEFAULT_SKIPPED_FILES_FILE}` | `--skipped-files` is set and the last index excluded files in at least one repo | main columns + skipped-files extras | -| `{DEFAULT_SKIPPED_FILE_REASONS_FILE}` | `--skipped-files-reason` is set without `REPO[@REV]`, and at least one skipped-file detail row is found | skipped-file reason columns, plus skipped-file metrics columns when `--skipped-file-metrics` is set | +| `{DEFAULT_SKIPPED_FILE_REASONS_FILE}` | `--skipped-files-reason` is set, and at least one skipped-file detail row is found | skipped-file reason columns, plus skipped-file metrics columns when `--skipped-file-metrics` is set | +| `{DEFAULT_SKIPPED_FILE_REASON_STATS_FILE}` | `--skipped-files-reason REPO[@REV]` is set, and at least one NOT-INDEXED reason category is found | `reason,count` | | `{DEFAULT_STATS_FILE_PREFIX}-*.csv` | `--stats` is set and repo rows were processed | `bucket,count` (see Stats section) | The optional `--count-commits` and `--run-search` flags append extra @@ -1579,7 +1581,7 @@ def write_csv_schema(path: Path) -> None: ## Skipped-file reason columns Written to `{DEFAULT_SKIPPED_FILE_REASONS_FILE}` when -`--skipped-files-reason` is used without `REPO[@REV]` +`--skipped-files-reason` finds detail rows {skipped_reason_list} @@ -2630,7 +2632,7 @@ def row_sort_key(row: list[Any]) -> tuple[Any, ...]: ) files_writer = LazyCSVWriter( - output_dir / "skipped-files.csv", + output_dir / DEFAULT_SKIPPED_FILE_REASONS_FILE, [name for name, _ in file_columns], ) with files_writer as writer: @@ -2638,7 +2640,7 @@ def row_sort_key(row: list[Any]) -> tuple[Any, ...]: writer.writerow(row) stats_writer = LazyCSVWriter( - output_dir / "skipped-stats.csv", + output_dir / DEFAULT_SKIPPED_FILE_REASON_STATS_FILE, [name for name, _ in stats_columns], ) with stats_writer as writer: @@ -2653,9 +2655,10 @@ def row_sort_key(row: list[Any]) -> tuple[Any, ...]: ) else: logger.info( - "No skipped-file matches found for %s@%s; skipped-files.csv not written", + "No skipped-file matches found for %s@%s; %s not written", name, display_rev, + DEFAULT_SKIPPED_FILE_REASONS_FILE, ) if stats_writer.count: logger.info( From 76b32a684916f5825aaa5b22cc1fcf53a0f66020 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Wed, 17 Jun 2026 17:20:21 -0600 Subject: [PATCH 13/26] Align targeted skipped-file detail columns Amp-Thread-ID: https://ampcode.com/threads/T-019ecf95-ee6d-7426-b1e7-d724bdf73e0f Co-authored-by: Amp --- repo-troubleshooting/list-repos/list-repos.py | 88 ++++++------------- 1 file changed, 29 insertions(+), 59 deletions(-) diff --git a/repo-troubleshooting/list-repos/list-repos.py b/repo-troubleshooting/list-repos/list-repos.py index 490fa69..e23b507 100755 --- a/repo-troubleshooting/list-repos/list-repos.py +++ b/repo-troubleshooting/list-repos/list-repos.py @@ -2207,8 +2207,8 @@ def verify_repo_rev( token: str, repo_rev: str, max_retries: int = DEFAULT_MAX_RETRIES, -) -> tuple[str, str, str]: - """Resolve repo/rev to an indexed ref; return display rev, indexed rev, and skip query""" +) -> tuple[str, str, str, int]: + """Resolve repo/rev to an indexed ref; return display rev, indexed rev, skip query, and skip count""" name = parse_repo_name(repo_rev) rev = parse_repo_rev(repo_rev) data = graphql_request( @@ -2238,9 +2238,11 @@ def verify_repo_rev( selected_ref_name = "" selected_indexed_oid = "" selected_skipped_query = "" + selected_skipped_count = 0 target_match_ref_name = "" target_match_indexed_oid = "" target_match_skipped_query = "" + target_match_skipped_count = 0 for ref in refs: if not ref.get("indexed"): continue @@ -2255,11 +2257,13 @@ def verify_repo_rev( if ref_name == requested_indexed_ref: selected_ref_name = ref_name selected_indexed_oid = oid + selected_skipped_count = skipped_count if skipped_count > 0: selected_skipped_query = skipped_query if oid == target_oid: target_match_ref_name = ref_name target_match_indexed_oid = oid + target_match_skipped_count = skipped_count if skipped_count > 0: target_match_skipped_query = skipped_query @@ -2267,6 +2271,7 @@ def verify_repo_rev( selected_ref_name = target_match_ref_name selected_indexed_oid = target_match_indexed_oid selected_skipped_query = target_match_skipped_query + selected_skipped_count = target_match_skipped_count if not selected_ref_name: if indexed_names: @@ -2293,7 +2298,7 @@ def verify_repo_rev( selected_ref_name, selected_indexed_oid, ) - return display_rev, indexed_rev, selected_skipped_query + return display_rev, indexed_rev, selected_skipped_query, selected_skipped_count def file_url(endpoint: str, repo_name: str, rev: str, file_path: str) -> str: @@ -2540,53 +2545,13 @@ def write_skipped_files_reason( skipped_file_metrics: bool = False, ) -> None: """Fetch skipped-file matches for repo_rev and write the per-file and stats CSVs""" - display_rev, indexed_rev, skipped_indexed_query = verify_repo_rev( + display_rev, indexed_rev, skipped_indexed_query, skipped_count = verify_repo_rev( endpoint, token, repo_rev, max_retries=max_retries, ) name = parse_repo_name(repo_rev) - - # Keep each local CSV header beside the extractor that writes its value - def chunk_matches_content(m: dict[str, Any]) -> str: - chunks: list[dict[str, Any]] = m.get("chunkMatches") or [] - return "\n".join(str(c.get("content") or "") for c in chunks) - - def match_file_byte_size(m: dict[str, Any]) -> int | str: - file_obj: dict[str, Any] = m.get("file") or {} - bs = file_obj.get("byteSize") - return int(bs) if bs is not None else "" - - def match_file_extension(m: dict[str, Any]) -> str: - file_obj: dict[str, Any] = m.get("file") or {} - return Path(str(file_obj.get("path") or "")).suffix.lstrip(".") - - def match_file_url(m: dict[str, Any]) -> str: - repo_obj: dict[str, Any] = m.get("repository") or {} - file_obj: dict[str, Any] = m.get("file") or {} - return file_url( - endpoint, - str(repo_obj.get("name") or ""), - indexed_rev, - str(file_obj.get("path") or ""), - ) - - distinct_trigram_counts_by_path: dict[str, int] = {} - - def match_distinct_trigram_count(m: dict[str, Any]) -> int | str: - file_obj: dict[str, Any] = m.get("file") or {} - file_path = str(file_obj.get("path") or "") - return distinct_trigram_counts_by_path.get(file_path, "") - - file_columns: list[tuple[str, Callable[[dict[str, Any]], Any]]] = [ - ("chunkMatches.content", chunk_matches_content), - ("file.extension", match_file_extension), - ("file.byteSize", match_file_byte_size), - ] - if skipped_file_metrics: - file_columns.append(("file.distinctTrigramCount", match_distinct_trigram_count)) - file_columns.append(("file_url", match_file_url)) stats_columns: list[tuple[str, Callable[[tuple[str, int]], Any]]] = [ ("reason", lambda r: r[0]), ("count", lambda r: r[1]), @@ -2601,6 +2566,7 @@ def match_distinct_trigram_count(m: dict[str, Any]) -> int | str: max_retries=max_retries, ) matches = query_result.matches + distinct_trigram_counts_by_path: dict[str, int] = {} if skipped_file_metrics: distinct_trigram_counts_by_path = collect_distinct_trigram_counts( endpoint, @@ -2612,32 +2578,36 @@ def match_distinct_trigram_count(m: dict[str, Any]) -> int | str: ) reason_counts: collections.Counter[str] = collections.Counter() - rows: list[list[Any]] = [] for match in matches: - rows.append([extract(match) for _, extract in file_columns]) reason = skipped_file_reason(match) if reason: reason_counts[reason] += 1 - # Sort by chunkMatches.content so files with the same NOT-INDEXED reason are - # grouped together; ties broken by extension, byteSize, metric, then file_url. - # Coerce numeric cells to int so an int/str union can't blow up the comparator. - def row_sort_key(row: list[Any]) -> tuple[Any, ...]: - byte_size = row[2] if isinstance(row[2], int) else -1 - metric = row[3] if skipped_file_metrics and isinstance(row[3], int) else -1 - return (row[0], row[1], byte_size, metric, row[-1]) - - rows.sort( - key=row_sort_key, + search_result = SkippedFileReasonSearchResult( + repository_name=name, + ref_name=display_rev, + indexed_rev=indexed_rev, + skipped_count=skipped_count, + matches=matches, + match_count=query_result.match_count, + limit_hit=query_result.limit_hit, + alert_title=query_result.alert_title, + alert_description=query_result.alert_description, + distinct_trigram_counts_by_path=distinct_trigram_counts_by_path, + error=None, ) files_writer = LazyCSVWriter( output_dir / DEFAULT_SKIPPED_FILE_REASONS_FILE, - [name for name, _ in file_columns], + skipped_file_reason_column_names(skipped_file_metrics), ) with files_writer as writer: - for row in rows: - writer.writerow(row) + write_skipped_file_reason_rows( + writer, + endpoint, + [search_result], + skipped_file_metrics, + ) stats_writer = LazyCSVWriter( output_dir / DEFAULT_SKIPPED_FILE_REASON_STATS_FILE, From 9c6035865a24aca977b1d397693a565ccc45e39b Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Wed, 17 Jun 2026 17:23:01 -0600 Subject: [PATCH 14/26] Rename skipped-files repo output Amp-Thread-ID: https://ampcode.com/threads/T-019ecf95-ee6d-7426-b1e7-d724bdf73e0f Co-authored-by: Amp --- repo-troubleshooting/list-repos/CSV_SCHEMA.md | 4 ++-- repo-troubleshooting/list-repos/list-repos.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/repo-troubleshooting/list-repos/CSV_SCHEMA.md b/repo-troubleshooting/list-repos/CSV_SCHEMA.md index ca80f11..b4d2ba5 100644 --- a/repo-troubleshooting/list-repos/CSV_SCHEMA.md +++ b/repo-troubleshooting/list-repos/CSV_SCHEMA.md @@ -21,7 +21,7 @@ rows for it | `repos.csv` | at least one repo row is written | main columns | | `repos-with-cloning-errors.csv` | at least one repo has a cloning error | main columns + cloning-error extras | | `repos-with-indexing-errors.csv` | at least one repo is cloned but is missing a search index | main columns | -| `skipped-files-repos.csv` | `--skipped-files` is set and the last index excluded files in at least one repo | main columns + skipped-files extras | +| `repos-with-skipped-files.csv` | `--skipped-files` is set and the last index excluded files in at least one repo | main columns + skipped-files extras | | `skipped-files-reason-details.csv` | `--skipped-files-reason` is set, and at least one skipped-file detail row is found | skipped-file reason columns, plus skipped-file metrics columns when `--skipped-file-metrics` is set | | `skipped-files-reason-stats.csv` | `--skipped-files-reason REPO[@REV]` is set, and at least one NOT-INDEXED reason category is found | `reason,count` | | `stats-*.csv` | `--stats` is set and repo rows were processed | `bucket,count` (see Stats section) | @@ -80,7 +80,7 @@ Appended to `repos-with-cloning-errors.csv` ## Skipped-files extras -Appended to `skipped-files-repos.csv` +Appended to `repos-with-skipped-files.csv` | Column | Type | Requires admin | Description | | --- | --- | --- | --- | diff --git a/repo-troubleshooting/list-repos/list-repos.py b/repo-troubleshooting/list-repos/list-repos.py index e23b507..d601f6b 100755 --- a/repo-troubleshooting/list-repos/list-repos.py +++ b/repo-troubleshooting/list-repos/list-repos.py @@ -84,7 +84,7 @@ def emit(self, record: logging.LogRecord) -> None: DEFAULT_LOG_FILE_STEM = "list-repos" DEFAULT_OUTPUT_FILE = "repos.csv" DEFAULT_RUNS_DIR = "list-repos-runs" -DEFAULT_SKIPPED_FILES_FILE = "skipped-files-repos.csv" +DEFAULT_SKIPPED_FILES_FILE = "repos-with-skipped-files.csv" DEFAULT_SKIPPED_FILE_REASONS_FILE = "skipped-files-reason-details.csv" DEFAULT_SKIPPED_FILE_REASON_STATS_FILE = "skipped-files-reason-stats.csv" DEFAULT_STATS_FILE_PREFIX = "stats" From f4ace550f56d2f71a565b283c06fb8e320d96945 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Wed, 17 Jun 2026 17:34:17 -0600 Subject: [PATCH 15/26] Sort output CSV files after writing Amp-Thread-ID: https://ampcode.com/threads/T-019ecf95-ee6d-7426-b1e7-d724bdf73e0f Co-authored-by: Amp --- repo-troubleshooting/list-repos/CSV_SCHEMA.md | 12 ++ repo-troubleshooting/list-repos/list-repos.py | 197 +++++++++++++++++- 2 files changed, 202 insertions(+), 7 deletions(-) diff --git a/repo-troubleshooting/list-repos/CSV_SCHEMA.md b/repo-troubleshooting/list-repos/CSV_SCHEMA.md index b4d2ba5..cc177b2 100644 --- a/repo-troubleshooting/list-repos/CSV_SCHEMA.md +++ b/repo-troubleshooting/list-repos/CSV_SCHEMA.md @@ -26,6 +26,18 @@ rows for it | `skipped-files-reason-stats.csv` | `--skipped-files-reason REPO[@REV]` is set, and at least one NOT-INDEXED reason category is found | `reason,count` | | `stats-*.csv` | `--stats` is set and repo rows were processed | `bucket,count` (see Stats section) | +The row-bearing output CSVs below are sorted before the run exits, using a +stdlib Python external sort that writes bounded temporary chunks instead of +holding every output row in memory + +| File | Sort columns | +| --- | --- | +| `repos.csv` | `url` | +| `repos-with-cloning-errors.csv` | `url` | +| `repos-with-indexing-errors.csv` | `url` | +| `repos-with-skipped-files.csv` | `url` | +| `skipped-files-reason-details.csv` | `repository.name`, `rev`, `reason`, `file.extension`, `file.path` | + The optional `--count-commits` and `--run-search` flags append extra columns to the repo-listing CSVs above, excluding the `--stats` files and the skipped-file reason detail CSV, in this order: main diff --git a/repo-troubleshooting/list-repos/list-repos.py b/repo-troubleshooting/list-repos/list-repos.py index d601f6b..99b41c2 100755 --- a/repo-troubleshooting/list-repos/list-repos.py +++ b/repo-troubleshooting/list-repos/list-repos.py @@ -8,6 +8,7 @@ import concurrent.futures import contextlib import csv +import heapq import http.client import json import logging @@ -15,6 +16,7 @@ import re import shlex import sys +import tempfile import textwrap import threading import time @@ -88,6 +90,7 @@ def emit(self, record: logging.LogRecord) -> None: DEFAULT_SKIPPED_FILE_REASONS_FILE = "skipped-files-reason-details.csv" DEFAULT_SKIPPED_FILE_REASON_STATS_FILE = "skipped-files-reason-stats.csv" DEFAULT_STATS_FILE_PREFIX = "stats" +CSV_SORT_CHUNK_ROWS = 50_000 CSV_RECORD_LINE_TERMINATOR = "\r\n" DEFAULT_MAX_RETRIES = 5 GRAPHQL_FIELD_COUNT_RETRY_HEADROOM_PERCENT = 95 @@ -116,6 +119,16 @@ def emit(self, record: logging.LogRecord) -> None: "transport:", ) TOO_MANY_TRIGRAMS_REASON = "contains too many trigrams" +SORTED_CSV_OUTPUTS: tuple[tuple[str, tuple[str, ...]], ...] = ( + (DEFAULT_OUTPUT_FILE, ("url",)), + (DEFAULT_CLONING_ERRORS_FILE, ("url",)), + (DEFAULT_INDEXING_ERRORS_FILE, ("url",)), + (DEFAULT_SKIPPED_FILES_FILE, ("url",)), + ( + DEFAULT_SKIPPED_FILE_REASONS_FILE, + ("repository.name", "rev", "reason", "file.extension", "file.path"), + ), +) def normalize_csv_value(value: Any) -> Any: @@ -1487,6 +1500,22 @@ def table_row(*cells: str) -> str: return "|" + "|".join(f" {c} " if c else " " for c in cells) + "|" +def format_output_sort_list() -> str: + """Render the configured output CSV sort keys as a Markdown table""" + rows = [ + table_row("File", "Sort columns"), + table_row("---", "---"), + ] + for file_name, sort_columns in SORTED_CSV_OUTPUTS: + rows.append( + table_row( + f"`{file_name}`", + ", ".join(f"`{column}`" for column in sort_columns), + ), + ) + return "\n".join(rows) + + def name_desc( columns: list[tuple[str, Callable[[dict[str, Any]], Any], str, bool, str]], ) -> list[tuple[str, str, bool, str]]: @@ -1526,6 +1555,7 @@ def write_csv_schema(path: Path) -> None: commit_count_list = format_columns_list(COMMIT_COUNT_COLUMNS) run_search_list = format_columns_list(RUN_SEARCH_COLUMNS) stats_files_list = format_stats_files_list() + output_sort_list = format_output_sort_list() content = f"""# `list-repos.py` CSV column reference @@ -1555,6 +1585,12 @@ def write_csv_schema(path: Path) -> None: | `{DEFAULT_SKIPPED_FILE_REASON_STATS_FILE}` | `--skipped-files-reason REPO[@REV]` is set, and at least one NOT-INDEXED reason category is found | `reason,count` | | `{DEFAULT_STATS_FILE_PREFIX}-*.csv` | `--stats` is set and repo rows were processed | `bucket,count` (see Stats section) | +The row-bearing output CSVs below are sorted before the run exits, using a +stdlib Python external sort that writes bounded temporary chunks instead of +holding every output row in memory + +{output_sort_list} + The optional `--count-commits` and `--run-search` flags append extra columns to the repo-listing CSVs above, excluding the `--stats` files and the skipped-file reason detail CSV, in this order: main @@ -2668,6 +2704,150 @@ def __exit__(self, *_args: object) -> None: self._file.close() +def csv_sort_key(row: list[str], column_indexes: list[int]) -> tuple[str, ...]: + """Return the configured sort key for one CSV row""" + return tuple( + row[column_index] if column_index < len(row) else "" + for column_index in column_indexes + ) + + +def make_temporary_csv_path(directory: Path, prefix: str) -> Path: + """Reserve and close a temporary CSV path for normal text writing""" + file_descriptor, temporary_name = tempfile.mkstemp( + dir=directory, + prefix=prefix, + suffix=".csv", + ) + os.close(file_descriptor) + return Path(temporary_name) + + +def write_sorted_csv_chunk( + rows: list[list[str]], + column_indexes: list[int], + directory: Path, + source_name: str, +) -> Path: + """Sort a bounded CSV row chunk and write it to a temporary file""" + rows.sort(key=lambda row: csv_sort_key(row, column_indexes)) + temporary_path = make_temporary_csv_path( + directory, + f".{source_name}.sort-chunk-", + ) + with temporary_path.open("w", newline="") as output_file: + writer = make_csv_writer(output_file) + for row in rows: + write_csv_row(writer, row) + return temporary_path + + +def replace_with_merged_csv_chunks( + path: Path, + header: list[str], + chunk_paths: list[Path], + column_indexes: list[int], +) -> None: + """Merge sorted temporary CSV chunks and atomically replace path""" + replacement_path = make_temporary_csv_path( + path.parent, + f".{path.name}.sorted-", + ) + try: + with replacement_path.open("w", newline="") as output_file: + writer = make_csv_writer(output_file) + write_csv_row(writer, header) + with contextlib.ExitStack() as stack: + readers: list[Any] = [] + for chunk_path in chunk_paths: + chunk_file = stack.enter_context(chunk_path.open(newline="")) + readers.append(csv.reader(chunk_file)) + for row in heapq.merge( + *readers, + key=lambda row: csv_sort_key(row, column_indexes), + ): + write_csv_row(writer, row) + replacement_path.replace(path) + except Exception: + replacement_path.unlink(missing_ok=True) + raise + + +def sort_csv_output_file( + path: Path, + sort_columns: tuple[str, ...], + chunk_rows: int = CSV_SORT_CHUNK_ROWS, +) -> None: + """Sort a CSV file by named columns using bounded memory""" + if not path.is_file(): + return + temporary_chunk_paths: list[Path] = [] + try: + with path.open(newline="") as input_file: + reader = csv.reader(input_file) + header = next(reader, None) + if header is None: + return + missing_columns = [ + column for column in sort_columns if column not in header + ] + if missing_columns: + logger.error( + "Cannot sort %s: missing column(s): %s", + path.name, + ", ".join(missing_columns), + ) + return + column_indexes = [header.index(column) for column in sort_columns] + current_rows: list[list[str]] = [] + row_count = 0 + for row in reader: + current_rows.append(row) + row_count += 1 + if len(current_rows) >= chunk_rows: + temporary_chunk_paths.append( + write_sorted_csv_chunk( + current_rows, + column_indexes, + path.parent, + path.name, + ), + ) + current_rows = [] + if row_count <= 1: + return + if current_rows: + temporary_chunk_paths.append( + write_sorted_csv_chunk( + current_rows, + column_indexes, + path.parent, + path.name, + ), + ) + replace_with_merged_csv_chunks( + path, + header, + temporary_chunk_paths, + column_indexes, + ) + logger.info( + "Sorted %d row(s) in %s by %s", + row_count, + path.name, + ", ".join(sort_columns), + ) + finally: + for chunk_path in temporary_chunk_paths: + chunk_path.unlink(missing_ok=True) + + +def sort_csv_outputs(output_dir: Path) -> None: + """Sort configured CSV outputs that were created during this run""" + for file_name, sort_columns in SORTED_CSV_OUTPUTS: + sort_csv_output_file(output_dir / file_name, sort_columns) + + @dataclass(frozen=True) class RepositoryPage: """One repository listing page plus the page size Sourcegraph accepted""" @@ -3026,6 +3206,7 @@ def write_skipped_file_reason_rows( for search_result in search_results: if search_result.error is not None: continue + matches = search_result.matches alert_parts = [ part for part in (search_result.alert_title, search_result.alert_description) @@ -3038,7 +3219,7 @@ def write_skipped_file_reason_rows( search_result.repository_name, search_result.ref_name, search_result.match_count, - len(search_result.matches), + len(matches), search_result.skipped_count, ) if alert_parts: @@ -3052,22 +3233,19 @@ def write_skipped_file_reason_rows( search_result.match_count is not None and search_result.match_count != search_result.skipped_count ) - if ( - len(search_result.matches) != search_result.skipped_count - or match_count_mismatch - ): + if len(matches) != search_result.skipped_count or match_count_mismatch: logger.warning( "Skipped-file reason search returned %d file match(es) " "(matchCount=%s, limitHit=%s) for %s@%s; " "textSearchIndex.refs.skippedIndexed.count reported %d", - len(search_result.matches), + len(matches), search_result.match_count, search_result.limit_hit, search_result.repository_name, search_result.ref_name, search_result.skipped_count, ) - for match in search_result.matches: + for match in matches: file_obj: dict[str, Any] = match.get("file") or {} file_path = str(file_obj.get("path") or "") byte_size = file_obj.get("byteSize") @@ -3098,6 +3276,8 @@ def write_skipped_file_reason_rows( writer.writerow( row, ) + matches.clear() + search_result.distinct_trigram_counts_by_path.clear() @dataclass(frozen=True) @@ -3923,6 +4103,7 @@ def run(args: argparse.Namespace, endpoint: str, token: str, output_dir: Path) - max_retries=args.max_retries, skipped_file_metrics=args.skipped_file_metrics, ) + sort_csv_outputs(output_dir) return include_index_failure_fields = supports_text_search_index_failure_fields( @@ -4038,6 +4219,8 @@ def run(args: argparse.Namespace, endpoint: str, token: str, output_dir: Path) - elif stats is not None: logger.info("No repo rows processed; stats files not written") + sort_csv_outputs(output_dir) + if output_writer.count: logger.info("Wrote %d repos to %s", output_writer.count, output_path.name) else: From f278266b5ec2d6250efa1e633d47cb4e48e994b8 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Wed, 17 Jun 2026 17:35:58 -0600 Subject: [PATCH 16/26] Update docs --- repo-troubleshooting/list-repos/.gitignore | 1 + repo-troubleshooting/list-repos/AGENTS.md | 1 + 2 files changed, 2 insertions(+) diff --git a/repo-troubleshooting/list-repos/.gitignore b/repo-troubleshooting/list-repos/.gitignore index aa42386..2825c19 100644 --- a/repo-troubleshooting/list-repos/.gitignore +++ b/repo-troubleshooting/list-repos/.gitignore @@ -1,3 +1,4 @@ **.csv **.log keepers/ +list-repos-runs/ diff --git a/repo-troubleshooting/list-repos/AGENTS.md b/repo-troubleshooting/list-repos/AGENTS.md index e45e7bd..7723dd2 100644 --- a/repo-troubleshooting/list-repos/AGENTS.md +++ b/repo-troubleshooting/list-repos/AGENTS.md @@ -19,6 +19,7 @@ ## Other +- This script needs to run the same on macOS, Linux, and Windows - Whenever changing / adding / removing / moving any columns in any of the CSV files, ensure the columns are updated in the `CSV_SCHEMA.md` printer function to match. `CSV_SCHEMA.md` is generated from the in-script column tuples by running From e01fce833a22af5b0402be4505aadb0b7e2a2aa9 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Wed, 17 Jun 2026 18:47:15 -0600 Subject: [PATCH 17/26] Truncate long request errors in logs Amp-Thread-ID: https://ampcode.com/threads/T-019ecf95-ee6d-7426-b1e7-d724bdf73e0f Co-authored-by: Amp --- repo-troubleshooting/list-repos/list-repos.py | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/repo-troubleshooting/list-repos/list-repos.py b/repo-troubleshooting/list-repos/list-repos.py index 99b41c2..d79b9d9 100755 --- a/repo-troubleshooting/list-repos/list-repos.py +++ b/repo-troubleshooting/list-repos/list-repos.py @@ -92,6 +92,8 @@ def emit(self, record: logging.LogRecord) -> None: DEFAULT_STATS_FILE_PREFIX = "stats" CSV_SORT_CHUNK_ROWS = 50_000 CSV_RECORD_LINE_TERMINATOR = "\r\n" +LOG_ERROR_TEXT_MAX_CHARS = 2_000 +LOG_GRAPHQL_ERROR_MAX_MESSAGES = 5 DEFAULT_MAX_RETRIES = 5 GRAPHQL_FIELD_COUNT_RETRY_HEADROOM_PERCENT = 95 PAGE_SIZE = 500 @@ -587,6 +589,14 @@ def truncate_lines(value: str, head: int = 5, tail: int = 5) -> str: ) +def truncate_log_text(value: str, max_chars: int = LOG_ERROR_TEXT_MAX_CHARS) -> str: + """Return value capped for one readable log record""" + if len(value) <= max_chars: + return value + omitted = len(value) - max_chars + return f"{value[:max_chars]}... [{omitted} chars truncated]" + + def has_cloning_error(repo: dict[str, Any]) -> bool: """Return True for errored, corrupted, or not-yet-cloned repos""" return derive_mirror_status(repo) in {"errored", "corrupted", "not_cloned"} @@ -1871,9 +1881,13 @@ def has_retryable_graphql_error(errors: object) -> bool: def summarize_graphql_errors(errors: object) -> str: """Return compact GraphQL error messages for retry logs""" if not isinstance(errors, list): - return str(errors) - messages = [graphql_error_message(error) for error in errors] - return "; ".join(messages) + return truncate_log_text(str(errors)) + displayed_errors = errors[:LOG_GRAPHQL_ERROR_MAX_MESSAGES] + messages = [graphql_error_message(error) for error in displayed_errors] + omitted = len(errors) - len(displayed_errors) + if omitted > 0: + messages.append(f"... [{omitted} GraphQL errors omitted]") + return truncate_log_text("; ".join(messages)) def request_error_summary(error: Exception) -> str: @@ -1881,9 +1895,9 @@ def request_error_summary(error: Exception) -> str: if isinstance(error, HTTPRequestError): body = error.body.decode(errors="replace").strip() if body: - return f"HTTP {error.status} {error.reason}: {body}" + return truncate_log_text(f"HTTP {error.status} {error.reason}: {body}") return f"HTTP {error.status} {error.reason}" - return str(error) + return truncate_log_text(str(error)) def request_failure_can_retry(error: Exception) -> bool: @@ -2031,7 +2045,7 @@ def graphql_request( max_retries, ) continue - msg = f"GraphQL errors: {json.dumps(errors, indent=2)}" + msg = f"GraphQL errors: {summarize_graphql_errors(errors)}" raise GraphQLError(msg) msg = "graphql_request retry loop exhausted unexpectedly" raise RuntimeError(msg) @@ -3696,7 +3710,7 @@ def log_http_error(exc: HTTPRequestError) -> None: logger.error(" %s: %s", header, value) body = exc.body.decode(errors="replace") if body: - logger.error("Response body:\n%s", body) + logger.error("Response body:\n%s", truncate_log_text(body)) logger.error("HTTP request failed", exc_info=exc) From aa0114ec07d24c881a13c01dc8b9b2db65b96a61 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Wed, 17 Jun 2026 18:50:57 -0600 Subject: [PATCH 18/26] Put retry metadata before error messages Amp-Thread-ID: https://ampcode.com/threads/T-019ecf95-ee6d-7426-b1e7-d724bdf73e0f Co-authored-by: Amp --- repo-troubleshooting/list-repos/list-repos.py | 59 ++++++++++++++----- 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/repo-troubleshooting/list-repos/list-repos.py b/repo-troubleshooting/list-repos/list-repos.py index d79b9d9..cc256ce 100755 --- a/repo-troubleshooting/list-repos/list-repos.py +++ b/repo-troubleshooting/list-repos/list-repos.py @@ -1839,17 +1839,39 @@ def retry_delay_seconds(retry_number: int) -> int: return 2 ** (retry_number - 1) -def sleep_before_retry(reason: str, retry_number: int, max_retries: int) -> None: +def retry_message_label(message: str) -> str: + """Return log suffix for a retry failure message""" + if message.startswith("GraphQL errors:"): + return message + return f"message: {message}" + + +def sleep_before_retry( + reason: str, + retry_number: int, + max_retries: int, + message: str | None = None, +) -> None: """Log and sleep before the next retry attempt for this request""" delay = retry_delay_seconds(retry_number) RUN_ISSUE_COUNTS.increment_retry() - logger.warning( - "%s; retrying (%d/%d) in %ds...", - reason, - retry_number, - max_retries, - delay, - ) + if message is None: + logger.warning( + "%s; retrying (%d/%d) in %ds", + reason, + retry_number, + max_retries, + delay, + ) + else: + logger.warning( + "%s; retrying (%d/%d) in %ds; %s", + reason, + retry_number, + max_retries, + delay, + retry_message_label(message), + ) time.sleep(delay) @@ -1973,9 +1995,10 @@ def graphql_request_with_validation( ) raise sleep_before_retry( - f"{request_description} failed: {error_summary}", + f"{request_description} failed", retry_number, max_retries, + message=error_summary, ) msg = f"{request_description} retry loop exhausted unexpectedly" raise RuntimeError(msg) @@ -1998,7 +2021,7 @@ def graphql_request( "Content-Type": "application/json", "User-Agent": "list-repos/0.0.1", } - retry_prefix = f"{request_description}: " if request_description else "" + request_label = request_description or "GraphQL request" for retry_count in range(max_retries + 1): retry_number = retry_count + 1 try: @@ -2007,18 +2030,20 @@ def graphql_request( if not retryable_http_error(error) or retry_count >= max_retries: raise sleep_before_retry( - f"{retry_prefix}HTTP {error.status} {error.reason}", + f"{request_label} failed", retry_number, max_retries, + message=request_error_summary(error), ) continue except (OSError, http.client.HTTPException, json.JSONDecodeError) as error: if retry_count >= max_retries: raise sleep_before_retry( - f"{retry_prefix}Request failed: {error}", + f"{request_label} failed", retry_number, max_retries, + message=truncate_log_text(str(error)), ) continue @@ -2029,9 +2054,10 @@ def graphql_request( return data if retry_count < max_retries: sleep_before_retry( - f"{retry_prefix}GraphQL response data missing or not an object", + f"{request_label} failed", retry_number, max_retries, + message="GraphQL response data missing or not an object", ) continue raise GraphQLError("GraphQL response data missing or not an object") @@ -2039,10 +2065,13 @@ def graphql_request( if retry_count < max_retries: retry_label = "retryable " if has_retryable_graphql_error(errors) else "" sleep_before_retry( - f"{retry_prefix}GraphQL returned {retry_label}error(s): " - f"{summarize_graphql_errors(errors)}", + f"{request_label} failed", retry_number, max_retries, + message=( + f"GraphQL returned {retry_label}error(s): " + f"{summarize_graphql_errors(errors)}" + ), ) continue msg = f"GraphQL errors: {summarize_graphql_errors(errors)}" From 8bb0d986c709cbaed5aa69a84d2b07770a19153c Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 18 Jun 2026 16:03:51 -0600 Subject: [PATCH 19/26] Increase retry resilience and flush CSV rows Amp-Thread-ID: https://ampcode.com/threads/T-019ecf95-ee6d-7426-b1e7-d724bdf73e0f Co-authored-by: Amp --- repo-troubleshooting/list-repos/list-repos.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/repo-troubleshooting/list-repos/list-repos.py b/repo-troubleshooting/list-repos/list-repos.py index cc256ce..f0db88e 100755 --- a/repo-troubleshooting/list-repos/list-repos.py +++ b/repo-troubleshooting/list-repos/list-repos.py @@ -94,7 +94,8 @@ def emit(self, record: logging.LogRecord) -> None: CSV_RECORD_LINE_TERMINATOR = "\r\n" LOG_ERROR_TEXT_MAX_CHARS = 2_000 LOG_GRAPHQL_ERROR_MAX_MESSAGES = 5 -DEFAULT_MAX_RETRIES = 5 +DEFAULT_MAX_RETRIES = 10 +MAX_RETRY_DELAY_SECONDS = 32 GRAPHQL_FIELD_COUNT_RETRY_HEADROOM_PERCENT = 95 PAGE_SIZE = 500 REQUEST_TIMEOUT_SECONDS = 60 @@ -1835,8 +1836,8 @@ def send_once( def retry_delay_seconds(retry_number: int) -> int: - """Return exponential retry delay: 1, 2, 4, 8, 16... seconds""" - return 2 ** (retry_number - 1) + """Return capped exponential retry delay in seconds""" + return min(2 ** (retry_number - 1), MAX_RETRY_DELAY_SECONDS) def retry_message_label(message: str) -> str: @@ -2737,6 +2738,8 @@ def writerow(self, row: list[Any]) -> None: self._writer = make_csv_writer(self._file) write_csv_row(self._writer, self.columns) write_csv_row(self._writer, row) + if self._file is not None: + self._file.flush() self.count += 1 def __enter__(self) -> LazyCSVWriter: @@ -3984,7 +3987,8 @@ def parse_args(argv: list[str]) -> argparse.Namespace: metavar="int", help=( "Retries per GraphQL request after the initial attempt " - f"(default {DEFAULT_MAX_RETRIES}; backoff 1s, 2s, 4s, ...)" + f"(default {DEFAULT_MAX_RETRIES}; backoff doubles from 1s " + f"to max {MAX_RETRY_DELAY_SECONDS}s)" ), ) parser.add_argument( @@ -4052,8 +4056,10 @@ def collect_scope(args: argparse.Namespace) -> tuple[str, str] | None: def run(args: argparse.Namespace, endpoint: str, token: str, output_dir: Path) -> None: """Confirm the connection, then stream every repo to the CSV file""" logger.info( - "Retry policy: %d retries per GraphQL request (backoff: 1s, 2s, 4s, ...)", + "Retry policy: %d retries per GraphQL request (backoff doubles from " + "1s to max %ds)", args.max_retries, + MAX_RETRY_DELAY_SECONDS, ) if args.skipped_file_metrics and args.skipped_files_reason is None: die("--skipped-file-metrics requires --skipped-files-reason") From 4f487f2abbf4972d0388f7563387ed8fae632257 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 18 Jun 2026 16:26:44 -0600 Subject: [PATCH 20/26] Compact skipped file reason CSV Amp-Thread-ID: https://ampcode.com/threads/T-019edcb2-c015-76fc-ab71-7ff166cb44e0 Co-authored-by: Amp --- repo-troubleshooting/list-repos/CSV_SCHEMA.md | 22 +-- repo-troubleshooting/list-repos/list-repos.py | 127 ++++++++---------- 2 files changed, 61 insertions(+), 88 deletions(-) diff --git a/repo-troubleshooting/list-repos/CSV_SCHEMA.md b/repo-troubleshooting/list-repos/CSV_SCHEMA.md index cc177b2..9691688 100644 --- a/repo-troubleshooting/list-repos/CSV_SCHEMA.md +++ b/repo-troubleshooting/list-repos/CSV_SCHEMA.md @@ -22,7 +22,7 @@ rows for it | `repos-with-cloning-errors.csv` | at least one repo has a cloning error | main columns + cloning-error extras | | `repos-with-indexing-errors.csv` | at least one repo is cloned but is missing a search index | main columns | | `repos-with-skipped-files.csv` | `--skipped-files` is set and the last index excluded files in at least one repo | main columns + skipped-files extras | -| `skipped-files-reason-details.csv` | `--skipped-files-reason` is set, and at least one skipped-file detail row is found | skipped-file reason columns, plus skipped-file metrics columns when `--skipped-file-metrics` is set | +| `skipped-files-reason-details.csv` | `--skipped-files-reason` is set, and at least one skipped-file detail row is found | skipped-file reason columns | | `skipped-files-reason-stats.csv` | `--skipped-files-reason REPO[@REV]` is set, and at least one NOT-INDEXED reason category is found | `reason,count` | | `stats-*.csv` | `--stats` is set and repo rows were processed | `bucket,count` (see Stats section) | @@ -36,7 +36,7 @@ holding every output row in memory | `repos-with-cloning-errors.csv` | `url` | | `repos-with-indexing-errors.csv` | `url` | | `repos-with-skipped-files.csv` | `url` | -| `skipped-files-reason-details.csv` | `repository.name`, `rev`, `reason`, `file.extension`, `file.path` | +| `skipped-files-reason-details.csv` | `reason`, `file.extension`, `rev`, `file_url` | The optional `--count-commits` and `--run-search` flags append extra columns to the repo-listing CSVs above, excluding the `--stats` @@ -107,25 +107,13 @@ Written to `skipped-files-reason-details.csv` when | Column | Type | Requires admin | Description | | --- | --- | --- | --- | -| `repository.name` | string | | Sourcegraph repository name containing the skipped file | -| `rev` | string | | Indexed revision parsed from Sourcegraph's skippedIndexed.query | -| `reason` | string | | NOT-INDEXED reason parsed from the indexed placeholder content | +| `reason` | string | | Compact NOT-INDEXED reason parsed from the indexed placeholder content | | `file.extension` | string | | File extension derived from file.path | | `file.byteSize` | integer | | Sourcegraph-reported file byte size | -| `repoRevSkippedIndexed.count` | integer | | Count Sourcegraph reported for this repo/ref before running the details search | -| `file.path` | string | | Path of the skipped file within the repository | +| `file.distinctTrigramCount` | integer | | Distinct three-rune trigrams computed from GitBlob.content. Only populated with --skipped-file-metrics for skipped files whose reason is `too_many_trigrams` | +| `rev` | string | | Indexed ref containing the skipped file | | `file_url` | string | | Sourcegraph blob URL for the skipped file at the indexed ref | -## Skipped-file metrics columns - -Appended to skipped-file detail CSVs only when `--skipped-file-metrics` -is used. These columns may require extra GitBlob content requests, and are -therefore not fetched by default - -| Column | Type | Requires admin | Description | -| --- | --- | --- | --- | -| `file.distinctTrigramCount` | integer | | Distinct three-rune trigrams computed from GitBlob.content. Only populated for skipped files whose reason is `contains too many trigrams` | - ## `--count-commits` columns Appended to CSV files when `--count-commits` is used diff --git a/repo-troubleshooting/list-repos/list-repos.py b/repo-troubleshooting/list-repos/list-repos.py index f0db88e..d6a7a6c 100755 --- a/repo-troubleshooting/list-repos/list-repos.py +++ b/repo-troubleshooting/list-repos/list-repos.py @@ -122,6 +122,14 @@ def emit(self, record: logging.LogRecord) -> None: "transport:", ) TOO_MANY_TRIGRAMS_REASON = "contains too many trigrams" +SKIPPED_FILE_REASON_CODES_BY_EXPLANATION = { + "contains binary content": "binary", + "contains too many trigrams": "too_many_trigrams", + "contains too few trigrams": "too_few_trigrams", + "exceeds the maximum size limit": "too_large", + "object missing from repository": "object_missing", + "unknown skip reason": "unknown", +} SORTED_CSV_OUTPUTS: tuple[tuple[str, tuple[str, ...]], ...] = ( (DEFAULT_OUTPUT_FILE, ("url",)), (DEFAULT_CLONING_ERRORS_FILE, ("url",)), @@ -129,7 +137,7 @@ def emit(self, record: logging.LogRecord) -> None: (DEFAULT_SKIPPED_FILES_FILE, ("url",)), ( DEFAULT_SKIPPED_FILE_REASONS_FILE, - ("repository.name", "rev", "reason", "file.extension", "file.path"), + ("reason", "file.extension", "rev", "file_url"), ), ) @@ -1195,8 +1203,7 @@ def validate(data: dict[str, Any]) -> None: # Extra columns appended only to the skipped-files CSV. The query is the # Sourcegraph search query produced by the API; running it lists each skipped -# file along with its NOT-INDEXED reason (too-large / binary / too-many-trigrams -# / too-small / blob-missing) +# file along with its NOT-INDEXED reason. SKIPPED_FILES_EXTRA_COLUMNS: list[ tuple[str, Callable[[dict[str, Any]], Any], str, bool, str] ] = [ @@ -1229,21 +1236,9 @@ def validate(data: dict[str, Any]) -> None: ] SKIPPED_FILE_REASON_COLUMNS: list[tuple[str, str, bool, str]] = [ - ( - "repository.name", - "Sourcegraph repository name containing the skipped file", - False, - "string", - ), - ( - "rev", - "Indexed revision parsed from Sourcegraph's skippedIndexed.query", - False, - "string", - ), ( "reason", - "NOT-INDEXED reason parsed from the indexed placeholder content", + "Compact NOT-INDEXED reason parsed from the indexed placeholder content", False, "string", ), @@ -1260,14 +1255,16 @@ def validate(data: dict[str, Any]) -> None: "integer", ), ( - "repoRevSkippedIndexed.count", - "Count Sourcegraph reported for this repo/ref before running the details search", + "file.distinctTrigramCount", + "Distinct three-rune trigrams computed from GitBlob.content. " + "Only populated with --skipped-file-metrics for skipped files whose " + "reason is `too_many_trigrams`", False, "integer", ), ( - "file.path", - "Path of the skipped file within the repository", + "rev", + "Indexed ref containing the skipped file", False, "string", ), @@ -1279,25 +1276,10 @@ def validate(data: dict[str, Any]) -> None: ), ] -SKIPPED_FILE_METRIC_COLUMNS: list[tuple[str, str, bool, str]] = [ - ( - "file.distinctTrigramCount", - "Distinct three-rune trigrams computed from GitBlob.content. " - "Only populated for skipped files whose reason is `contains too many trigrams`", - False, - "integer", - ), -] - -def skipped_file_reason_column_names(skipped_file_metrics: bool) -> list[str]: - """Return skipped-file detail CSV columns for the enabled modes""" - columns: list[str] = [] - for name, _, _, _ in SKIPPED_FILE_REASON_COLUMNS: - columns.append(name) - if skipped_file_metrics and name == "file.byteSize": - columns.extend(name for name, _, _, _ in SKIPPED_FILE_METRIC_COLUMNS) - return columns +def skipped_file_reason_column_names() -> list[str]: + """Return skipped-file detail CSV columns""" + return [name for name, _, _, _ in SKIPPED_FILE_REASON_COLUMNS] # --- Stats -------------------------------------------------------------------- @@ -1562,7 +1544,6 @@ def write_csv_schema(path: Path) -> None: cloning_list = format_columns_list(name_desc(CLONING_ERROR_EXTRA_COLUMNS)) skipped_list = format_columns_list(name_desc(SKIPPED_FILES_EXTRA_COLUMNS)) skipped_reason_list = format_columns_list(SKIPPED_FILE_REASON_COLUMNS) - skipped_metrics_list = format_columns_list(SKIPPED_FILE_METRIC_COLUMNS) commit_count_list = format_columns_list(COMMIT_COUNT_COLUMNS) run_search_list = format_columns_list(RUN_SEARCH_COLUMNS) stats_files_list = format_stats_files_list() @@ -1592,7 +1573,7 @@ def write_csv_schema(path: Path) -> None: | `{DEFAULT_CLONING_ERRORS_FILE}` | at least one repo has a cloning error | main columns + cloning-error extras | | `{DEFAULT_INDEXING_ERRORS_FILE}` | at least one repo is cloned but is missing a search index | main columns | | `{DEFAULT_SKIPPED_FILES_FILE}` | `--skipped-files` is set and the last index excluded files in at least one repo | main columns + skipped-files extras | -| `{DEFAULT_SKIPPED_FILE_REASONS_FILE}` | `--skipped-files-reason` is set, and at least one skipped-file detail row is found | skipped-file reason columns, plus skipped-file metrics columns when `--skipped-file-metrics` is set | +| `{DEFAULT_SKIPPED_FILE_REASONS_FILE}` | `--skipped-files-reason` is set, and at least one skipped-file detail row is found | skipped-file reason columns | | `{DEFAULT_SKIPPED_FILE_REASON_STATS_FILE}` | `--skipped-files-reason REPO[@REV]` is set, and at least one NOT-INDEXED reason category is found | `reason,count` | | `{DEFAULT_STATS_FILE_PREFIX}-*.csv` | `--stats` is set and repo rows were processed | `bucket,count` (see Stats section) | @@ -1632,14 +1613,6 @@ def write_csv_schema(path: Path) -> None: {skipped_reason_list} -## Skipped-file metrics columns - -Appended to skipped-file detail CSVs only when `--skipped-file-metrics` -is used. These columns may require extra GitBlob content requests, and are -therefore not fetched by default - -{skipped_metrics_list} - ## `--count-commits` columns Appended to CSV files when `--count-commits` is used @@ -2389,7 +2362,7 @@ def file_url(endpoint: str, repo_name: str, rev: str, file_path: str) -> str: def skipped_file_reason(match: dict[str, Any]) -> str: - """Extract the NOT-INDEXED reason from a skipped-file search match""" + """Extract the NOT-INDEXED explanation from a skipped-file search match""" chunks: list[dict[str, Any]] = match.get("chunkMatches") or [] for chunk in chunks: reason_match = re.search( @@ -2401,6 +2374,23 @@ def skipped_file_reason(match: dict[str, Any]) -> str: return "" +def skipped_file_reason_code(explanation: str) -> str: + """Return the compact CSV reason value for a NOT-INDEXED explanation""" + if not explanation: + return "" + if explanation in SKIPPED_FILE_REASON_CODES_BY_EXPLANATION: + return SKIPPED_FILE_REASON_CODES_BY_EXPLANATION[explanation] + normalized_explanation = re.sub(r"[^a-z0-9]+", "_", explanation.lower()).strip( + "_", + ) + return normalized_explanation or "unknown" + + +def skipped_file_reason_value(match: dict[str, Any]) -> str: + """Return the skipped-file detail CSV value for reason""" + return skipped_file_reason_code(skipped_file_reason(match)) + + def skipped_file_needs_metrics(match: dict[str, Any]) -> bool: """Return True when extra GitBlob metrics are useful for this skipped file""" return skipped_file_reason(match) == TOO_MANY_TRIGRAMS_REASON @@ -2659,7 +2649,7 @@ def write_skipped_files_reason( reason_counts: collections.Counter[str] = collections.Counter() for match in matches: - reason = skipped_file_reason(match) + reason = skipped_file_reason_value(match) if reason: reason_counts[reason] += 1 @@ -2679,7 +2669,7 @@ def write_skipped_files_reason( files_writer = LazyCSVWriter( output_dir / DEFAULT_SKIPPED_FILE_REASONS_FILE, - skipped_file_reason_column_names(skipped_file_metrics), + skipped_file_reason_column_names(), ) with files_writer as writer: write_skipped_file_reason_rows( @@ -3296,29 +3286,24 @@ def write_skipped_file_reason_rows( file_path = str(file_obj.get("path") or "") byte_size = file_obj.get("byteSize") file_extension = Path(file_path).suffix.lstrip(".") + distinct_trigram_count: int | str = "" + if skipped_file_metrics: + distinct_trigram_count = ( + search_result.distinct_trigram_counts_by_path.get(file_path, "") + ) row = [ - search_result.repository_name, - search_result.ref_name, - skipped_file_reason(match), + skipped_file_reason_value(match), file_extension, int(byte_size) if byte_size is not None else "", - ] - if skipped_file_metrics: - row.append( - search_result.distinct_trigram_counts_by_path.get(file_path, ""), - ) - row.extend( - [ - search_result.skipped_count, + distinct_trigram_count, + search_result.ref_name, + file_url( + endpoint, + search_result.repository_name, + search_result.indexed_rev, file_path, - file_url( - endpoint, - search_result.repository_name, - search_result.indexed_rev, - file_path, - ), - ], - ) + ), + ] writer.writerow( row, ) @@ -4216,7 +4201,7 @@ def run(args: argparse.Namespace, endpoint: str, token: str, output_dir: Path) - skipped_file_reason_writer = ( LazyCSVWriter( skipped_file_reasons_path, - skipped_file_reason_column_names(args.skipped_file_metrics), + skipped_file_reason_column_names(), ) if skipped_file_reasons_path is not None else None From d55f62cefd5cc059bf965f8d6e4599d7dc217d27 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 18 Jun 2026 16:32:52 -0600 Subject: [PATCH 21/26] Handle empty repo commit counts Amp-Thread-ID: https://ampcode.com/threads/T-019ecf95-ee6d-7426-b1e7-d724bdf73e0f Co-authored-by: Amp --- repo-troubleshooting/list-repos/list-repos.py | 56 +++++++++---------- 1 file changed, 26 insertions(+), 30 deletions(-) diff --git a/repo-troubleshooting/list-repos/list-repos.py b/repo-troubleshooting/list-repos/list-repos.py index d6a7a6c..4f32b9e 100755 --- a/repo-troubleshooting/list-repos/list-repos.py +++ b/repo-troubleshooting/list-repos/list-repos.py @@ -718,12 +718,24 @@ def fetch_commit_count( """Return exact rev count, approximate all-refs count, elapsed time, extras""" start = time.monotonic() + def all_refs_count_from_data(data: dict[str, Any]) -> int: + search_block = require_dict(data.get("search"), "commit-count search") + search_results = require_dict( + search_block.get("results"), + "commit-count search results", + ) + return require_int( + search_results.get("matchCount"), "commit-count all-refs matchCount" + ) + def validate(data: dict[str, Any]) -> None: repo_block = require_dict(data.get("repository"), "commit-count repository") + all_refs_count = all_refs_count_from_data(data) + commit_block = repo_block.get("commit") + if commit_block is None and all_refs_count == 0: + return commit_block = require_dict( - repo_block.get("commit"), - "commit-count commit", - retryable=False, + commit_block, "commit-count commit", retryable=False ) ancestors_block = require_dict( commit_block.get("ancestors"), @@ -733,14 +745,6 @@ def validate(data: dict[str, Any]) -> None: ancestors_block.get("totalCount"), "commit-count default branch totalCount", ) - search_block = require_dict(data.get("search"), "commit-count search") - search_results = require_dict( - search_block.get("results"), - "commit-count search results", - ) - require_int( - search_results.get("matchCount"), "commit-count all-refs matchCount" - ) data = graphql_request_with_validation( endpoint, @@ -758,25 +762,17 @@ def validate(data: dict[str, Any]) -> None: ) elapsed = time.monotonic() - start repo = require_dict(data.get("repository"), "commit-count repository") - commit = require_dict( - repo.get("commit"), - "commit-count commit", - retryable=False, - ) - ancestors = require_dict(commit.get("ancestors"), "commit-count ancestors") - default_count = require_int( - ancestors.get("totalCount"), - "commit-count default branch totalCount", - ) - search_block = require_dict(data.get("search"), "commit-count search") - search_results = require_dict( - search_block.get("results"), - "commit-count search results", - ) - all_refs_count = require_int( - search_results.get("matchCount"), - "commit-count all-refs matchCount", - ) + all_refs_count = all_refs_count_from_data(data) + commit = repo.get("commit") + if commit is None and all_refs_count == 0: + default_count = 0 + else: + commit = require_dict(commit, "commit-count commit", retryable=False) + ancestors = require_dict(commit.get("ancestors"), "commit-count ancestors") + default_count = require_int( + ancestors.get("totalCount"), + "commit-count default branch totalCount", + ) optimization_values = [ extract(repo) for _, extract, _, _, _ in COMMIT_COUNT_OPTIMIZATION_COLUMNS ] From b4c86061f6dec213ab6ec4f0c22e1ee4df397c60 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 18 Jun 2026 16:42:47 -0600 Subject: [PATCH 22/26] Classify skipped file extensions Amp-Thread-ID: https://ampcode.com/threads/T-019edcb2-c015-76fc-ab71-7ff166cb44e0 Co-authored-by: Amp --- .../list-repos/file-extensions.txt | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 repo-troubleshooting/list-repos/file-extensions.txt diff --git a/repo-troubleshooting/list-repos/file-extensions.txt b/repo-troubleshooting/list-repos/file-extensions.txt new file mode 100644 index 0000000..84b4f64 --- /dev/null +++ b/repo-troubleshooting/list-repos/file-extensions.txt @@ -0,0 +1,106 @@ +Expected UTF-8 text files +10K +S +as +b64 +base64 +bzl +c +cc +cjs +cpp +cs +cshtml +css +csv +drawio +eml +eps +excalidraw +go +golden +graphml +h +hpp +html +in +inc +ipynb +java +js +json +jsonl +lock +log +map +md +mdx +mht +min +mjs +nix +out +patch +pbtxt +pem +plist +po +podspec +pjson +properties +py +rb +response +resx +result +rs +scss +sql +sum +svg +svelte +tbl +test +test_slow +tex +texi +toml +ts +tsx +txt +uue +xlf +xml +yaml +yml + +Expected non-text or unsupported files +3gp +11 +afdesign +afphoto +avif +cr3 +crt +data +data-00000-of-00001 +dat +gpg +graffle +heic +heif +ico +jp2 +lcl +m4a +m4r +m4v +mov +mp4 +mpg +pdf +pf +snap +storage_i +ttf +wasm From a7e8a12e960bb124524098f7a2a65909535b7a20 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 18 Jun 2026 16:53:39 -0600 Subject: [PATCH 23/26] Describe skipped file extension types Amp-Thread-ID: https://ampcode.com/threads/T-019edcb2-c015-76fc-ab71-7ff166cb44e0 Co-authored-by: Amp --- .../list-repos/file-extensions.tsv | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 repo-troubleshooting/list-repos/file-extensions.tsv diff --git a/repo-troubleshooting/list-repos/file-extensions.tsv b/repo-troubleshooting/list-repos/file-extensions.tsv new file mode 100644 index 0000000..70caa70 --- /dev/null +++ b/repo-troubleshooting/list-repos/file-extensions.tsv @@ -0,0 +1,104 @@ +category extension common_app_or_tool description +Expected UTF-8 text files 10K SEC EDGAR SEC Form 10-K annual report filing, often stored as plain text or HTML. +Expected UTF-8 text files S GNU assembler / GCC Preprocessed assembly source file. +Expected UTF-8 text files as Adobe Animate / ActionScript ActionScript source code. +Expected UTF-8 text files b64 base64 utility Base64-encoded text payload. +Expected UTF-8 text files base64 base64 utility Base64-encoded text payload. +Expected UTF-8 text files bzl Bazel Starlark build definitions loaded by Bazel. +Expected UTF-8 text files c GCC / Clang C source code. +Expected UTF-8 text files cc GCC / Clang C++ source code. +Expected UTF-8 text files cjs Node.js CommonJS JavaScript module. +Expected UTF-8 text files cpp GCC / Clang C++ source code. +Expected UTF-8 text files cs .NET SDK / Visual Studio C# source code. +Expected UTF-8 text files cshtml ASP.NET Razor Razor view template combining HTML and C#. +Expected UTF-8 text files css Web browser Cascading Style Sheets stylesheet. +Expected UTF-8 text files csv Excel / Google Sheets Comma-separated tabular data. +Expected UTF-8 text files drawio diagrams.net XML diagram document used by draw.io/diagrams.net. +Expected UTF-8 text files eml Outlook / Thunderbird RFC 5322 email message source. +Expected UTF-8 text files eps Adobe Illustrator / Ghostscript Encapsulated PostScript vector graphic source. +Expected UTF-8 text files excalidraw Excalidraw JSON drawing document used by Excalidraw. +Expected UTF-8 text files go Go toolchain Go source code. +Expected UTF-8 text files golden Test framework Golden expected-output fixture for tests. +Expected UTF-8 text files graphml yEd / Gephi GraphML XML graph document. +Expected UTF-8 text files h GCC / Clang C or C++ header file. +Expected UTF-8 text files hpp GCC / Clang C++ header file. +Expected UTF-8 text files html Web browser HyperText Markup Language document. +Expected UTF-8 text files in Autoconf / build tools Input template consumed by build/configuration generators. +Expected UTF-8 text files inc Text editor / compiler Include fragment for another source or config file. +Expected UTF-8 text files ipynb Jupyter Notebook Notebook JSON containing cells, metadata, and outputs. +Expected UTF-8 text files java JDK / IntelliJ IDEA Java source code. +Expected UTF-8 text files js Web browser / Node.js JavaScript source code. +Expected UTF-8 text files json APIs / config tools JSON structured data document. +Expected UTF-8 text files jsonl Log pipelines / data tools Newline-delimited JSON records. +Expected UTF-8 text files lock Package manager Dependency lockfile generated by a package manager. +Expected UTF-8 text files log Log viewer / text editor Plain-text log output. +Expected UTF-8 text files map Browser devtools Source map or mapping data file. +Expected UTF-8 text files md GitHub / Markdown renderer Markdown document. +Expected UTF-8 text files mdx MDX / React docs Markdown with JSX components. +Expected UTF-8 text files mht Web browser / Outlook MHTML web archive stored as MIME text. +Expected UTF-8 text files min Web browser / bundler Minified text asset, commonly JavaScript or CSS. +Expected UTF-8 text files mjs Node.js / web browser ECMAScript JavaScript module. +Expected UTF-8 text files nix Nix Nix expression or package definition. +Expected UTF-8 text files out Text editor / test runner Captured command or test output. +Expected UTF-8 text files patch Git Unified diff patch file. +Expected UTF-8 text files pbtxt Protocol Buffers / TensorFlow Protocol Buffer text-format file. +Expected UTF-8 text files pem OpenSSL PEM-armored certificate, key, or other cryptographic data. +Expected UTF-8 text files plist Xcode / plutil Apple property list, often XML text. +Expected UTF-8 text files po GNU gettext Portable Object translation catalog. +Expected UTF-8 text files podspec CocoaPods Ruby DSL package specification for CocoaPods. +Expected UTF-8 text files pjson Package manager / JSON tools Project or package metadata stored as JSON-like text. +Expected UTF-8 text files properties Java Java properties key-value configuration. +Expected UTF-8 text files py Python Python source code. +Expected UTF-8 text files rb Ruby Ruby source code. +Expected UTF-8 text files response Postman / HTTP client Captured HTTP response text. +Expected UTF-8 text files resx Visual Studio .NET XML resource file. +Expected UTF-8 text files result Test framework Captured or expected test result text. +Expected UTF-8 text files rs Rust / Cargo Rust source code. +Expected UTF-8 text files scss Sass Sassy CSS stylesheet source. +Expected UTF-8 text files sql Database client SQL script or query file. +Expected UTF-8 text files sum Go toolchain Checksum manifest such as go.sum. +Expected UTF-8 text files svg Web browser / vector editor Scalable Vector Graphics XML document. +Expected UTF-8 text files svelte Svelte Svelte component file with markup, script, and style. +Expected UTF-8 text files tbl Text editor Table data or troff tbl source. +Expected UTF-8 text files test Test runner Generic test fixture or script. +Expected UTF-8 text files test_slow Test runner Generic slow-test fixture or marker file. +Expected UTF-8 text files tex LaTeX TeX or LaTeX source document. +Expected UTF-8 text files texi GNU Texinfo Texinfo documentation source. +Expected UTF-8 text files toml Config tools TOML configuration file. +Expected UTF-8 text files ts TypeScript TypeScript source code. +Expected UTF-8 text files tsx TypeScript / React TypeScript source with JSX. +Expected UTF-8 text files txt Text editor Plain text document. +Expected UTF-8 text files uue uudecode Uuencoded text representation of binary data. +Expected UTF-8 text files xlf Translation tool XLIFF XML localization file. +Expected UTF-8 text files xml XML tools XML structured data document. +Expected UTF-8 text files yaml Kubernetes / CI tools YAML configuration document. +Expected UTF-8 text files yml Kubernetes / CI tools YAML configuration document. +Expected non-text or unsupported files 3gp Media player 3GPP mobile video container, binary media. +Expected non-text or unsupported files 11 Application-specific tool Ambiguous numeric extension; not safe to assume UTF-8 text. +Expected non-text or unsupported files afdesign Affinity Designer Affinity Designer binary design document. +Expected non-text or unsupported files afphoto Affinity Photo Affinity Photo binary image document. +Expected non-text or unsupported files avif Image viewer / web browser AVIF compressed image file. +Expected non-text or unsupported files cr3 Canon Digital Photo Professional Canon RAW camera image file. +Expected non-text or unsupported files crt OpenSSL / OS certificate manager Certificate file that may be PEM text or DER binary; not safe to assume UTF-8. +Expected non-text or unsupported files data Application-specific tool Generic application data file, often binary. +Expected non-text or unsupported files data-00000-of-00001 TensorFlow / ML tooling Sharded machine-learning data file, usually binary. +Expected non-text or unsupported files dat Application-specific tool Generic data file, often binary. +Expected non-text or unsupported files gpg GnuPG Encrypted or signed GnuPG data, often binary or armored ciphertext. +Expected non-text or unsupported files graffle OmniGraffle OmniGraffle diagram document, not safe to treat as plain UTF-8 text. +Expected non-text or unsupported files heic Image viewer / Apple Photos HEIC compressed image file. +Expected non-text or unsupported files heif Image viewer / Apple Photos HEIF compressed image file. +Expected non-text or unsupported files ico Image viewer / web browser Windows icon image file. +Expected non-text or unsupported files jp2 Image viewer JPEG 2000 compressed image file. +Expected non-text or unsupported files lcl Application-specific tool Ambiguous local or cache file, not safe to assume UTF-8 text. +Expected non-text or unsupported files m4a Media player MPEG-4 audio file. +Expected non-text or unsupported files m4r Media player / iTunes MPEG-4 ringtone audio file. +Expected non-text or unsupported files m4v Media player MPEG-4 video file. +Expected non-text or unsupported files mov QuickTime / media player QuickTime video container. +Expected non-text or unsupported files mp4 Media player MPEG-4 video container. +Expected non-text or unsupported files mpg Media player MPEG video file. +Expected non-text or unsupported files pdf PDF reader Portable Document Format file, not plain UTF-8 text. +Expected non-text or unsupported files pf Application-specific tool Ambiguous prefetch or application data file, often binary. +Expected non-text or unsupported files snap Snap / application-specific tool Snap package or snapshot data, not plain UTF-8 text. +Expected non-text or unsupported files storage_i Storage engine Application storage shard or index file, likely binary. +Expected non-text or unsupported files ttf Font viewer TrueType font file. +Expected non-text or unsupported files wasm WebAssembly runtime WebAssembly binary module. From f7429befa68a76d736767034af581a2d079f868e Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 18 Jun 2026 17:30:59 -0600 Subject: [PATCH 24/26] Delete unneeded files --- .../list-repos/file-extensions.tsv | 104 ----------------- .../list-repos/file-extensions.txt | 106 ------------------ 2 files changed, 210 deletions(-) delete mode 100644 repo-troubleshooting/list-repos/file-extensions.tsv delete mode 100644 repo-troubleshooting/list-repos/file-extensions.txt diff --git a/repo-troubleshooting/list-repos/file-extensions.tsv b/repo-troubleshooting/list-repos/file-extensions.tsv deleted file mode 100644 index 70caa70..0000000 --- a/repo-troubleshooting/list-repos/file-extensions.tsv +++ /dev/null @@ -1,104 +0,0 @@ -category extension common_app_or_tool description -Expected UTF-8 text files 10K SEC EDGAR SEC Form 10-K annual report filing, often stored as plain text or HTML. -Expected UTF-8 text files S GNU assembler / GCC Preprocessed assembly source file. -Expected UTF-8 text files as Adobe Animate / ActionScript ActionScript source code. -Expected UTF-8 text files b64 base64 utility Base64-encoded text payload. -Expected UTF-8 text files base64 base64 utility Base64-encoded text payload. -Expected UTF-8 text files bzl Bazel Starlark build definitions loaded by Bazel. -Expected UTF-8 text files c GCC / Clang C source code. -Expected UTF-8 text files cc GCC / Clang C++ source code. -Expected UTF-8 text files cjs Node.js CommonJS JavaScript module. -Expected UTF-8 text files cpp GCC / Clang C++ source code. -Expected UTF-8 text files cs .NET SDK / Visual Studio C# source code. -Expected UTF-8 text files cshtml ASP.NET Razor Razor view template combining HTML and C#. -Expected UTF-8 text files css Web browser Cascading Style Sheets stylesheet. -Expected UTF-8 text files csv Excel / Google Sheets Comma-separated tabular data. -Expected UTF-8 text files drawio diagrams.net XML diagram document used by draw.io/diagrams.net. -Expected UTF-8 text files eml Outlook / Thunderbird RFC 5322 email message source. -Expected UTF-8 text files eps Adobe Illustrator / Ghostscript Encapsulated PostScript vector graphic source. -Expected UTF-8 text files excalidraw Excalidraw JSON drawing document used by Excalidraw. -Expected UTF-8 text files go Go toolchain Go source code. -Expected UTF-8 text files golden Test framework Golden expected-output fixture for tests. -Expected UTF-8 text files graphml yEd / Gephi GraphML XML graph document. -Expected UTF-8 text files h GCC / Clang C or C++ header file. -Expected UTF-8 text files hpp GCC / Clang C++ header file. -Expected UTF-8 text files html Web browser HyperText Markup Language document. -Expected UTF-8 text files in Autoconf / build tools Input template consumed by build/configuration generators. -Expected UTF-8 text files inc Text editor / compiler Include fragment for another source or config file. -Expected UTF-8 text files ipynb Jupyter Notebook Notebook JSON containing cells, metadata, and outputs. -Expected UTF-8 text files java JDK / IntelliJ IDEA Java source code. -Expected UTF-8 text files js Web browser / Node.js JavaScript source code. -Expected UTF-8 text files json APIs / config tools JSON structured data document. -Expected UTF-8 text files jsonl Log pipelines / data tools Newline-delimited JSON records. -Expected UTF-8 text files lock Package manager Dependency lockfile generated by a package manager. -Expected UTF-8 text files log Log viewer / text editor Plain-text log output. -Expected UTF-8 text files map Browser devtools Source map or mapping data file. -Expected UTF-8 text files md GitHub / Markdown renderer Markdown document. -Expected UTF-8 text files mdx MDX / React docs Markdown with JSX components. -Expected UTF-8 text files mht Web browser / Outlook MHTML web archive stored as MIME text. -Expected UTF-8 text files min Web browser / bundler Minified text asset, commonly JavaScript or CSS. -Expected UTF-8 text files mjs Node.js / web browser ECMAScript JavaScript module. -Expected UTF-8 text files nix Nix Nix expression or package definition. -Expected UTF-8 text files out Text editor / test runner Captured command or test output. -Expected UTF-8 text files patch Git Unified diff patch file. -Expected UTF-8 text files pbtxt Protocol Buffers / TensorFlow Protocol Buffer text-format file. -Expected UTF-8 text files pem OpenSSL PEM-armored certificate, key, or other cryptographic data. -Expected UTF-8 text files plist Xcode / plutil Apple property list, often XML text. -Expected UTF-8 text files po GNU gettext Portable Object translation catalog. -Expected UTF-8 text files podspec CocoaPods Ruby DSL package specification for CocoaPods. -Expected UTF-8 text files pjson Package manager / JSON tools Project or package metadata stored as JSON-like text. -Expected UTF-8 text files properties Java Java properties key-value configuration. -Expected UTF-8 text files py Python Python source code. -Expected UTF-8 text files rb Ruby Ruby source code. -Expected UTF-8 text files response Postman / HTTP client Captured HTTP response text. -Expected UTF-8 text files resx Visual Studio .NET XML resource file. -Expected UTF-8 text files result Test framework Captured or expected test result text. -Expected UTF-8 text files rs Rust / Cargo Rust source code. -Expected UTF-8 text files scss Sass Sassy CSS stylesheet source. -Expected UTF-8 text files sql Database client SQL script or query file. -Expected UTF-8 text files sum Go toolchain Checksum manifest such as go.sum. -Expected UTF-8 text files svg Web browser / vector editor Scalable Vector Graphics XML document. -Expected UTF-8 text files svelte Svelte Svelte component file with markup, script, and style. -Expected UTF-8 text files tbl Text editor Table data or troff tbl source. -Expected UTF-8 text files test Test runner Generic test fixture or script. -Expected UTF-8 text files test_slow Test runner Generic slow-test fixture or marker file. -Expected UTF-8 text files tex LaTeX TeX or LaTeX source document. -Expected UTF-8 text files texi GNU Texinfo Texinfo documentation source. -Expected UTF-8 text files toml Config tools TOML configuration file. -Expected UTF-8 text files ts TypeScript TypeScript source code. -Expected UTF-8 text files tsx TypeScript / React TypeScript source with JSX. -Expected UTF-8 text files txt Text editor Plain text document. -Expected UTF-8 text files uue uudecode Uuencoded text representation of binary data. -Expected UTF-8 text files xlf Translation tool XLIFF XML localization file. -Expected UTF-8 text files xml XML tools XML structured data document. -Expected UTF-8 text files yaml Kubernetes / CI tools YAML configuration document. -Expected UTF-8 text files yml Kubernetes / CI tools YAML configuration document. -Expected non-text or unsupported files 3gp Media player 3GPP mobile video container, binary media. -Expected non-text or unsupported files 11 Application-specific tool Ambiguous numeric extension; not safe to assume UTF-8 text. -Expected non-text or unsupported files afdesign Affinity Designer Affinity Designer binary design document. -Expected non-text or unsupported files afphoto Affinity Photo Affinity Photo binary image document. -Expected non-text or unsupported files avif Image viewer / web browser AVIF compressed image file. -Expected non-text or unsupported files cr3 Canon Digital Photo Professional Canon RAW camera image file. -Expected non-text or unsupported files crt OpenSSL / OS certificate manager Certificate file that may be PEM text or DER binary; not safe to assume UTF-8. -Expected non-text or unsupported files data Application-specific tool Generic application data file, often binary. -Expected non-text or unsupported files data-00000-of-00001 TensorFlow / ML tooling Sharded machine-learning data file, usually binary. -Expected non-text or unsupported files dat Application-specific tool Generic data file, often binary. -Expected non-text or unsupported files gpg GnuPG Encrypted or signed GnuPG data, often binary or armored ciphertext. -Expected non-text or unsupported files graffle OmniGraffle OmniGraffle diagram document, not safe to treat as plain UTF-8 text. -Expected non-text or unsupported files heic Image viewer / Apple Photos HEIC compressed image file. -Expected non-text or unsupported files heif Image viewer / Apple Photos HEIF compressed image file. -Expected non-text or unsupported files ico Image viewer / web browser Windows icon image file. -Expected non-text or unsupported files jp2 Image viewer JPEG 2000 compressed image file. -Expected non-text or unsupported files lcl Application-specific tool Ambiguous local or cache file, not safe to assume UTF-8 text. -Expected non-text or unsupported files m4a Media player MPEG-4 audio file. -Expected non-text or unsupported files m4r Media player / iTunes MPEG-4 ringtone audio file. -Expected non-text or unsupported files m4v Media player MPEG-4 video file. -Expected non-text or unsupported files mov QuickTime / media player QuickTime video container. -Expected non-text or unsupported files mp4 Media player MPEG-4 video container. -Expected non-text or unsupported files mpg Media player MPEG video file. -Expected non-text or unsupported files pdf PDF reader Portable Document Format file, not plain UTF-8 text. -Expected non-text or unsupported files pf Application-specific tool Ambiguous prefetch or application data file, often binary. -Expected non-text or unsupported files snap Snap / application-specific tool Snap package or snapshot data, not plain UTF-8 text. -Expected non-text or unsupported files storage_i Storage engine Application storage shard or index file, likely binary. -Expected non-text or unsupported files ttf Font viewer TrueType font file. -Expected non-text or unsupported files wasm WebAssembly runtime WebAssembly binary module. diff --git a/repo-troubleshooting/list-repos/file-extensions.txt b/repo-troubleshooting/list-repos/file-extensions.txt deleted file mode 100644 index 84b4f64..0000000 --- a/repo-troubleshooting/list-repos/file-extensions.txt +++ /dev/null @@ -1,106 +0,0 @@ -Expected UTF-8 text files -10K -S -as -b64 -base64 -bzl -c -cc -cjs -cpp -cs -cshtml -css -csv -drawio -eml -eps -excalidraw -go -golden -graphml -h -hpp -html -in -inc -ipynb -java -js -json -jsonl -lock -log -map -md -mdx -mht -min -mjs -nix -out -patch -pbtxt -pem -plist -po -podspec -pjson -properties -py -rb -response -resx -result -rs -scss -sql -sum -svg -svelte -tbl -test -test_slow -tex -texi -toml -ts -tsx -txt -uue -xlf -xml -yaml -yml - -Expected non-text or unsupported files -3gp -11 -afdesign -afphoto -avif -cr3 -crt -data -data-00000-of-00001 -dat -gpg -graffle -heic -heif -ico -jp2 -lcl -m4a -m4r -m4v -mov -mp4 -mpg -pdf -pf -snap -storage_i -ttf -wasm From 7abc6d39fcee3678627172774b9a0863b9995d61 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 18 Jun 2026 17:43:13 -0600 Subject: [PATCH 25/26] Retry stale skipped-file detail searches Amp-Thread-ID: https://ampcode.com/threads/T-019ecf95-ee6d-7426-b1e7-d724bdf73e0f Co-authored-by: Amp --- repo-troubleshooting/list-repos/CSV_SCHEMA.md | 7 +- repo-troubleshooting/list-repos/list-repos.py | 496 ++++++++++++++---- 2 files changed, 391 insertions(+), 112 deletions(-) diff --git a/repo-troubleshooting/list-repos/CSV_SCHEMA.md b/repo-troubleshooting/list-repos/CSV_SCHEMA.md index 9691688..5e72b6d 100644 --- a/repo-troubleshooting/list-repos/CSV_SCHEMA.md +++ b/repo-troubleshooting/list-repos/CSV_SCHEMA.md @@ -36,7 +36,7 @@ holding every output row in memory | `repos-with-cloning-errors.csv` | `url` | | `repos-with-indexing-errors.csv` | `url` | | `repos-with-skipped-files.csv` | `url` | -| `skipped-files-reason-details.csv` | `reason`, `file.extension`, `rev`, `file_url` | +| `skipped-files-reason-details.csv` | `repository.name`, `rev`, `reason`, `file.extension`, `file.path` | The optional `--count-commits` and `--run-search` flags append extra columns to the repo-listing CSVs above, excluding the `--stats` @@ -107,11 +107,14 @@ Written to `skipped-files-reason-details.csv` when | Column | Type | Requires admin | Description | | --- | --- | --- | --- | +| `repository.name` | string | | Sourcegraph repository name containing the skipped file | +| `rev` | string | | Indexed ref containing the skipped file | | `reason` | string | | Compact NOT-INDEXED reason parsed from the indexed placeholder content | | `file.extension` | string | | File extension derived from file.path | | `file.byteSize` | integer | | Sourcegraph-reported file byte size | | `file.distinctTrigramCount` | integer | | Distinct three-rune trigrams computed from GitBlob.content. Only populated with --skipped-file-metrics for skipped files whose reason is `too_many_trigrams` | -| `rev` | string | | Indexed ref containing the skipped file | +| `repoRevSkippedIndexed.count` | integer | | Skipped-file count Sourcegraph reported for this repository ref | +| `file.path` | string | | Path of the skipped file inside the repository | | `file_url` | string | | Sourcegraph blob URL for the skipped file at the indexed ref | ## `--count-commits` columns diff --git a/repo-troubleshooting/list-repos/list-repos.py b/repo-troubleshooting/list-repos/list-repos.py index 4f32b9e..d1da0be 100755 --- a/repo-troubleshooting/list-repos/list-repos.py +++ b/repo-troubleshooting/list-repos/list-repos.py @@ -137,7 +137,7 @@ def emit(self, record: logging.LogRecord) -> None: (DEFAULT_SKIPPED_FILES_FILE, ("url",)), ( DEFAULT_SKIPPED_FILE_REASONS_FILE, - ("reason", "file.extension", "rev", "file_url"), + ("repository.name", "rev", "reason", "file.extension", "file.path"), ), ) @@ -477,6 +477,29 @@ def build_run_search_query(repo_name: str, pattern: str) -> str: } """ +SKIPPED_FILE_REF_METADATA_QUERY = """ +query SkippedFileRefMetadata($name: String!) { + repository(name: $name) { + name + textSearchIndex { + refs { + ref { + displayName + } + indexed + indexedCommit { + oid + } + skippedIndexed { + count + query + } + } + } + } +} +""" + # --- Metadata extractors used by the COLUMNS table -------------------------------- @@ -653,26 +676,43 @@ def refs_with_skipped_file_queries( """Return (ref name, skipped count, API search query, indexed commit) tuples""" refs: list[tuple[str, int, str, str]] = [] for ref in _index_refs(repo): - if ref.get("indexed") is False: - continue - skipped: dict[str, Any] = ref.get("skippedIndexed") or {} - count = skipped.get("count") - if count is None: + ref_state = skipped_file_ref_state(ref) + if ref_state is None: continue - skipped_count = int(count) + name, skipped_count, skipped_indexed_query, indexed_oid = ref_state if skipped_count <= 0: continue - ref_node: dict[str, Any] = ref.get("ref") or {} - name = str(ref_node.get("displayName") or "") - if name: - indexed_commit: dict[str, Any] = ref.get("indexedCommit") or {} - indexed_oid = str(indexed_commit.get("oid") or "") - refs.append( - (name, skipped_count, str(skipped.get("query") or ""), indexed_oid) - ) + refs.append((name, skipped_count, skipped_indexed_query, indexed_oid)) return refs +def skipped_file_ref_state(ref: dict[str, Any]) -> tuple[str, int, str, str] | None: + """Return skipped-file metadata for one indexed ref, including zero skips""" + if ref.get("indexed") is False: + return None + ref_node: dict[str, Any] = ref.get("ref") or {} + name = str(ref_node.get("displayName") or "") + if not name: + return None + skipped: dict[str, Any] = ref.get("skippedIndexed") or {} + count = int(skipped.get("count") or 0) + indexed_commit: dict[str, Any] = ref.get("indexedCommit") or {} + indexed_oid = str(indexed_commit.get("oid") or "") + return name, count, str(skipped.get("query") or ""), indexed_oid + + +def skipped_file_ref_state_by_name( + repo: dict[str, Any], + ref_name: str, +) -> tuple[str, int, str, str] | None: + """Return skipped-file metadata for a named indexed ref""" + for ref in _index_refs(repo): + ref_state = skipped_file_ref_state(ref) + if ref_state is not None and ref_state[0] == ref_name: + return ref_state + return None + + def refs_with_skipped_file_counts(repo: dict[str, Any]) -> list[tuple[str, int]]: """Return (ref name, skipped count) pairs for refs with skipped files""" return [ @@ -1232,6 +1272,18 @@ def validate(data: dict[str, Any]) -> None: ] SKIPPED_FILE_REASON_COLUMNS: list[tuple[str, str, bool, str]] = [ + ( + "repository.name", + "Sourcegraph repository name containing the skipped file", + False, + "string", + ), + ( + "rev", + "Indexed ref containing the skipped file", + False, + "string", + ), ( "reason", "Compact NOT-INDEXED reason parsed from the indexed placeholder content", @@ -1259,8 +1311,14 @@ def validate(data: dict[str, Any]) -> None: "integer", ), ( - "rev", - "Indexed ref containing the skipped file", + "repoRevSkippedIndexed.count", + "Skipped-file count Sourcegraph reported for this repository ref", + False, + "integer", + ), + ( + "file.path", + "Path of the skipped file inside the repository", False, "string", ), @@ -2148,6 +2206,38 @@ def fetch_single_repo( return cast("dict[str, Any]", repo) +def fetch_skipped_file_ref_metadata( + endpoint: str, + token: str, + repo_name: str, + *, + max_retries: int = DEFAULT_MAX_RETRIES, +) -> dict[str, Any]: + """Fetch fresh text-search skipped-file ref metadata for one repo""" + + def validate(data: dict[str, Any]) -> None: + require_dict( + data.get("repository"), + "skipped-file metadata repository", + retryable=False, + ) + + data = graphql_request_with_validation( + endpoint, + token, + SKIPPED_FILE_REF_METADATA_QUERY, + {"name": repo_name}, + max_retries=max_retries, + request_description=f"Skipped-file metadata refresh for {repo_name}", + validate=validate, + ) + return require_dict( + data.get("repository"), + "skipped-file metadata repository", + retryable=False, + ) + + def trigger_reclone( endpoint: str, token: str, @@ -2602,6 +2692,233 @@ def validate(data: dict[str, Any]) -> None: ) +def skipped_file_reason_query_issue( + query_result: SkippedFileReasonQueryResult, + expected_skipped_count: int, +) -> str | None: + """Return why a skipped-file reason result is incomplete, or None""" + file_match_count = len(query_result.matches) + if query_result.limit_hit: + return ( + "search hit result limit: " + f"matchCount={query_result.match_count}, " + f"fileMatches={file_match_count}, " + f"textSearchIndex.refs.skippedIndexed.count={expected_skipped_count}" + ) + if query_result.match_count is None: + return ( + "search response was missing matchCount: " + f"fileMatches={file_match_count}, " + f"textSearchIndex.refs.skippedIndexed.count={expected_skipped_count}" + ) + if ( + file_match_count != expected_skipped_count + or query_result.match_count != expected_skipped_count + ): + return ( + f"search returned {file_match_count} file match(es) " + f"(matchCount={query_result.match_count}, " + f"limitHit={query_result.limit_hit}) but " + "textSearchIndex.refs.skippedIndexed.count reported " + f"{expected_skipped_count}" + ) + return None + + +def build_skipped_file_reason_search_result( + repository_name: str, + reference_name: str, + indexed_revision: str, + skipped_count: int, + query_result: SkippedFileReasonQueryResult | None, + error: str | None, +) -> SkippedFileReasonSearchResult: + """Build a skipped-file reason search result without retaining partial data on error""" + if query_result is None or error is not None: + return SkippedFileReasonSearchResult( + repository_name=repository_name, + ref_name=reference_name, + indexed_rev=indexed_revision, + skipped_count=skipped_count, + matches=[], + match_count=None, + limit_hit=False, + alert_title=None, + alert_description=None, + distinct_trigram_counts_by_path={}, + error=error, + ) + return SkippedFileReasonSearchResult( + repository_name=repository_name, + ref_name=reference_name, + indexed_rev=indexed_revision, + skipped_count=skipped_count, + matches=query_result.matches, + match_count=query_result.match_count, + limit_hit=query_result.limit_hit, + alert_title=query_result.alert_title, + alert_description=query_result.alert_description, + distinct_trigram_counts_by_path={}, + error=None, + ) + + +def fetch_consistent_skipped_file_reason_search_result( + endpoint: str, + token: str, + repository_name: str, + reference_name: str, + skipped_count: int, + skipped_indexed_query: str, + indexed_commit_oid: str, + max_retries: int = DEFAULT_MAX_RETRIES, +) -> SkippedFileReasonSearchResult: + """Fetch skipped-file details, refreshing metadata when counts disagree""" + current_ref_state: tuple[str, int, str, str] | None = ( + reference_name, + skipped_count, + skipped_indexed_query, + indexed_commit_oid, + ) + latest_indexed_revision = indexed_commit_oid or skipped_file_query_revision( + skipped_indexed_query, + reference_name, + ) + latest_skipped_count = skipped_count + last_issue = "skipped-file reason search did not run" + + for retry_count in range(max_retries + 1): + retry_number = retry_count + 1 + if current_ref_state is None: + last_issue = f"textSearchIndex.refs no longer contains indexed ref {reference_name!r}" + else: + ( + _current_reference_name, + latest_skipped_count, + current_skipped_indexed_query, + current_indexed_commit_oid, + ) = current_ref_state + latest_indexed_revision = ( + current_indexed_commit_oid + or skipped_file_query_revision( + current_skipped_indexed_query, + reference_name, + ) + ) + if latest_skipped_count <= 0: + if retry_count > 0: + logger.warning( + "Skipped-file metadata for %s@%s now reports 0 skipped " + "file(s) after retry; no skipped-file detail rows needed", + repository_name, + reference_name, + ) + empty_query_result = SkippedFileReasonQueryResult( + matches=[], + match_count=0, + limit_hit=False, + alert_title=None, + alert_description=None, + ) + return build_skipped_file_reason_search_result( + repository_name, + reference_name, + latest_indexed_revision, + latest_skipped_count, + empty_query_result, + error=None, + ) + try: + query_result = fetch_skipped_file_reason_query( + endpoint, + token, + repository_name, + latest_indexed_revision, + current_skipped_indexed_query, + max_retries=max_retries, + ) + except REQUEST_FAILURE_TYPES as error: + return build_skipped_file_reason_search_result( + repository_name, + reference_name, + latest_indexed_revision, + latest_skipped_count, + query_result=None, + error=request_error_summary(error), + ) + + last_issue = ( + skipped_file_reason_query_issue( + query_result, + latest_skipped_count, + ) + or "" + ) + if not last_issue: + return build_skipped_file_reason_search_result( + repository_name, + reference_name, + latest_indexed_revision, + latest_skipped_count, + query_result, + error=None, + ) + + if retry_count >= max_retries: + logger.error( + "Skipped-file reason search for %s@%s failed after %d attempt(s): %s", + repository_name, + reference_name, + retry_number, + last_issue, + ) + return build_skipped_file_reason_search_result( + repository_name, + reference_name, + latest_indexed_revision, + latest_skipped_count, + query_result=None, + error=last_issue, + ) + + sleep_before_retry( + f"Skipped-file reason search for {repository_name}@{reference_name} " + "returned inconsistent results", + retry_number, + max_retries, + message=last_issue, + ) + try: + refreshed_repo = fetch_skipped_file_ref_metadata( + endpoint, + token, + repository_name, + max_retries=max_retries, + ) + except REQUEST_FAILURE_TYPES as error: + return build_skipped_file_reason_search_result( + repository_name, + reference_name, + latest_indexed_revision, + latest_skipped_count, + query_result=None, + error=request_error_summary(error), + ) + current_ref_state = skipped_file_ref_state_by_name( + refreshed_repo, + reference_name, + ) + + return build_skipped_file_reason_search_result( + repository_name, + reference_name, + latest_indexed_revision, + latest_skipped_count, + query_result=None, + error=last_issue, + ) + + def write_skipped_files_reason( endpoint: str, token: str, @@ -2623,25 +2940,33 @@ def write_skipped_files_reason( ("count", lambda r: r[1]), ] - query_result = fetch_skipped_file_reason_query( + search_result = fetch_consistent_skipped_file_reason_search_result( endpoint, token, name, - indexed_rev, + display_rev, + skipped_count, skipped_indexed_query, + indexed_rev, max_retries=max_retries, ) - matches = query_result.matches + if search_result.error is not None: + return + + matches = search_result.matches distinct_trigram_counts_by_path: dict[str, int] = {} if skipped_file_metrics: distinct_trigram_counts_by_path = collect_distinct_trigram_counts( endpoint, token, name, - indexed_rev, + search_result.indexed_rev, matches, max_retries, ) + search_result.distinct_trigram_counts_by_path.update( + distinct_trigram_counts_by_path + ) reason_counts: collections.Counter[str] = collections.Counter() for match in matches: @@ -2649,20 +2974,6 @@ def write_skipped_files_reason( if reason: reason_counts[reason] += 1 - search_result = SkippedFileReasonSearchResult( - repository_name=name, - ref_name=display_rev, - indexed_rev=indexed_rev, - skipped_count=skipped_count, - matches=matches, - match_count=query_result.match_count, - limit_hit=query_result.limit_hit, - alert_title=query_result.alert_title, - alert_description=query_result.alert_description, - distinct_trigram_counts_by_path=distinct_trigram_counts_by_path, - error=None, - ) - files_writer = LazyCSVWriter( output_dir / DEFAULT_SKIPPED_FILE_REASONS_FILE, skipped_file_reason_column_names(), @@ -3172,59 +3483,28 @@ def collect_skipped_file_reason_search_results( skipped_indexed_query, display_ref_name, ) - try: - query_result = fetch_skipped_file_reason_query( - endpoint, - token, - repo_name, - indexed_rev, - skipped_indexed_query, - max_retries=max_retries, - ) - except REQUEST_FAILURE_TYPES as error: - results.append( - SkippedFileReasonSearchResult( - repository_name=repo_name, - ref_name=display_ref_name, - indexed_rev=indexed_rev, - skipped_count=skipped_count, - matches=[], - match_count=None, - limit_hit=False, - alert_title=None, - alert_description=None, - distinct_trigram_counts_by_path={}, - error=request_error_summary(error), + search_result = fetch_consistent_skipped_file_reason_search_result( + endpoint, + token, + repo_name, + display_ref_name, + skipped_count, + skipped_indexed_query, + indexed_rev, + max_retries=max_retries, + ) + if skipped_file_metrics and search_result.error is None: + search_result.distinct_trigram_counts_by_path.update( + collect_distinct_trigram_counts( + endpoint, + token, + repo_name, + search_result.indexed_rev, + search_result.matches, + max_retries, ), ) - continue - distinct_trigram_counts_by_path = ( - collect_distinct_trigram_counts( - endpoint, - token, - repo_name, - indexed_rev, - query_result.matches, - max_retries, - ) - if skipped_file_metrics - else {} - ) - results.append( - SkippedFileReasonSearchResult( - repository_name=repo_name, - ref_name=display_ref_name, - indexed_rev=indexed_rev, - skipped_count=skipped_count, - matches=query_result.matches, - match_count=query_result.match_count, - limit_hit=query_result.limit_hit, - alert_title=query_result.alert_title, - alert_description=query_result.alert_description, - distinct_trigram_counts_by_path=distinct_trigram_counts_by_path, - error=None, - ), - ) + results.append(search_result) return results @@ -3244,16 +3524,25 @@ def write_skipped_file_reason_rows( for part in (search_result.alert_title, search_result.alert_description) if part ] - if search_result.limit_hit: - logger.warning( - "Skipped-file reason search hit a result limit for %s@%s: " - "matchCount=%s, fileMatches=%d, skippedIndexed.count=%d", + query_issue = skipped_file_reason_query_issue( + SkippedFileReasonQueryResult( + matches=matches, + match_count=search_result.match_count, + limit_hit=search_result.limit_hit, + alert_title=search_result.alert_title, + alert_description=search_result.alert_description, + ), + search_result.skipped_count, + ) + if query_issue is not None: + logger.error( + "Skipped-file reason search for %s@%s has incomplete results " + "after retry handling: %s", search_result.repository_name, search_result.ref_name, - search_result.match_count, - len(matches), - search_result.skipped_count, + query_issue, ) + continue if alert_parts: logger.warning( "Skipped-file reason search returned alert for %s@%s: %s", @@ -3261,22 +3550,6 @@ def write_skipped_file_reason_rows( search_result.ref_name, "; ".join(alert_parts), ) - match_count_mismatch = ( - search_result.match_count is not None - and search_result.match_count != search_result.skipped_count - ) - if len(matches) != search_result.skipped_count or match_count_mismatch: - logger.warning( - "Skipped-file reason search returned %d file match(es) " - "(matchCount=%s, limitHit=%s) for %s@%s; " - "textSearchIndex.refs.skippedIndexed.count reported %d", - len(matches), - search_result.match_count, - search_result.limit_hit, - search_result.repository_name, - search_result.ref_name, - search_result.skipped_count, - ) for match in matches: file_obj: dict[str, Any] = match.get("file") or {} file_path = str(file_obj.get("path") or "") @@ -3288,11 +3561,14 @@ def write_skipped_file_reason_rows( search_result.distinct_trigram_counts_by_path.get(file_path, "") ) row = [ + search_result.repository_name, + search_result.ref_name, skipped_file_reason_value(match), file_extension, int(byte_size) if byte_size is not None else "", distinct_trigram_count, - search_result.ref_name, + search_result.skipped_count, + file_path, file_url( endpoint, search_result.repository_name, From d06f76605272744ec411e20545ee858272e59821 Mon Sep 17 00:00:00 2001 From: Marc LeBlanc <7050295+marcleblanc2@users.noreply.github.com> Date: Thu, 18 Jun 2026 17:49:23 -0600 Subject: [PATCH 26/26] Add Zoekt trigram counting script Amp-Thread-ID: https://ampcode.com/threads/T-019edd14-9163-71fe-b863-25c4c69087f3 Co-authored-by: Amp --- .../list-repos/zoekt-trigram-counts.py | 755 ++++++++++++++++++ 1 file changed, 755 insertions(+) create mode 100755 repo-troubleshooting/list-repos/zoekt-trigram-counts.py diff --git a/repo-troubleshooting/list-repos/zoekt-trigram-counts.py b/repo-troubleshooting/list-repos/zoekt-trigram-counts.py new file mode 100755 index 0000000..208e804 --- /dev/null +++ b/repo-troubleshooting/list-repos/zoekt-trigram-counts.py @@ -0,0 +1,755 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import concurrent.futures +import csv +import os +import stat +import subprocess +import sys +from collections.abc import Iterator, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Final, TextIO + + +DEFAULT_ROOT = Path("~/git") +DEFAULT_OUTPUT = Path("zoekt-trigram-counts.tsv") +DEFAULT_THRESHOLD = 20_000 +DEFAULT_PATH_SUFFIX_LENGTH = 20 +READ_CHUNK_SIZE = 1_048_576 +RUNE_ERROR: Final = 0xFFFD + + +@dataclass(frozen=True) +class DiscoveredFiles: + """Files found under the requested root.""" + + git_worktrees: list[Path] + plain_files: list[Path] + + +@dataclass(frozen=True) +class DecodedRune: + """A rune decoded the same way Go's utf8.DecodeRune decodes bytes.""" + + codepoint: int + byte_count: int + + +@dataclass(frozen=True) +class CountedFile: + """Zoekt-style unique trigram count for one text file.""" + + path: str + byte_size: int + unique_trigrams: int + + +@dataclass(frozen=True) +class SkippedFile: + """A file that could not or should not be counted.""" + + path: str + reason: str + detail: str = "" + + +FileProcessingResult = CountedFile | SkippedFile + + +def parse_non_negative_integer(value_text: str) -> int: + """Parse an argparse integer value that must be zero or greater.""" + try: + value = int(value_text) + except ValueError: + message = f"{value_text!r} is not an integer" + raise argparse.ArgumentTypeError(message) from None + if value < 0: + message = f"{value_text!r} must be zero or greater" + raise argparse.ArgumentTypeError(message) + return value + + +def parse_positive_integer(value_text: str) -> int: + """Parse an argparse integer value that must be one or greater.""" + value = parse_non_negative_integer(value_text) + if value == 0: + message = f"{value_text!r} must be one or greater" + raise argparse.ArgumentTypeError(message) + return value + + +def parse_arguments(argument_values: Sequence[str]) -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser( + description=( + "Count Sourcegraph Zoekt-style unique trigrams for local files. " + "Git worktrees are listed with `git ls-files --cached --others " + "--exclude-standard`, so .gitignore-covered files are excluded. " + "Binary means Zoekt's null-byte binary check." + ) + ) + parser.add_argument( + "--root", + type=Path, + default=DEFAULT_ROOT, + help="Root directory to scan. Default: %(default)s", + ) + parser.add_argument( + "--output", + type=Path, + default=DEFAULT_OUTPUT, + help="TSV output path, or '-' for stdout. Default: %(default)s", + ) + parser.add_argument( + "--threshold", + type=parse_non_negative_integer, + default=DEFAULT_THRESHOLD, + help="Zoekt TrigramMax threshold used for the would-skip column.", + ) + parser.add_argument( + "--dedupe-path-suffix-length", + type=parse_non_negative_integer, + default=DEFAULT_PATH_SUFFIX_LENGTH, + help=( + "Skip later files whose full path has the same final N characters. " + "Use 0 to disable. Default: %(default)s" + ), + ) + parser.add_argument( + "--workers", + type=parse_positive_integer, + default=max(1, os.cpu_count() or 1), + help="Worker processes used while counting. Default: CPU count.", + ) + parser.add_argument( + "--sort", + choices=("trigrams", "path"), + default="trigrams", + help="Sort TSV rows by descending trigram count or by path.", + ) + parser.add_argument( + "--progress-every", + type=parse_non_negative_integer, + default=1_000, + help="Print progress every N files to stderr. Use 0 to disable.", + ) + return parser.parse_args(argument_values) + + +def resolve_scan_root(root: Path) -> Path: + """Expand and validate the scan root.""" + resolved_root = root.expanduser().resolve() + if not resolved_root.exists(): + raise SystemExit(f"scan root does not exist: {resolved_root}") + if not resolved_root.is_dir(): + raise SystemExit(f"scan root is not a directory: {resolved_root}") + return resolved_root + + +def find_containing_git_worktree(root: Path) -> Path | None: + """Return the Git worktree containing root, if root is inside one.""" + command = ["git", "-C", str(root), "rev-parse", "--show-toplevel"] + try: + completed_process = subprocess.run( + command, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + except FileNotFoundError: + return None + + if completed_process.returncode != 0: + return None + + worktree_text = completed_process.stdout.decode("utf-8", errors="replace").strip() + if not worktree_text: + return None + return Path(worktree_text).resolve() + + +def discover_files(root: Path) -> DiscoveredFiles: + """Find Git worktrees and non-Git files under root.""" + containing_git_worktree = find_containing_git_worktree(root) + if containing_git_worktree is not None and containing_git_worktree != root: + return DiscoveredFiles(git_worktrees=[containing_git_worktree], plain_files=[]) + + git_worktrees: list[Path] = [] + plain_files: list[Path] = [] + + for current_directory_text, directory_names, file_names in os.walk(root): + if ".git" in directory_names or ".git" in file_names: + git_worktrees.append(Path(current_directory_text)) + directory_names.clear() + continue + + directory_names[:] = [name for name in directory_names if name != ".git"] + + current_directory = Path(current_directory_text) + for file_name in file_names: + if file_name == ".git": + continue + plain_files.append(current_directory / file_name) + + return DiscoveredFiles( + git_worktrees=sorted(git_worktrees), + plain_files=sorted(plain_files), + ) + + +def iter_git_worktree_files(git_worktree: Path) -> Iterator[Path]: + """Yield tracked and untracked, non-ignored files from one Git worktree.""" + command = [ + "git", + "-C", + str(git_worktree), + "ls-files", + "--cached", + "--others", + "--exclude-standard", + "-z", + ] + try: + completed_process = subprocess.run( + command, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + except FileNotFoundError: + raise SystemExit("git was not found in PATH") from None + + if completed_process.returncode != 0: + error_text = completed_process.stderr.decode("utf-8", errors="replace").strip() + print( + f"warning: could not list files in {git_worktree}: {error_text}", + file=sys.stderr, + ) + return + + for relative_path_bytes in completed_process.stdout.split(b"\0"): + if not relative_path_bytes: + continue + yield git_worktree / Path(os.fsdecode(relative_path_bytes)) + + +def collect_candidate_files( + discovered_files: DiscoveredFiles, root: Path +) -> list[Path]: + """Collect all files that should be considered for trigram counting.""" + candidate_files = list(discovered_files.plain_files) + for git_worktree in discovered_files.git_worktrees: + candidate_files.extend(iter_git_worktree_files(git_worktree)) + return sorted( + ( + candidate_file + for candidate_file in candidate_files + if is_relative_to(candidate_file, root) + ), + key=path_sort_text, + ) + + +def is_relative_to(path: Path, root: Path) -> bool: + """Return whether path is under root without following symlinks.""" + try: + path.relative_to(root) + except ValueError: + return False + return True + + +def path_sort_text(path: Path) -> str: + """Return a stable text form for path sorting and suffix comparison.""" + return path.as_posix() + + +def dedupe_by_path_suffix( + candidate_files: list[Path], path_suffix_length: int +) -> tuple[list[Path], int]: + """Skip later paths with the same final N characters.""" + if path_suffix_length == 0: + return candidate_files, 0 + + seen_path_suffixes: set[str] = set() + kept_files: list[Path] = [] + skipped_count = 0 + + for file_path in candidate_files: + path_text = path_sort_text(file_path) + path_suffix = path_text[-path_suffix_length:] + if path_suffix in seen_path_suffixes: + skipped_count += 1 + continue + seen_path_suffixes.add(path_suffix) + kept_files.append(file_path) + + return kept_files, skipped_count + + +def exclude_output_file(candidate_files: list[Path], output_path: Path) -> list[Path]: + """Do not count a pre-existing TSV from an earlier run.""" + if str(output_path) == "-": + return candidate_files + + expanded_output_path = output_path.expanduser() + if not expanded_output_path.is_absolute(): + expanded_output_path = Path.cwd() / expanded_output_path + output_path_text = path_sort_text(expanded_output_path.resolve(strict=False)) + return [ + candidate_file + for candidate_file in candidate_files + if path_sort_text(candidate_file) != output_path_text + ] + + +def decode_go_utf8_prefix( + content: bytes, start: int, end: int, *, final: bool +) -> DecodedRune | None: + """Decode one rune like Go's utf8.DecodeRune. + + Returns None only when more bytes are needed from the next chunk. + """ + first_byte = content[start] + if first_byte < 0x80: + return DecodedRune(first_byte, 1) + if 0xC2 <= first_byte <= 0xDF: + return decode_go_multibyte_utf8( + content, + start, + end, + final=final, + byte_count=2, + second_byte_minimum=0x80, + second_byte_maximum=0xBF, + ) + if first_byte == 0xE0: + return decode_go_multibyte_utf8( + content, + start, + end, + final=final, + byte_count=3, + second_byte_minimum=0xA0, + second_byte_maximum=0xBF, + ) + if 0xE1 <= first_byte <= 0xEC or 0xEE <= first_byte <= 0xEF: + return decode_go_multibyte_utf8( + content, + start, + end, + final=final, + byte_count=3, + second_byte_minimum=0x80, + second_byte_maximum=0xBF, + ) + if first_byte == 0xED: + return decode_go_multibyte_utf8( + content, + start, + end, + final=final, + byte_count=3, + second_byte_minimum=0x80, + second_byte_maximum=0x9F, + ) + if first_byte == 0xF0: + return decode_go_multibyte_utf8( + content, + start, + end, + final=final, + byte_count=4, + second_byte_minimum=0x90, + second_byte_maximum=0xBF, + ) + if 0xF1 <= first_byte <= 0xF3: + return decode_go_multibyte_utf8( + content, + start, + end, + final=final, + byte_count=4, + second_byte_minimum=0x80, + second_byte_maximum=0xBF, + ) + if first_byte == 0xF4: + return decode_go_multibyte_utf8( + content, + start, + end, + final=final, + byte_count=4, + second_byte_minimum=0x80, + second_byte_maximum=0x8F, + ) + return DecodedRune(RUNE_ERROR, 1) + + +def decode_go_multibyte_utf8( + content: bytes, + start: int, + end: int, + *, + final: bool, + byte_count: int, + second_byte_minimum: int, + second_byte_maximum: int, +) -> DecodedRune | None: + """Decode a valid multi-byte UTF-8 prefix or Go's replacement rune.""" + available_bytes = end - start + if available_bytes < 2: + return DecodedRune(RUNE_ERROR, 1) if final else None + + second_byte = content[start + 1] + if second_byte < second_byte_minimum or second_byte > second_byte_maximum: + return DecodedRune(RUNE_ERROR, 1) + + for byte_offset in range(2, byte_count): + if available_bytes <= byte_offset: + return DecodedRune(RUNE_ERROR, 1) if final else None + continuation_byte = content[start + byte_offset] + if continuation_byte < 0x80 or continuation_byte > 0xBF: + return DecodedRune(RUNE_ERROR, 1) + + first_byte = content[start] + if byte_count == 2: + codepoint = ((first_byte & 0x1F) << 6) | (second_byte & 0x3F) + elif byte_count == 3: + third_byte = content[start + 2] + codepoint = ( + ((first_byte & 0x0F) << 12) + | ((second_byte & 0x3F) << 6) + | (third_byte & 0x3F) + ) + else: + third_byte = content[start + 2] + fourth_byte = content[start + 3] + codepoint = ( + ((first_byte & 0x07) << 18) + | ((second_byte & 0x3F) << 12) + | ((third_byte & 0x3F) << 6) + | (fourth_byte & 0x3F) + ) + return DecodedRune(codepoint, byte_count) + + +def add_trigram_codepoint( + trigrams: set[int], first_previous: int, second_previous: int, codepoint: int +) -> tuple[int, int]: + """Shift one rune into the current trigram window.""" + if first_previous != 0: + trigrams.add((first_previous << 42) | (second_previous << 21) | codepoint) + return second_previous, codepoint + + +def count_buffer_trigrams( + content: bytes, + *, + final: bool, + trigrams: set[int], + first_previous: int, + second_previous: int, +) -> tuple[bytes, int, int]: + """Count all complete runes in content and return leftover bytes.""" + offset = 0 + content_length = len(content) + while offset < content_length: + decoded_rune = decode_go_utf8_prefix( + content, offset, content_length, final=final + ) + if decoded_rune is None: + break + first_previous, second_previous = add_trigram_codepoint( + trigrams, + first_previous, + second_previous, + decoded_rune.codepoint, + ) + offset += decoded_rune.byte_count + + return content[offset:], first_previous, second_previous + + +def count_bytes_trigrams(content: bytes) -> int: + """Count unique Zoekt-style three-rune trigrams in a byte string.""" + trigrams: set[int] = set() + leftover_bytes, first_previous, second_previous = count_buffer_trigrams( + content, + final=True, + trigrams=trigrams, + first_previous=0, + second_previous=0, + ) + if leftover_bytes: + raise RuntimeError("final trigram count left undecoded bytes") + return len(trigrams) + + +def process_regular_file(file_path: Path, byte_size: int) -> FileProcessingResult: + """Count trigrams in a regular file without loading it all into memory.""" + trigrams: set[int] = set() + leftover_bytes = b"" + first_previous = 0 + second_previous = 0 + + try: + with file_path.open("rb") as file_handle: + while True: + chunk = file_handle.read(READ_CHUNK_SIZE) + if not chunk: + break + if b"\0" in chunk: + return SkippedFile(str(file_path), "binary") + + content = leftover_bytes + chunk + leftover_bytes, first_previous, second_previous = count_buffer_trigrams( + content, + final=False, + trigrams=trigrams, + first_previous=first_previous, + second_previous=second_previous, + ) + + if leftover_bytes: + leftover_bytes, first_previous, second_previous = count_buffer_trigrams( + leftover_bytes, + final=True, + trigrams=trigrams, + first_previous=first_previous, + second_previous=second_previous, + ) + if leftover_bytes: + return SkippedFile(str(file_path), "decode_error") + except OSError as error: + return SkippedFile(str(file_path), "read_error", str(error)) + + return CountedFile(str(file_path), byte_size, len(trigrams)) + + +def process_file(file_path_text: str) -> FileProcessingResult: + """Count one regular file or Git-style symlink blob.""" + file_path = Path(file_path_text) + try: + file_status = file_path.lstat() + except OSError as error: + return SkippedFile(file_path_text, "stat_error", str(error)) + + if stat.S_ISLNK(file_status.st_mode): + try: + content = os.fsencode(os.readlink(file_path)) + except OSError as error: + return SkippedFile(file_path_text, "readlink_error", str(error)) + return CountedFile(file_path_text, len(content), count_bytes_trigrams(content)) + + if not stat.S_ISREG(file_status.st_mode): + return SkippedFile(file_path_text, "not_regular_file") + + return process_regular_file(file_path, file_status.st_size) + + +def process_files( + candidate_files: list[Path], *, workers: int, progress_every: int +) -> tuple[list[CountedFile], dict[str, int], list[SkippedFile]]: + """Count files and summarize skipped files.""" + counted_files: list[CountedFile] = [] + skipped_counts: dict[str, int] = {} + skipped_examples: list[SkippedFile] = [] + path_texts = [str(file_path) for file_path in candidate_files] + + if workers == 1: + processing_results = map(process_file, path_texts) + counted_files, skipped_counts, skipped_examples = collect_processing_results( + processing_results, + total_files=len(path_texts), + progress_every=progress_every, + ) + else: + with concurrent.futures.ProcessPoolExecutor(max_workers=workers) as executor: + processing_results = executor.map(process_file, path_texts, chunksize=16) + counted_files, skipped_counts, skipped_examples = ( + collect_processing_results( + processing_results, + total_files=len(path_texts), + progress_every=progress_every, + ) + ) + + return counted_files, skipped_counts, skipped_examples + + +def collect_processing_results( + processing_results: Iterator[FileProcessingResult], + *, + total_files: int, + progress_every: int, +) -> tuple[list[CountedFile], dict[str, int], list[SkippedFile]]: + """Collect worker results and print progress.""" + counted_files: list[CountedFile] = [] + skipped_counts: dict[str, int] = {} + skipped_examples: list[SkippedFile] = [] + + for processed_count, processing_result in enumerate(processing_results, start=1): + if isinstance(processing_result, CountedFile): + counted_files.append(processing_result) + else: + skipped_counts[processing_result.reason] = ( + skipped_counts.get(processing_result.reason, 0) + 1 + ) + if len(skipped_examples) < 10: + skipped_examples.append(processing_result) + + if progress_every and processed_count % progress_every == 0: + print( + f"processed {processed_count:,}/{total_files:,} files...", + file=sys.stderr, + ) + + return counted_files, skipped_counts, skipped_examples + + +def sort_counted_files(counted_files: list[CountedFile], sort: str) -> None: + """Sort counted files in place.""" + if sort == "path": + counted_files.sort(key=lambda counted_file: counted_file.path) + else: + counted_files.sort( + key=lambda counted_file: (-counted_file.unique_trigrams, counted_file.path) + ) + + +def write_tab_separated_values( + output_path: Path, + counted_files: list[CountedFile], + *, + threshold: int, +) -> None: + """Write counted files to TSV.""" + if str(output_path) == "-": + write_tab_separated_values_to_file( + sys.stdout, counted_files, threshold=threshold + ) + return + + resolved_output_path = output_path.expanduser() + resolved_output_path.parent.mkdir(parents=True, exist_ok=True) + with resolved_output_path.open("w", encoding="utf-8", newline="") as file_handle: + write_tab_separated_values_to_file( + file_handle, counted_files, threshold=threshold + ) + + +def write_tab_separated_values_to_file( + file_handle: TextIO, + counted_files: list[CountedFile], + *, + threshold: int, +) -> None: + """Write TSV rows to an open file handle.""" + writer = csv.writer(file_handle, delimiter="\t", lineterminator="\n") + writer.writerow( + [ + "unique_trigrams", + "would_skip_too_many_trigrams", + "byte_size", + "path", + ] + ) + for counted_file in counted_files: + writer.writerow( + [ + counted_file.unique_trigrams, + str(counted_file.unique_trigrams > threshold).lower(), + counted_file.byte_size, + counted_file.path, + ] + ) + + +def print_summary( + *, + output_path: Path, + discovered_files: DiscoveredFiles, + candidate_count: int, + suffix_duplicate_count: int, + counted_files: list[CountedFile], + skipped_counts: dict[str, int], + skipped_examples: list[SkippedFile], + threshold: int, +) -> None: + """Print a short run summary to stderr.""" + too_many_trigrams_count = sum( + 1 for counted_file in counted_files if counted_file.unique_trigrams > threshold + ) + print(f"Git worktrees: {len(discovered_files.git_worktrees):,}", file=sys.stderr) + print( + f"Plain non-Git files: {len(discovered_files.plain_files):,}", file=sys.stderr + ) + print( + f"Candidate files before suffix de-dupe: {candidate_count:,}", file=sys.stderr + ) + print( + f"Skipped duplicate path suffixes: {suffix_duplicate_count:,}", file=sys.stderr + ) + print(f"Counted text files: {len(counted_files):,}", file=sys.stderr) + print( + f"Files over {threshold:,} unique trigrams: {too_many_trigrams_count:,}", + file=sys.stderr, + ) + for reason, count in sorted(skipped_counts.items()): + print(f"Skipped {reason}: {count:,}", file=sys.stderr) + for skipped_file in skipped_examples: + detail = f": {skipped_file.detail}" if skipped_file.detail else "" + print( + f"example skipped {skipped_file.reason}: {skipped_file.path}{detail}", + file=sys.stderr, + ) + if str(output_path) != "-": + print(f"Wrote {output_path.expanduser()}", file=sys.stderr) + + +def run(parsed_arguments: argparse.Namespace) -> None: + """Run the trigram count command.""" + root = resolve_scan_root(parsed_arguments.root) + discovered_files = discover_files(root) + candidate_files = collect_candidate_files(discovered_files, root) + candidate_files = exclude_output_file(candidate_files, parsed_arguments.output) + candidate_count = len(candidate_files) + candidate_files, suffix_duplicate_count = dedupe_by_path_suffix( + candidate_files, + parsed_arguments.dedupe_path_suffix_length, + ) + counted_files, skipped_counts, skipped_examples = process_files( + candidate_files, + workers=parsed_arguments.workers, + progress_every=parsed_arguments.progress_every, + ) + sort_counted_files(counted_files, parsed_arguments.sort) + write_tab_separated_values( + parsed_arguments.output, + counted_files, + threshold=parsed_arguments.threshold, + ) + print_summary( + output_path=parsed_arguments.output, + discovered_files=discovered_files, + candidate_count=candidate_count, + suffix_duplicate_count=suffix_duplicate_count, + counted_files=counted_files, + skipped_counts=skipped_counts, + skipped_examples=skipped_examples, + threshold=parsed_arguments.threshold, + ) + + +def main(argument_values: Sequence[str] | None = None) -> None: + """Command-line entry point.""" + run(parse_arguments(sys.argv[1:] if argument_values is None else argument_values)) + + +if __name__ == "__main__": + main()