From 58b7000c79c157daa45f5b9d72cd3cd4849271d0 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 20:42:36 +0000 Subject: [PATCH] Scan each intake source commit once per run, not once per source verify_package() checked every pinned source on its own. When several sources pin the same commit, it repeated that commit's whole reachability scan (up to one compare per listed branch) and its recursive tree fetch for each of them. main#386 pins five sources across two commits, and with 193 branches that was about 311 compare calls, more than the job's five-minute limit (run 38462363478 was cancelled with no rejection). Within one run, each (repository, commit) is now scanned once and its tree is fetched once. The cache key includes the full verified commit, so one commit's answer is never reused for another. A failed scan still rejects at once, and non-404 errors still raise before anything is cached. Acceptance rules, the workflow and every other request are unchanged. Tests in tests/test_public_intake.py: - three sources at one commit cost one branch listing, four compares and one tree fetch (fails on the old checker: 12 compares); - a repeated unreachable commit is still refused; - a reachable commit never vouches for another commit of the same repository (fails if the cache is keyed by repository alone). The diff is byte-identical to the patch frozen at recovery 21c99d92, blob 7415354e, SHA-256 18072d70fadf27a0bfba9654039ce00ced903e51e6de7df426e10b2e1ab7bc8e. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Cz7WZybv8znP64SpPj6sWY --- tests/test_public_intake.py | 39 ++++++++++++++++++++++++++++++++++++ tools/public_intake_check.py | 14 ++++++++++--- 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/tests/test_public_intake.py b/tests/test_public_intake.py index d531e2816..4730501aa 100644 --- a/tests/test_public_intake.py +++ b/tests/test_public_intake.py @@ -472,6 +472,45 @@ def test_orphan_branch_404_alone_is_not_reachability(self): self.api.routes[f'/repos/{REPO}/compare/{orphan}...{"d" * 40}?per_page=1'] = urllib.error.HTTPError('u', 404, 'Not Found', None, None) self.reject('not reachable from any branch') + def test_repeated_source_commit_is_scanned_once(self): + # Three sources pin one commit that only a late branch contains. Its reachability scan and + # its tree are fetched once, not once per source (main#386 timed out on the repeats). + commit, vouching = 'd' * 40, 'claude/recovery-real' + files = {'research/a.json': b'a\n', 'research/b.json': b'b\n', 'research/c.json': b'c\n'} + self.manifest['sources'] = [{'repository': REPO, 'path': p, 'commit': commit, 'sha256': sha(raw)} for p, raw in files.items()] + self.save() + for p, raw in files.items(): + self.install_source(REPO, commit, p, raw, branches=('main', 'chatgpt/one', 'chatgpt/two', vouching), reachable_from=(vouching,)) + self.api.routes[f'/repos/{REPO}/git/trees/{commit}?recursive=1'] = {'truncated': False, 'tree': [ + {'path': p, 'mode': '100644', 'type': 'blob', 'size': len(raw), 'sha': blob(raw)} for p, raw in files.items()]} + self.assertEqual(self.run_check()['verified_sources'], 3) + self.assertEqual(len([c for c in self.api.calls if f'...{commit}?per_page=1' in c]), 4) + self.assertEqual(len([c for c in self.api.calls if c == f'/repos/{REPO}/branches?per_page=100&page=1']), 1) + self.assertEqual(len([c for c in self.api.calls if c == f'/repos/{REPO}/git/trees/{commit}?recursive=1']), 1) + + def test_repeated_unreachable_source_commit_still_rejects(self): + # Negative control: reuse never creates reachability; a repeated unreachable commit is refused. + commit = 'd' * 40 + files = {'research/a.json': b'a\n', 'research/b.json': b'b\n'} + self.manifest['sources'] = [{'repository': REPO, 'path': p, 'commit': commit, 'sha256': sha(raw)} for p, raw in files.items()] + self.save() + for p, raw in files.items(): + self.install_source(REPO, commit, p, raw, branches=('main', 'chatgpt/one'), reachable_from=()) + self.reject('not reachable from any branch') + + def test_reachable_commit_never_vouches_for_another_commit(self): + # Negative control: the reuse is keyed by commit. A reachable first source must not let a + # second source at an unreachable commit of the same repository through. + good, bad, vouching = 'd' * 40, 'e' * 40, 'claude/recovery-real' + good_raw, bad_raw = b'reachable\n', b'unreachable\n' + self.manifest['sources'] = [{'repository': REPO, 'path': 'research/good.json', 'commit': good, 'sha256': sha(good_raw)}, + {'repository': REPO, 'path': 'research/bad.json', 'commit': bad, 'sha256': sha(bad_raw)}] + self.save() + self.install_source(REPO, good, 'research/good.json', good_raw, branches=('main', vouching), reachable_from=(vouching,)) + self.install_source(REPO, bad, 'research/bad.json', bad_raw, branches=('main', vouching), reachable_from=()) + self.reject('not reachable from any branch') + self.assertFalse(any(f'/git/trees/{bad}' in c for c in self.api.calls)) + def test_compare_non_404_http_error_still_rejects(self): # Negative control: only 404 means "this branch does not vouch"; a 403 or 500 is not evidence and rejects. route = f'/repos/{MATH}/compare/main...{SOURCE}?per_page=1' diff --git a/tools/public_intake_check.py b/tools/public_intake_check.py index 0bdfe5063..b96674d4d 100644 --- a/tools/public_intake_check.py +++ b/tools/public_intake_check.py @@ -340,6 +340,11 @@ def verify_package(api, repo, head, base, rows): sources = identity['sources'] require(isinstance(sources, list) and 1 <= len(sources) <= 8, 'one to eight pinned sources required') seen_sources = set() + # Several sources may pin the same commit. Its reachability scan (up to one compare per + # listed branch) and its recursive tree are the same for each of them, so each is fetched + # once per (repository, commit) within this run. The key includes the commit, so one + # commit's answer is never reused for another. + reachability, source_trees = {}, {} for row in sources: require(isinstance(row, dict) and set(row) == {'repository', 'path', 'commit', 'sha256'}, 'invalid source record') source_repo = row['repository'] @@ -355,11 +360,14 @@ def verify_package(api, repo, head, base, rows): require(isinstance(metadata, dict) and metadata.get('private') is False and metadata.get('full_name') == source_repo, 'source is not verified public') commit_record = api.get(f'/repos/{source_repo}/git/commits/{commit}') require(isinstance(commit_record, dict) and commit_record.get('sha') == commit, 'source is not an exact commit') - require(reachable(api, source_repo, commit, metadata.get('default_branch')), - 'source commit is not reachable from any branch of the public repository') + if (source_repo, commit) not in reachability: + reachability[(source_repo, commit)] = reachable(api, source_repo, commit, metadata.get('default_branch')) + require(reachability[(source_repo, commit)], 'source commit is not reachable from any branch of the public repository') # The contents endpoint follows in-repository symlinks. Resolve the # exact path from the commit tree instead, then read that exact blob. - source_tree = tree(api, source_repo, commit) + if (source_repo, commit) not in source_trees: + source_trees[(source_repo, commit)] = tree(api, source_repo, commit) + source_tree = source_trees[(source_repo, commit)] source_entry = source_tree.get(path) require(isinstance(source_entry, dict) and source_entry.get('type') == 'blob' and source_entry.get('mode') == '100644', 'source must be a regular non-executable public file')