diff --git a/tests/test_public_intake.py b/tests/test_public_intake.py index d531e2816..4730501aa 100644 --- a/tests/test_public_intake.py +++ b/tests/test_public_intake.py @@ -472,6 +472,45 @@ def test_orphan_branch_404_alone_is_not_reachability(self): self.api.routes[f'/repos/{REPO}/compare/{orphan}...{"d" * 40}?per_page=1'] = urllib.error.HTTPError('u', 404, 'Not Found', None, None) self.reject('not reachable from any branch') + def test_repeated_source_commit_is_scanned_once(self): + # Three sources pin one commit that only a late branch contains. Its reachability scan and + # its tree are fetched once, not once per source (main#386 timed out on the repeats). + commit, vouching = 'd' * 40, 'claude/recovery-real' + files = {'research/a.json': b'a\n', 'research/b.json': b'b\n', 'research/c.json': b'c\n'} + self.manifest['sources'] = [{'repository': REPO, 'path': p, 'commit': commit, 'sha256': sha(raw)} for p, raw in files.items()] + self.save() + for p, raw in files.items(): + self.install_source(REPO, commit, p, raw, branches=('main', 'chatgpt/one', 'chatgpt/two', vouching), reachable_from=(vouching,)) + self.api.routes[f'/repos/{REPO}/git/trees/{commit}?recursive=1'] = {'truncated': False, 'tree': [ + {'path': p, 'mode': '100644', 'type': 'blob', 'size': len(raw), 'sha': blob(raw)} for p, raw in files.items()]} + self.assertEqual(self.run_check()['verified_sources'], 3) + self.assertEqual(len([c for c in self.api.calls if f'...{commit}?per_page=1' in c]), 4) + self.assertEqual(len([c for c in self.api.calls if c == f'/repos/{REPO}/branches?per_page=100&page=1']), 1) + self.assertEqual(len([c for c in self.api.calls if c == f'/repos/{REPO}/git/trees/{commit}?recursive=1']), 1) + + def test_repeated_unreachable_source_commit_still_rejects(self): + # Negative control: reuse never creates reachability; a repeated unreachable commit is refused. + commit = 'd' * 40 + files = {'research/a.json': b'a\n', 'research/b.json': b'b\n'} + self.manifest['sources'] = [{'repository': REPO, 'path': p, 'commit': commit, 'sha256': sha(raw)} for p, raw in files.items()] + self.save() + for p, raw in files.items(): + self.install_source(REPO, commit, p, raw, branches=('main', 'chatgpt/one'), reachable_from=()) + self.reject('not reachable from any branch') + + def test_reachable_commit_never_vouches_for_another_commit(self): + # Negative control: the reuse is keyed by commit. A reachable first source must not let a + # second source at an unreachable commit of the same repository through. + good, bad, vouching = 'd' * 40, 'e' * 40, 'claude/recovery-real' + good_raw, bad_raw = b'reachable\n', b'unreachable\n' + self.manifest['sources'] = [{'repository': REPO, 'path': 'research/good.json', 'commit': good, 'sha256': sha(good_raw)}, + {'repository': REPO, 'path': 'research/bad.json', 'commit': bad, 'sha256': sha(bad_raw)}] + self.save() + self.install_source(REPO, good, 'research/good.json', good_raw, branches=('main', vouching), reachable_from=(vouching,)) + self.install_source(REPO, bad, 'research/bad.json', bad_raw, branches=('main', vouching), reachable_from=()) + self.reject('not reachable from any branch') + self.assertFalse(any(f'/git/trees/{bad}' in c for c in self.api.calls)) + def test_compare_non_404_http_error_still_rejects(self): # Negative control: only 404 means "this branch does not vouch"; a 403 or 500 is not evidence and rejects. route = f'/repos/{MATH}/compare/main...{SOURCE}?per_page=1' diff --git a/tools/public_intake_check.py b/tools/public_intake_check.py index 0bdfe5063..b96674d4d 100644 --- a/tools/public_intake_check.py +++ b/tools/public_intake_check.py @@ -340,6 +340,11 @@ def verify_package(api, repo, head, base, rows): sources = identity['sources'] require(isinstance(sources, list) and 1 <= len(sources) <= 8, 'one to eight pinned sources required') seen_sources = set() + # Several sources may pin the same commit. Its reachability scan (up to one compare per + # listed branch) and its recursive tree are the same for each of them, so each is fetched + # once per (repository, commit) within this run. The key includes the commit, so one + # commit's answer is never reused for another. + reachability, source_trees = {}, {} for row in sources: require(isinstance(row, dict) and set(row) == {'repository', 'path', 'commit', 'sha256'}, 'invalid source record') source_repo = row['repository'] @@ -355,11 +360,14 @@ def verify_package(api, repo, head, base, rows): require(isinstance(metadata, dict) and metadata.get('private') is False and metadata.get('full_name') == source_repo, 'source is not verified public') commit_record = api.get(f'/repos/{source_repo}/git/commits/{commit}') require(isinstance(commit_record, dict) and commit_record.get('sha') == commit, 'source is not an exact commit') - require(reachable(api, source_repo, commit, metadata.get('default_branch')), - 'source commit is not reachable from any branch of the public repository') + if (source_repo, commit) not in reachability: + reachability[(source_repo, commit)] = reachable(api, source_repo, commit, metadata.get('default_branch')) + require(reachability[(source_repo, commit)], 'source commit is not reachable from any branch of the public repository') # The contents endpoint follows in-repository symlinks. Resolve the # exact path from the commit tree instead, then read that exact blob. - source_tree = tree(api, source_repo, commit) + if (source_repo, commit) not in source_trees: + source_trees[(source_repo, commit)] = tree(api, source_repo, commit) + source_tree = source_trees[(source_repo, commit)] source_entry = source_tree.get(path) require(isinstance(source_entry, dict) and source_entry.get('type') == 'blob' and source_entry.get('mode') == '100644', 'source must be a regular non-executable public file')