diff --git a/tests/test_public_intake.py b/tests/test_public_intake.py index 62270eb32..bfae7772f 100644 --- a/tests/test_public_intake.py +++ b/tests/test_public_intake.py @@ -7,10 +7,13 @@ import contextlib import io import os +import struct import tempfile from unittest import mock from pathlib import Path import unittest +import zipfile +import zlib spec = importlib.util.spec_from_file_location('intake', Path(__file__).resolve().parents[1] / 'tools/public_intake_check.py') intake = importlib.util.module_from_spec(spec) @@ -20,6 +23,8 @@ HEAD = 'a' * 40 BASE = 'b' * 40 SOURCE = 'c' * 40 +PARENT = 'f' * 40 # an ancestor of the PR head that no branch of the pillar contains +MATH = 'd6g8k5htny-coder/Math-' def sha(raw): @@ -30,6 +35,32 @@ def blob(raw): return hashlib.sha1(b'blob ' + str(len(raw)).encode() + b'\0' + raw).hexdigest() +def png_chunk(kind, data): + return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data)) + + +def png(width=1, height=1, before=(), after=(), idat=True, iend=True): + # Structurally valid 8-bit greyscale PNG built from struct and zlib; pixels are never decoded. + chunks = list(before) + [(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 0, 0, 0, 0))] + list(after) + if idat: + chunks.append((b'IDAT', zlib.compress(b''.join(b'\0' + b'\x80' * min(width, 64) for _ in range(min(height, 64)))))) + if iend: + chunks.append((b'IEND', b'')) + return b'\x89PNG\r\n\x1a\n' + b''.join(png_chunk(kind, data) for kind, data in chunks) + + +def zip_archive(): + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, 'w') as archive: + archive.writestr('run.py', 'import os\nos.system("id")\n') + return buffer.getvalue() + + +def compare(commit, status='behind', ahead_by=0, merge_base=None): + return {'status': status, 'ahead_by': ahead_by, 'behind_by': 3, + 'merge_base_commit': {'sha': commit if merge_base is None else merge_base}} + + class FakeAPI: def __init__(self): self.routes = {} @@ -83,15 +114,26 @@ def install(self): self.api.routes[f'/repos/{REPO}/git/trees/{HEAD}?recursive=1'] = {'truncated': False, 'tree': entries} self.api.routes[f'/repos/{REPO}/git/trees/{BASE}?recursive=1'] = {'truncated': False, 'tree': []} sr = self.manifest['sources'][0] - source_repo = sr['repository'] - self.api.routes[f'/repos/{source_repo}'] = {'private': False, 'full_name': source_repo} - self.api.routes[f'/repos/{source_repo}/git/commits/{sr["commit"]}'] = {'sha': sr['commit']} - self.api.routes[f'/repos/{source_repo}/git/trees/{sr["commit"]}?recursive=1'] = {'truncated': False, 'tree': [ - {'path': sr['path'], 'mode': '100644', 'type': 'blob', 'size': len(self.source), 'sha': blob(self.source)}]} - self.api.routes[f'/repos/{source_repo}/git/blobs/{blob(self.source)}'] = { - 'encoding': 'base64', 'size': len(self.source), 'content': base64.b64encode(self.source).decode(), 'sha': blob(self.source)} - self.api.routes[f'/repos/{source_repo}/contents/{sr["path"]}?ref={sr["commit"]}'] = { - 'type': 'file', 'encoding': 'base64', 'size': len(self.source), 'content': base64.b64encode(self.source).decode(), 'sha': blob(self.source)} + if sr['commit'] not in (HEAD, BASE): # a test citing the PR's own commits sets its routes itself + self.install_source(sr['repository'], sr['commit'], sr['path'], self.source) + + def install_source(self, source_repo, commit, path, raw, branches=('main',), reachable_from=('main',)): + # What GitHub serves for any commit object in the repository network, plus the + # compare/branches routes that decide whether a branch of the pillar contains it. + self.api.routes[f'/repos/{source_repo}'] = {'private': False, 'full_name': source_repo, 'default_branch': 'main'} + self.api.routes[f'/repos/{source_repo}/git/commits/{commit}'] = {'sha': commit} + self.api.routes[f'/repos/{source_repo}/git/trees/{commit}?recursive=1'] = {'truncated': False, 'tree': [ + {'path': path, 'mode': '100644', 'type': 'blob', 'size': len(raw), 'sha': blob(raw)}]} + self.api.routes[f'/repos/{source_repo}/git/blobs/{blob(raw)}'] = { + 'encoding': 'base64', 'size': len(raw), 'content': base64.b64encode(raw).decode(), 'sha': blob(raw)} + self.api.routes[f'/repos/{source_repo}/contents/{path}?ref={commit}'] = { + 'type': 'file', 'encoding': 'base64', 'size': len(raw), 'content': base64.b64encode(raw).decode(), 'sha': blob(raw)} + self.api.routes[f'/repos/{source_repo}/branches?per_page=100&page=1'] = [{'name': b, 'commit': {'sha': 'd' * 40}} for b in branches] + for b in branches: + # Not contained: GitHub reports the object as diverged from the branch + # ("this commit does not belong to any branch on this repository"). + self.api.routes[f'/repos/{source_repo}/compare/{b}...{commit}?per_page=1'] = ( + compare(commit) if b in reachable_from else compare(commit, 'diverged', 2, BASE)) def run_check(self): return intake.check(self.event, self.api, REPO) @@ -364,6 +406,152 @@ def test_known_credential_in_package_path_is_refused_without_logging(self): self.assertIn('possible credential in path', output.getvalue()) self.assertNotIn(token, output.getvalue()) + def reachability_calls(self): + return [c for c in self.api.calls if '/compare/' in c or '/branches?' in c] + + def test_fabricated_source_in_fork_head_ancestor_refused(self): + # refs/pull/7/head brings the submitter's own PARENT commit into the pillar's object + # store, so /git/commits, /git/trees and /git/blobs resolve it although no branch has it. + forged = b'{"enclosure": [0, 0], "note": "bytes chosen by the submitter"}\n' + self.manifest['sources'] = [{'repository': REPO, 'path': 'coefficients/side24_v1/ENCLOSURE.json', + 'commit': PARENT, 'sha256': sha(forged)}] + self.save() + self.install_source(REPO, PARENT, 'coefficients/side24_v1/ENCLOSURE.json', forged, reachable_from=()) + self.reject('not reachable from any branch') + self.assertFalse(any(f'/git/trees/{PARENT}' in c or f'/git/blobs/{blob(forged)}' in c for c in self.api.calls)) + + def test_self_citation_of_pr_head_refused(self): + # HEAD's tree and blobs are served already because the checker reads the package from them. + own = self.raw['output.json'] + self.manifest['sources'] = [{'repository': REPO, 'path': self.package + 'output.json', 'commit': HEAD, 'sha256': sha(own)}] + self.save() + self.api.routes[f'/repos/{REPO}'] = {'private': False, 'full_name': REPO, 'default_branch': 'main'} + self.api.routes[f'/repos/{REPO}/git/commits/{HEAD}'] = {'sha': HEAD} + self.api.routes[f'/repos/{REPO}/branches?per_page=100&page=1'] = [{'name': 'main', 'commit': {'sha': BASE}}] + self.api.routes[f'/repos/{REPO}/compare/main...{HEAD}?per_page=1'] = compare(HEAD, 'ahead', 1, BASE) + self.reject('not reachable from any branch') + + def test_source_reachable_only_from_research_branch_accepted(self): + raw = b'public numbers on a research branch\n' + branch = 'chatgpt/drive-github-hardening-20260919' + self.manifest['sources'] = [{'repository': REPO, 'path': 'research/numbers.json', 'commit': 'd' * 40, 'sha256': sha(raw)}] + self.save() + self.install_source(REPO, 'd' * 40, 'research/numbers.json', raw, branches=('main', branch), reachable_from=(branch,)) + self.assertEqual(self.run_check()['verified_sources'], 1) + self.assertIn(f'/repos/{REPO}/compare/{branch}...{"d" * 40}?per_page=1', self.api.calls) + self.assertFalse(any('page=2' in c for c in self.api.calls)) # a short page ends the enumeration + + def test_reachable_default_branch_costs_one_compare_call(self): + self.assertEqual(self.run_check()['verified_sources'], 1) + self.assertEqual(self.reachability_calls(), [f'/repos/{MATH}/compare/main...{SOURCE}?per_page=1']) + self.assertEqual(len([c for c in self.api.calls if MATH in c]), 5) + + def test_compare_ahead_diverged_or_nonzero_ahead_by_refused(self): + route = f'/repos/{MATH}/compare/main...{SOURCE}?per_page=1' + for status, ahead_by, merge_base in [('ahead', 1, BASE), ('diverged', 2, BASE), ('behind', 1, SOURCE), + ('identical', 1, SOURCE), ('ahead', 0, SOURCE), ('unknown', 0, SOURCE)]: + with self.subTest(status=status, ahead_by=ahead_by): + self.api.routes[route] = compare(SOURCE, status, ahead_by, merge_base) + self.reject('not reachable from any branch') + + def test_malformed_compare_response_is_never_reachability(self): + route = f'/repos/{MATH}/compare/main...{SOURCE}?per_page=1' + good = compare(SOURCE) + shapes = [('non-dict', [good]), ('empty', {}), ('missing merge base', {k: v for k, v in good.items() if k != 'merge_base_commit'}), + ('merge base not a dict', dict(good, merge_base_commit=SOURCE)), ('wrong merge base', compare(SOURCE, merge_base=BASE)), + ('bool ahead_by', dict(good, ahead_by=False)), ('string ahead_by', dict(good, ahead_by='0')), + ('missing status', {k: v for k, v in good.items() if k != 'status'})] + for label, data in shapes: + with self.subTest(label=label): + self.api.routes[route] = data + self.reject('not reachable from any branch') + self.api.routes[route] = good + self.api.routes[f'/repos/{MATH}/branches?per_page=100&page=1'] = {'name': 'main'} + self.api.routes[route] = compare(SOURCE, 'diverged', 2, BASE) + self.reject('branch listing unavailable') + + def test_unsafe_default_branch_is_skipped_and_branch_list_consulted(self): + main_compare = f'/repos/{MATH}/compare/main...{SOURCE}?per_page=1' + for name in ['main..evil', 'ma in', '-flag', 'x\ny', 'tab\t', '.hidden', '/main', 'main?x=1', 'main#1', None, 5, ['main']]: + with self.subTest(name=repr(name)): + self.api.routes[f'/repos/{MATH}']['default_branch'] = name + self.api.routes[f'/repos/{MATH}/branches?per_page=100&page=1'] = [{'name': name}, {'name': 'main'}] + self.api.calls.clear() + self.assertEqual(self.run_check()['verified_sources'], 1) + self.assertEqual(self.reachability_calls(), [f'/repos/{MATH}/branches?per_page=100&page=1', main_compare]) + self.api.routes[f'/repos/{MATH}/branches?per_page=100&page=1'] = [{'name': name}, 'main', {'commit': {}}] + self.api.calls.clear() + self.reject('not reachable from any branch') + self.assertEqual([c for c in self.api.calls if '/compare/' in c], []) + + def test_branch_enumeration_stops_at_page_bound(self): + self.api.routes[f'/repos/{MATH}/compare/main...{SOURCE}?per_page=1'] = compare(SOURCE, 'diverged', 2, BASE) + names = ['main'] + [f'topic-{i}' for i in range(1, 300)] + for page in range(1, 4): + self.api.routes[f'/repos/{MATH}/branches?per_page=100&page={page}'] = [{'name': n} for n in names[(page - 1) * 100:page * 100]] + for n in names[1:]: + self.api.routes[f'/repos/{MATH}/compare/{n}...{SOURCE}?per_page=1'] = compare(SOURCE, 'diverged', 2, BASE) + # A fourth page would vouch; FakeAPI raises AssertionError if it is ever requested. + self.api.routes[f'/repos/{MATH}/branches?per_page=100&page=4'] = [{'name': 'vouching'}] + self.api.routes[f'/repos/{MATH}/compare/vouching...{SOURCE}?per_page=1'] = compare(SOURCE) + self.reject('not reachable from any branch') + self.assertEqual([c for c in self.api.calls if '/branches?' in c], + [f'/repos/{MATH}/branches?per_page=100&page={p}' for p in (1, 2, 3)]) + self.assertEqual(len([c for c in self.api.calls if '/compare/' in c]), 300) + self.assertEqual(len([c for c in self.api.calls if '/compare/main...' in c]), 1) + + def refuse_png(self, raw, message, name='plot.png'): + self.raw[name] = raw + self.save() + self.reject(message) + self.raw.pop(name) + + def accept_png(self, raw, name='plot.png'): + self.raw[name] = raw + self.save() + self.assertEqual(self.run_check()['artifact_files'], len(self.raw)) + self.raw.pop(name) + + def test_png_archive_polyglots_refused(self): + with self.subTest('zip behind a PNG signature'): + self.refuse_png(b'\x89PNG\r\n\x1a\n' + zip_archive(), 'PNG chunk structure') + with self.subTest('valid PNG with an appended zip'): + self.refuse_png(png() + zip_archive(), 'PNG chunk structure or trailing data') + with self.subTest('zip carried inside a non-empty IEND'): + self.refuse_png(png(iend=False) + png_chunk(b'IEND', zip_archive()), 'PNG chunk structure or trailing data') + + def test_png_corrupt_chunk_crc_refused(self): + image = bytearray(png()) + image[-1] ^= 0xFF # the IEND CRC + self.refuse_png(bytes(image), 'PNG chunk CRC') + image = bytearray(png()) + image[16] ^= 0x01 # inside IHDR data; its stored CRC no longer matches + self.refuse_png(bytes(image), 'PNG chunk CRC') + + def test_png_dimension_bomb_refused_without_decoding(self): + for width, height in [(30000, 30000), (16385, 1), (1, 16385), (4097, 4097), (0, 1), (1, 0)]: + with self.subTest(width=width, height=height): + self.refuse_png(png(width, height), 'PNG dimensions exceed limit') + + def test_png_chunk_structure_refused(self): + phys = (b'pHYs', struct.pack('>IIB', 2835, 2835, 1)) + cases = [('IHDR not first', png(before=[phys])), ('missing IDAT', png(idat=False)), ('missing IEND', png(iend=False)), + ('signature only', b'\x89PNG\r\n\x1a\n'), ('IHDR length 12', b'\x89PNG\r\n\x1a\n' + png_chunk(b'IHDR', b'\0' * 12)), + ('chunk length past end', png()[:8] + struct.pack('>I', 1 << 20) + png()[12:]), + ('non-ASCII chunk type', png(after=[(b'\x00\x01\x02\x03', b'x')])), + ('data after IEND', png() + b'\0')] + for label, raw in cases: + with self.subTest(label=label): + self.refuse_png(raw, 'PNG chunk structure') + self.refuse_png(b'not a png', 'PNG signature') + + def test_valid_png_variants_accepted(self): + self.accept_png(png()) + self.accept_png(png(4, 3, after=[(b'pHYs', struct.pack('>IIB', 2835, 2835, 1)), (b'tEXt', b'Software\0matplotlib'), + (b'iCCP', b'sRGB\0\0' + zlib.compress(b'\0' * 32))])) + self.accept_png(png(4096, 4096)) # exactly MAX_PNG_PIXELS + self.accept_png(png(), name='plot.PNG') + if __name__ == '__main__': unittest.main() diff --git a/tools/public_intake_check.py b/tools/public_intake_check.py index 100efb345..06f64e597 100644 --- a/tools/public_intake_check.py +++ b/tools/public_intake_check.py @@ -13,7 +13,9 @@ import os from pathlib import Path, PurePosixPath import re +import struct import urllib.request +import zlib MAX_FILES = 500 # Below the API's 3000-file ceiling; pagination is mandatory. MAX_PACKAGE_FILES = 50 @@ -22,10 +24,15 @@ MAX_RESPONSE_BYTES = 8388608 SOURCE_REPOS = {'main', 'Math-', 'query-', 'Universal-Law-Workspace'} TEXT_SUFFIXES = {'.md', '.txt', '.json', '.csv'} +PNG_SIGNATURE = b'\x89PNG\r\n\x1a\n' +MAX_PNG_SIDE = 16384 +MAX_PNG_PIXELS = 1 << 24 HEX40 = re.compile(r'[0-9a-f]{40}\Z') HEX64 = re.compile(r'[0-9a-f]{64}\Z') REPOSITORY = re.compile(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+\Z') SAFE_PATH = re.compile(r'[A-Za-z0-9_][A-Za-z0-9_. /-]*\Z') +BRANCH = re.compile(r'[A-Za-z0-9_][A-Za-z0-9_./-]*\Z') # a name this route can carry; '..' refused below +MAX_BRANCH_PAGES = 3 CREDENTIALS = [ re.compile(rb'-----BEGIN (?:RSA |EC |OPENSSH |DSA )?PRIVATE KEY-----'), re.compile(rb'\bgh[pousr]_[A-Za-z0-9]{36,}\b'), @@ -111,6 +118,30 @@ def unpack_blob(data, max_size, expected_sha=None): return raw +def png_structure(raw): + """Walk the chunk list without decoding pixels: IHDR first, ASCII chunk types, + a CRC per chunk, IDAT present, an empty IEND last with nothing after it, and + bounded dimensions. Bytes inside a chunk's data field are not inspected.""" + require(raw.startswith(PNG_SIGNATURE), 'invalid PNG signature') + offset, kinds = 8, [] + while offset < len(raw) and kinds[-1:] != [b'IEND']: + require(len(raw) - offset >= 12, 'invalid PNG chunk structure or trailing data') + length, kind = struct.unpack('>I4s', raw[offset:offset + 8]) + require(kind.isalpha() and length <= len(raw) - offset - 12, 'invalid PNG chunk structure or trailing data') + data = raw[offset + 8:offset + 8 + length] + (crc,) = struct.unpack('>I', raw[offset + 8 + length:offset + 12 + length]) + require(zlib.crc32(kind + data) == crc, 'invalid PNG chunk CRC') + if not kinds: + require(kind == b'IHDR' and length == 13, 'invalid PNG chunk structure or trailing data') + width, height = struct.unpack('>II', data[:8]) + require(1 <= width <= MAX_PNG_SIDE and 1 <= height <= MAX_PNG_SIDE and width * height <= MAX_PNG_PIXELS, + 'PNG dimensions exceed limit') + require(kind != b'IEND' or length == 0, 'invalid PNG chunk structure or trailing data') + kinds.append(kind) + offset += 12 + length + require(offset == len(raw) and kinds[-1:] == [b'IEND'] and b'IDAT' in kinds, 'invalid PNG chunk structure or trailing data') + + def tree(api, repo, ref): data = api.get(f'/repos/{repo}/git/trees/{ref}?recursive=1') require(isinstance(data, dict) and data.get('truncated') is False and isinstance(data.get('tree'), list), @@ -122,6 +153,34 @@ def tree(api, repo, ref): return rows +def ancestor_of_branch(api, repo, branch, commit): + if not (isinstance(branch, str) and BRANCH.fullmatch(branch) and '..' not in branch): + return False # a name this route cannot carry safely never vouches for a source + data = api.get(f'/repos/{repo}/compare/{branch}...{commit}?per_page=1') + if not (isinstance(data, dict) and isinstance(data.get('merge_base_commit'), dict)): + return False # any other shape is not evidence of reachability + return (type(data.get('ahead_by')) is int and data['ahead_by'] == 0 and data.get('status') in ('behind', 'identical') + and data['merge_base_commit'].get('sha') == commit) + + +def reachable(api, repo, commit, default_branch): + # /git/commits, /git/trees and /git/blobs serve any object in the repository's + # store, including refs/pull/N/head of an outsider's PR and fork-network objects. + # Only a branch of the pillar, which outsiders cannot create, vouches for a source. + if ancestor_of_branch(api, repo, default_branch, commit): + return True + for page in range(1, MAX_BRANCH_PAGES + 1): + batch = api.get(f'/repos/{repo}/branches?per_page=100&page={page}') + require(isinstance(batch, list), 'branch listing unavailable') + for row in batch: + name = row.get('name') if isinstance(row, dict) else None + if name != default_branch and ancestor_of_branch(api, repo, name, commit): + return True + if len(batch) < 100: + break + return False + + def snapshot(pr, repo): require(isinstance(pr, dict) and pr.get('state') == 'open', 'PR is not open') base, head = pr.get('base'), pr.get('head') @@ -202,7 +261,7 @@ def verify_package(api, repo, head, base, rows): if suffix == '.json': strict_json(text) else: - require(raw.startswith(b'\x89PNG\r\n\x1a\n'), 'invalid PNG signature') + png_structure(raw) payload[name[len(prefix):]] = raw require('RESULT.md' in payload and payload['RESULT.md'].strip() and 'IDENTITY.json' in payload, 'RESULT.md and IDENTITY.json required') identity = strict_json(payload['IDENTITY.json']) @@ -240,6 +299,8 @@ def verify_package(api, repo, head, base, rows): require(isinstance(metadata, dict) and metadata.get('private') is False and metadata.get('full_name') == source_repo, 'source is not verified public') commit_record = api.get(f'/repos/{source_repo}/git/commits/{commit}') require(isinstance(commit_record, dict) and commit_record.get('sha') == commit, 'source is not an exact commit') + require(reachable(api, source_repo, commit, metadata.get('default_branch')), + 'source commit is not reachable from any branch of the public repository') # The contents endpoint follows in-repository symlinks. Resolve the # exact path from the commit tree instead, then read that exact blob. source_tree = tree(api, source_repo, commit)