From e025b9ba7e90a2047c40da46862114626cb27d86 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 17:29:04 +0000 Subject: [PATCH 1/2] Refuse intake sources that no branch of the pillar reaches Defect: verify_package accepted any source commit that resolved through /git/commits, /git/trees and /git/blobs. Those routes serve every object in the repository's object store, including ancestors of refs/pull/N/head and fork-network objects, so an outsider could cite a commit from their own fork (or the PR head itself) as a "verified public source" and have the checker read back bytes they chose. Fix: after the exact-commit check, require the commit to be an ancestor of a branch of the pillar repository. ancestor_of_branch() asks /repos/{repo}/compare/{branch}...{commit}?per_page=1 and accepts only a dict with ahead_by == 0 (int, not bool), status in {behind, identical} and merge_base_commit.sha == commit; any other shape is not reachability. reachable() tries the default branch from the metadata already fetched, then at most MAX_BRANCH_PAGES = 3 pages of 100 from /repos/{repo}/branches. A branch name is validated before use (BRANCH regex: no spaces, control characters, leading '-', '.' or '/', or characters outside [A-Za-z0-9_./-]; '..' refused separately). A name the route cannot carry, including an unusable default_branch, is skipped and never sent; only a positive compare against a validated branch vouches. The refusal reads 'source commit is not reachable from any branch of the public repository'. Sources reachable only from a non-default branch stay accepted. Controls (tests/test_public_intake.py, 46 tests, normal and -O): fork-head-ancestor fabrication refused before any tree or blob read; self-citation of the PR head refused; a source on a research branch only is accepted and a short page ends enumeration; ahead, diverged and ahead_by 1 refused; malformed compare (non-dict, empty, missing or wrong or non-dict merge base, bool or string ahead_by, missing status) refused; non-list branch listing refused; unsafe default_branch (twelve shapes) skipped, the branch list consulted, the unsafe name never in a route; enumeration stops at page 3 with exactly 300 compares. Weakening proof: deleting the new require() on a scratch copy fails 32 checks including both fabrication controls. Reviewer probe suite identity-schema/probe_reachability.py: green on the old checker (the defect), 4/4 with EXPECT=red on this one. Live read-only probe on the main pillar: documented research snapshot 7caac254 accepted after 10 calls; tip and root commits accepted after 1 call each. API budget per source: common case (commit on the default branch) 5 calls, up from 4: metadata, compare, commit, tree, blob; asserted by test_reachable_default_branch_costs_one_compare_call. Worst case: 1 default compare + 3 branch pages + up to 299 further compares (300 listed names minus the default, which is not retried) = 303 reachability calls, 305 per source before a refusal, 307 when the last listed branch vouches; at most 8 sources per package. An ancestor compare carries no file diff (about 10 KB observed); a diverged one carries up to 300 files and stays under MAX_RESPONSE_BYTES, whose breach is already a refusal. Scientific effect NONE: an engineering gate; no status word changes. Same-provider authorship and review; zero independence credit. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Fij9F1cYrAUDW7PhWQze1G --- tests/test_public_intake.py | 130 ++++++++++++++++++++++++++++++++--- tools/public_intake_check.py | 32 +++++++++ 2 files changed, 153 insertions(+), 9 deletions(-) diff --git a/tests/test_public_intake.py b/tests/test_public_intake.py index 62270eb32..c3e5d3d98 100644 --- a/tests/test_public_intake.py +++ b/tests/test_public_intake.py @@ -20,6 +20,8 @@ HEAD = 'a' * 40 BASE = 'b' * 40 SOURCE = 'c' * 40 +PARENT = 'f' * 40 # an ancestor of the PR head that no branch of the pillar contains +MATH = 'd6g8k5htny-coder/Math-' def sha(raw): @@ -30,6 +32,11 @@ def blob(raw): return hashlib.sha1(b'blob ' + str(len(raw)).encode() + b'\0' + raw).hexdigest() +def compare(commit, status='behind', ahead_by=0, merge_base=None): + return {'status': status, 'ahead_by': ahead_by, 'behind_by': 3, + 'merge_base_commit': {'sha': commit if merge_base is None else merge_base}} + + class FakeAPI: def __init__(self): self.routes = {} @@ -83,15 +90,26 @@ def install(self): self.api.routes[f'/repos/{REPO}/git/trees/{HEAD}?recursive=1'] = {'truncated': False, 'tree': entries} self.api.routes[f'/repos/{REPO}/git/trees/{BASE}?recursive=1'] = {'truncated': False, 'tree': []} sr = self.manifest['sources'][0] - source_repo = sr['repository'] - self.api.routes[f'/repos/{source_repo}'] = {'private': False, 'full_name': source_repo} - self.api.routes[f'/repos/{source_repo}/git/commits/{sr["commit"]}'] = {'sha': sr['commit']} - self.api.routes[f'/repos/{source_repo}/git/trees/{sr["commit"]}?recursive=1'] = {'truncated': False, 'tree': [ - {'path': sr['path'], 'mode': '100644', 'type': 'blob', 'size': len(self.source), 'sha': blob(self.source)}]} - self.api.routes[f'/repos/{source_repo}/git/blobs/{blob(self.source)}'] = { - 'encoding': 'base64', 'size': len(self.source), 'content': base64.b64encode(self.source).decode(), 'sha': blob(self.source)} - self.api.routes[f'/repos/{source_repo}/contents/{sr["path"]}?ref={sr["commit"]}'] = { - 'type': 'file', 'encoding': 'base64', 'size': len(self.source), 'content': base64.b64encode(self.source).decode(), 'sha': blob(self.source)} + if sr['commit'] not in (HEAD, BASE): # a test citing the PR's own commits sets its routes itself + self.install_source(sr['repository'], sr['commit'], sr['path'], self.source) + + def install_source(self, source_repo, commit, path, raw, branches=('main',), reachable_from=('main',)): + # What GitHub serves for any commit object in the repository network, plus the + # compare/branches routes that decide whether a branch of the pillar contains it. + self.api.routes[f'/repos/{source_repo}'] = {'private': False, 'full_name': source_repo, 'default_branch': 'main'} + self.api.routes[f'/repos/{source_repo}/git/commits/{commit}'] = {'sha': commit} + self.api.routes[f'/repos/{source_repo}/git/trees/{commit}?recursive=1'] = {'truncated': False, 'tree': [ + {'path': path, 'mode': '100644', 'type': 'blob', 'size': len(raw), 'sha': blob(raw)}]} + self.api.routes[f'/repos/{source_repo}/git/blobs/{blob(raw)}'] = { + 'encoding': 'base64', 'size': len(raw), 'content': base64.b64encode(raw).decode(), 'sha': blob(raw)} + self.api.routes[f'/repos/{source_repo}/contents/{path}?ref={commit}'] = { + 'type': 'file', 'encoding': 'base64', 'size': len(raw), 'content': base64.b64encode(raw).decode(), 'sha': blob(raw)} + self.api.routes[f'/repos/{source_repo}/branches?per_page=100&page=1'] = [{'name': b, 'commit': {'sha': 'd' * 40}} for b in branches] + for b in branches: + # Not contained: GitHub reports the object as diverged from the branch + # ("this commit does not belong to any branch on this repository"). + self.api.routes[f'/repos/{source_repo}/compare/{b}...{commit}?per_page=1'] = ( + compare(commit) if b in reachable_from else compare(commit, 'diverged', 2, BASE)) def run_check(self): return intake.check(self.event, self.api, REPO) @@ -364,6 +382,100 @@ def test_known_credential_in_package_path_is_refused_without_logging(self): self.assertIn('possible credential in path', output.getvalue()) self.assertNotIn(token, output.getvalue()) + def reachability_calls(self): + return [c for c in self.api.calls if '/compare/' in c or '/branches?' in c] + + def test_fabricated_source_in_fork_head_ancestor_refused(self): + # refs/pull/7/head brings the submitter's own PARENT commit into the pillar's object + # store, so /git/commits, /git/trees and /git/blobs resolve it although no branch has it. + forged = b'{"enclosure": [0, 0], "note": "bytes chosen by the submitter"}\n' + self.manifest['sources'] = [{'repository': REPO, 'path': 'coefficients/side24_v1/ENCLOSURE.json', + 'commit': PARENT, 'sha256': sha(forged)}] + self.save() + self.install_source(REPO, PARENT, 'coefficients/side24_v1/ENCLOSURE.json', forged, reachable_from=()) + self.reject('not reachable from any branch') + self.assertFalse(any(f'/git/trees/{PARENT}' in c or f'/git/blobs/{blob(forged)}' in c for c in self.api.calls)) + + def test_self_citation_of_pr_head_refused(self): + # HEAD's tree and blobs are served already because the checker reads the package from them. + own = self.raw['output.json'] + self.manifest['sources'] = [{'repository': REPO, 'path': self.package + 'output.json', 'commit': HEAD, 'sha256': sha(own)}] + self.save() + self.api.routes[f'/repos/{REPO}'] = {'private': False, 'full_name': REPO, 'default_branch': 'main'} + self.api.routes[f'/repos/{REPO}/git/commits/{HEAD}'] = {'sha': HEAD} + self.api.routes[f'/repos/{REPO}/branches?per_page=100&page=1'] = [{'name': 'main', 'commit': {'sha': BASE}}] + self.api.routes[f'/repos/{REPO}/compare/main...{HEAD}?per_page=1'] = compare(HEAD, 'ahead', 1, BASE) + self.reject('not reachable from any branch') + + def test_source_reachable_only_from_research_branch_accepted(self): + raw = b'public numbers on a research branch\n' + branch = 'chatgpt/drive-github-hardening-20260919' + self.manifest['sources'] = [{'repository': REPO, 'path': 'research/numbers.json', 'commit': 'd' * 40, 'sha256': sha(raw)}] + self.save() + self.install_source(REPO, 'd' * 40, 'research/numbers.json', raw, branches=('main', branch), reachable_from=(branch,)) + self.assertEqual(self.run_check()['verified_sources'], 1) + self.assertIn(f'/repos/{REPO}/compare/{branch}...{"d" * 40}?per_page=1', self.api.calls) + self.assertFalse(any('page=2' in c for c in self.api.calls)) # a short page ends the enumeration + + def test_reachable_default_branch_costs_one_compare_call(self): + self.assertEqual(self.run_check()['verified_sources'], 1) + self.assertEqual(self.reachability_calls(), [f'/repos/{MATH}/compare/main...{SOURCE}?per_page=1']) + self.assertEqual(len([c for c in self.api.calls if MATH in c]), 5) + + def test_compare_ahead_diverged_or_nonzero_ahead_by_refused(self): + route = f'/repos/{MATH}/compare/main...{SOURCE}?per_page=1' + for status, ahead_by, merge_base in [('ahead', 1, BASE), ('diverged', 2, BASE), ('behind', 1, SOURCE), + ('identical', 1, SOURCE), ('ahead', 0, SOURCE), ('unknown', 0, SOURCE)]: + with self.subTest(status=status, ahead_by=ahead_by): + self.api.routes[route] = compare(SOURCE, status, ahead_by, merge_base) + self.reject('not reachable from any branch') + + def test_malformed_compare_response_is_never_reachability(self): + route = f'/repos/{MATH}/compare/main...{SOURCE}?per_page=1' + good = compare(SOURCE) + shapes = [('non-dict', [good]), ('empty', {}), ('missing merge base', {k: v for k, v in good.items() if k != 'merge_base_commit'}), + ('merge base not a dict', dict(good, merge_base_commit=SOURCE)), ('wrong merge base', compare(SOURCE, merge_base=BASE)), + ('bool ahead_by', dict(good, ahead_by=False)), ('string ahead_by', dict(good, ahead_by='0')), + ('missing status', {k: v for k, v in good.items() if k != 'status'})] + for label, data in shapes: + with self.subTest(label=label): + self.api.routes[route] = data + self.reject('not reachable from any branch') + self.api.routes[route] = good + self.api.routes[f'/repos/{MATH}/branches?per_page=100&page=1'] = {'name': 'main'} + self.api.routes[route] = compare(SOURCE, 'diverged', 2, BASE) + self.reject('branch listing unavailable') + + def test_unsafe_default_branch_is_skipped_and_branch_list_consulted(self): + main_compare = f'/repos/{MATH}/compare/main...{SOURCE}?per_page=1' + for name in ['main..evil', 'ma in', '-flag', 'x\ny', 'tab\t', '.hidden', '/main', 'main?x=1', 'main#1', None, 5, ['main']]: + with self.subTest(name=repr(name)): + self.api.routes[f'/repos/{MATH}']['default_branch'] = name + self.api.routes[f'/repos/{MATH}/branches?per_page=100&page=1'] = [{'name': name}, {'name': 'main'}] + self.api.calls.clear() + self.assertEqual(self.run_check()['verified_sources'], 1) + self.assertEqual(self.reachability_calls(), [f'/repos/{MATH}/branches?per_page=100&page=1', main_compare]) + self.api.routes[f'/repos/{MATH}/branches?per_page=100&page=1'] = [{'name': name}, 'main', {'commit': {}}] + self.api.calls.clear() + self.reject('not reachable from any branch') + self.assertEqual([c for c in self.api.calls if '/compare/' in c], []) + + def test_branch_enumeration_stops_at_page_bound(self): + self.api.routes[f'/repos/{MATH}/compare/main...{SOURCE}?per_page=1'] = compare(SOURCE, 'diverged', 2, BASE) + names = ['main'] + [f'topic-{i}' for i in range(1, 300)] + for page in range(1, 4): + self.api.routes[f'/repos/{MATH}/branches?per_page=100&page={page}'] = [{'name': n} for n in names[(page - 1) * 100:page * 100]] + for n in names[1:]: + self.api.routes[f'/repos/{MATH}/compare/{n}...{SOURCE}?per_page=1'] = compare(SOURCE, 'diverged', 2, BASE) + # A fourth page would vouch; FakeAPI raises AssertionError if it is ever requested. + self.api.routes[f'/repos/{MATH}/branches?per_page=100&page=4'] = [{'name': 'vouching'}] + self.api.routes[f'/repos/{MATH}/compare/vouching...{SOURCE}?per_page=1'] = compare(SOURCE) + self.reject('not reachable from any branch') + self.assertEqual([c for c in self.api.calls if '/branches?' in c], + [f'/repos/{MATH}/branches?per_page=100&page={p}' for p in (1, 2, 3)]) + self.assertEqual(len([c for c in self.api.calls if '/compare/' in c]), 300) + self.assertEqual(len([c for c in self.api.calls if '/compare/main...' in c]), 1) + if __name__ == '__main__': unittest.main() diff --git a/tools/public_intake_check.py b/tools/public_intake_check.py index 100efb345..73127bf0d 100644 --- a/tools/public_intake_check.py +++ b/tools/public_intake_check.py @@ -26,6 +26,8 @@ HEX64 = re.compile(r'[0-9a-f]{64}\Z') REPOSITORY = re.compile(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+\Z') SAFE_PATH = re.compile(r'[A-Za-z0-9_][A-Za-z0-9_. /-]*\Z') +BRANCH = re.compile(r'[A-Za-z0-9_][A-Za-z0-9_./-]*\Z') # a name this route can carry; '..' refused below +MAX_BRANCH_PAGES = 3 CREDENTIALS = [ re.compile(rb'-----BEGIN (?:RSA |EC |OPENSSH |DSA )?PRIVATE KEY-----'), re.compile(rb'\bgh[pousr]_[A-Za-z0-9]{36,}\b'), @@ -122,6 +124,34 @@ def tree(api, repo, ref): return rows +def ancestor_of_branch(api, repo, branch, commit): + if not (isinstance(branch, str) and BRANCH.fullmatch(branch) and '..' not in branch): + return False # a name this route cannot carry safely never vouches for a source + data = api.get(f'/repos/{repo}/compare/{branch}...{commit}?per_page=1') + if not (isinstance(data, dict) and isinstance(data.get('merge_base_commit'), dict)): + return False # any other shape is not evidence of reachability + return (type(data.get('ahead_by')) is int and data['ahead_by'] == 0 and data.get('status') in ('behind', 'identical') + and data['merge_base_commit'].get('sha') == commit) + + +def reachable(api, repo, commit, default_branch): + # /git/commits, /git/trees and /git/blobs serve any object in the repository's + # store, including refs/pull/N/head of an outsider's PR and fork-network objects. + # Only a branch of the pillar, which outsiders cannot create, vouches for a source. + if ancestor_of_branch(api, repo, default_branch, commit): + return True + for page in range(1, MAX_BRANCH_PAGES + 1): + batch = api.get(f'/repos/{repo}/branches?per_page=100&page={page}') + require(isinstance(batch, list), 'branch listing unavailable') + for row in batch: + name = row.get('name') if isinstance(row, dict) else None + if name != default_branch and ancestor_of_branch(api, repo, name, commit): + return True + if len(batch) < 100: + break + return False + + def snapshot(pr, repo): require(isinstance(pr, dict) and pr.get('state') == 'open', 'PR is not open') base, head = pr.get('base'), pr.get('head') @@ -240,6 +270,8 @@ def verify_package(api, repo, head, base, rows): require(isinstance(metadata, dict) and metadata.get('private') is False and metadata.get('full_name') == source_repo, 'source is not verified public') commit_record = api.get(f'/repos/{source_repo}/git/commits/{commit}') require(isinstance(commit_record, dict) and commit_record.get('sha') == commit, 'source is not an exact commit') + require(reachable(api, source_repo, commit, metadata.get('default_branch')), + 'source commit is not reachable from any branch of the public repository') # The contents endpoint follows in-repository symlinks. Resolve the # exact path from the commit tree instead, then read that exact blob. source_tree = tree(api, source_repo, commit) From e98ee774fcbdfa1a53e1abaee830d3c51a693717 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 17:30:52 +0000 Subject: [PATCH 2/2] Validate PNG chunk structure in the intake checker Defect: a submitted .png passed once its first eight bytes were the PNG signature. A ZIP archive behind that prefix, a valid PNG with an archive appended after IEND, a chunk whose CRC does not match, and a tiny file whose IHDR claims 30000 x 30000 pixels all passed as "packaging checks passed", although CONTRIBUTING refuses archives and says not to encode a refused file inside an allowed format. Fix: png_structure() walks the chunk list without decoding a pixel and replaces the signature-only check at the same place in verify_package. It keeps 'invalid PNG signature' for a bad signature and requires: the first chunk is IHDR with length 13; every chunk type is ASCII letters and its declared length fits the remaining bytes; every chunk's CRC32 over type and data matches; width and height are each in 1..16384 and their product is at most 2**24; IEND is empty, is the last chunk, and nothing follows it; an IDAT chunk is present. Messages: 'invalid PNG chunk structure or trailing data', 'invalid PNG chunk CRC', 'PNG dimensions exceed limit'. Bytes inside a chunk's data field are not inspected; the walk is bounded by the file's 256 KiB size limit. No new API calls; the per-source budget is unchanged from the previous commit. Controls (tests/test_public_intake.py, 51 tests, normal and -O, PNG fixtures built from struct and zlib in the test file): refused - ZIP behind a PNG signature, valid PNG plus appended ZIP, ZIP inside a non-empty IEND, corrupt IEND CRC, corrupt IHDR data with stale CRC, 30000x30000, 16385x1, 1x16385, 4097x4097, 0x1, 1x0, IHDR not first, missing IDAT, missing IEND, signature only, IHDR of length 12, chunk length past the end, non-ASCII chunk type, a byte after IEND, and the existing non-PNG bytes still 'invalid PNG signature'; accepted - the minimal PNG, a PNG with pHYs, tEXt and iCCP chunks, 4096x4096 (exactly the pixel bound), and an upper-case .PNG suffix. Weakening proof: replacing png_structure with a signature-only check on a scratch copy fails 18 checks including all three archive polyglots. Reviewer probe suite content-rules/probe_content_rules.py F1: the four refusals pass against this checker and the ancillary-chunk control stays accepted. Scientific effect NONE: an engineering gate; no status word changes. Same-provider authorship and review; zero independence credit. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Fij9F1cYrAUDW7PhWQze1G --- tests/test_public_intake.py | 76 ++++++++++++++++++++++++++++++++++++ tools/public_intake_check.py | 31 ++++++++++++++- 2 files changed, 106 insertions(+), 1 deletion(-) diff --git a/tests/test_public_intake.py b/tests/test_public_intake.py index c3e5d3d98..bfae7772f 100644 --- a/tests/test_public_intake.py +++ b/tests/test_public_intake.py @@ -7,10 +7,13 @@ import contextlib import io import os +import struct import tempfile from unittest import mock from pathlib import Path import unittest +import zipfile +import zlib spec = importlib.util.spec_from_file_location('intake', Path(__file__).resolve().parents[1] / 'tools/public_intake_check.py') intake = importlib.util.module_from_spec(spec) @@ -32,6 +35,27 @@ def blob(raw): return hashlib.sha1(b'blob ' + str(len(raw)).encode() + b'\0' + raw).hexdigest() +def png_chunk(kind, data): + return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data)) + + +def png(width=1, height=1, before=(), after=(), idat=True, iend=True): + # Structurally valid 8-bit greyscale PNG built from struct and zlib; pixels are never decoded. + chunks = list(before) + [(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 0, 0, 0, 0))] + list(after) + if idat: + chunks.append((b'IDAT', zlib.compress(b''.join(b'\0' + b'\x80' * min(width, 64) for _ in range(min(height, 64)))))) + if iend: + chunks.append((b'IEND', b'')) + return b'\x89PNG\r\n\x1a\n' + b''.join(png_chunk(kind, data) for kind, data in chunks) + + +def zip_archive(): + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, 'w') as archive: + archive.writestr('run.py', 'import os\nos.system("id")\n') + return buffer.getvalue() + + def compare(commit, status='behind', ahead_by=0, merge_base=None): return {'status': status, 'ahead_by': ahead_by, 'behind_by': 3, 'merge_base_commit': {'sha': commit if merge_base is None else merge_base}} @@ -476,6 +500,58 @@ def test_branch_enumeration_stops_at_page_bound(self): self.assertEqual(len([c for c in self.api.calls if '/compare/' in c]), 300) self.assertEqual(len([c for c in self.api.calls if '/compare/main...' in c]), 1) + def refuse_png(self, raw, message, name='plot.png'): + self.raw[name] = raw + self.save() + self.reject(message) + self.raw.pop(name) + + def accept_png(self, raw, name='plot.png'): + self.raw[name] = raw + self.save() + self.assertEqual(self.run_check()['artifact_files'], len(self.raw)) + self.raw.pop(name) + + def test_png_archive_polyglots_refused(self): + with self.subTest('zip behind a PNG signature'): + self.refuse_png(b'\x89PNG\r\n\x1a\n' + zip_archive(), 'PNG chunk structure') + with self.subTest('valid PNG with an appended zip'): + self.refuse_png(png() + zip_archive(), 'PNG chunk structure or trailing data') + with self.subTest('zip carried inside a non-empty IEND'): + self.refuse_png(png(iend=False) + png_chunk(b'IEND', zip_archive()), 'PNG chunk structure or trailing data') + + def test_png_corrupt_chunk_crc_refused(self): + image = bytearray(png()) + image[-1] ^= 0xFF # the IEND CRC + self.refuse_png(bytes(image), 'PNG chunk CRC') + image = bytearray(png()) + image[16] ^= 0x01 # inside IHDR data; its stored CRC no longer matches + self.refuse_png(bytes(image), 'PNG chunk CRC') + + def test_png_dimension_bomb_refused_without_decoding(self): + for width, height in [(30000, 30000), (16385, 1), (1, 16385), (4097, 4097), (0, 1), (1, 0)]: + with self.subTest(width=width, height=height): + self.refuse_png(png(width, height), 'PNG dimensions exceed limit') + + def test_png_chunk_structure_refused(self): + phys = (b'pHYs', struct.pack('>IIB', 2835, 2835, 1)) + cases = [('IHDR not first', png(before=[phys])), ('missing IDAT', png(idat=False)), ('missing IEND', png(iend=False)), + ('signature only', b'\x89PNG\r\n\x1a\n'), ('IHDR length 12', b'\x89PNG\r\n\x1a\n' + png_chunk(b'IHDR', b'\0' * 12)), + ('chunk length past end', png()[:8] + struct.pack('>I', 1 << 20) + png()[12:]), + ('non-ASCII chunk type', png(after=[(b'\x00\x01\x02\x03', b'x')])), + ('data after IEND', png() + b'\0')] + for label, raw in cases: + with self.subTest(label=label): + self.refuse_png(raw, 'PNG chunk structure') + self.refuse_png(b'not a png', 'PNG signature') + + def test_valid_png_variants_accepted(self): + self.accept_png(png()) + self.accept_png(png(4, 3, after=[(b'pHYs', struct.pack('>IIB', 2835, 2835, 1)), (b'tEXt', b'Software\0matplotlib'), + (b'iCCP', b'sRGB\0\0' + zlib.compress(b'\0' * 32))])) + self.accept_png(png(4096, 4096)) # exactly MAX_PNG_PIXELS + self.accept_png(png(), name='plot.PNG') + if __name__ == '__main__': unittest.main() diff --git a/tools/public_intake_check.py b/tools/public_intake_check.py index 73127bf0d..06f64e597 100644 --- a/tools/public_intake_check.py +++ b/tools/public_intake_check.py @@ -13,7 +13,9 @@ import os from pathlib import Path, PurePosixPath import re +import struct import urllib.request +import zlib MAX_FILES = 500 # Below the API's 3000-file ceiling; pagination is mandatory. MAX_PACKAGE_FILES = 50 @@ -22,6 +24,9 @@ MAX_RESPONSE_BYTES = 8388608 SOURCE_REPOS = {'main', 'Math-', 'query-', 'Universal-Law-Workspace'} TEXT_SUFFIXES = {'.md', '.txt', '.json', '.csv'} +PNG_SIGNATURE = b'\x89PNG\r\n\x1a\n' +MAX_PNG_SIDE = 16384 +MAX_PNG_PIXELS = 1 << 24 HEX40 = re.compile(r'[0-9a-f]{40}\Z') HEX64 = re.compile(r'[0-9a-f]{64}\Z') REPOSITORY = re.compile(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+\Z') @@ -113,6 +118,30 @@ def unpack_blob(data, max_size, expected_sha=None): return raw +def png_structure(raw): + """Walk the chunk list without decoding pixels: IHDR first, ASCII chunk types, + a CRC per chunk, IDAT present, an empty IEND last with nothing after it, and + bounded dimensions. Bytes inside a chunk's data field are not inspected.""" + require(raw.startswith(PNG_SIGNATURE), 'invalid PNG signature') + offset, kinds = 8, [] + while offset < len(raw) and kinds[-1:] != [b'IEND']: + require(len(raw) - offset >= 12, 'invalid PNG chunk structure or trailing data') + length, kind = struct.unpack('>I4s', raw[offset:offset + 8]) + require(kind.isalpha() and length <= len(raw) - offset - 12, 'invalid PNG chunk structure or trailing data') + data = raw[offset + 8:offset + 8 + length] + (crc,) = struct.unpack('>I', raw[offset + 8 + length:offset + 12 + length]) + require(zlib.crc32(kind + data) == crc, 'invalid PNG chunk CRC') + if not kinds: + require(kind == b'IHDR' and length == 13, 'invalid PNG chunk structure or trailing data') + width, height = struct.unpack('>II', data[:8]) + require(1 <= width <= MAX_PNG_SIDE and 1 <= height <= MAX_PNG_SIDE and width * height <= MAX_PNG_PIXELS, + 'PNG dimensions exceed limit') + require(kind != b'IEND' or length == 0, 'invalid PNG chunk structure or trailing data') + kinds.append(kind) + offset += 12 + length + require(offset == len(raw) and kinds[-1:] == [b'IEND'] and b'IDAT' in kinds, 'invalid PNG chunk structure or trailing data') + + def tree(api, repo, ref): data = api.get(f'/repos/{repo}/git/trees/{ref}?recursive=1') require(isinstance(data, dict) and data.get('truncated') is False and isinstance(data.get('tree'), list), @@ -232,7 +261,7 @@ def verify_package(api, repo, head, base, rows): if suffix == '.json': strict_json(text) else: - require(raw.startswith(b'\x89PNG\r\n\x1a\n'), 'invalid PNG signature') + png_structure(raw) payload[name[len(prefix):]] = raw require('RESULT.md' in payload and payload['RESULT.md'].strip() and 'IDENTITY.json' in payload, 'RESULT.md and IDENTITY.json required') identity = strict_json(payload['IDENTITY.json'])