diff --git a/.gitattributes b/.gitattributes index 0281361..131170e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,6 @@ *.sh text eol=lf +*.repos text eol=lf +*.json text eol=lf *.py text eol=lf +*.yml text eol=lf /evidence/** -text diff --git a/.github/workflows/pinned-release.yml b/.github/workflows/pinned-release.yml new file mode 100644 index 0000000..5810959 --- /dev/null +++ b/.github/workflows/pinned-release.yml @@ -0,0 +1,44 @@ +name: Pinned rc.1 release + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + pinned-release: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install -r requirements-release.txt + - name: Regression and fault-injection tests + run: python -m unittest discover -s tests -v + - name: Fetch workspace release manifest from main + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: openAMRobot/openamrobot-manifest + ref: main + path: manifest-reference + persist-credentials: false + - name: Reject release manifest drift + run: python scripts/check_manifest_sync.py manifests/v2.0.0-rc.1.repos manifest-reference/releases/v2.0.0-rc.1.repos + - name: First independent fetch and build + run: python scripts/build_release.py --config release-config.rc1.json --input-dir input/first --output-dir dist/first --fetch + - name: Second independent fetch and build + run: python scripts/build_release.py --config release-config.rc1.json --input-dir input/second --output-dir dist/second --fetch + - name: Verify integrity and reproducibility + run: | + python scripts/validate_release.py dist/first/OpenAMRobot-v2.0.0-rc.1-source.zip + python scripts/validate_release.py dist/second/OpenAMRobot-v2.0.0-rc.1-source.zip + cmp dist/first/OpenAMRobot-v2.0.0-rc.1-source.zip dist/second/OpenAMRobot-v2.0.0-rc.1-source.zip + (cd dist/first && sha256sum -c OpenAMRobot-v2.0.0-rc.1-source.zip.sha256) + - uses: actions/upload-artifact@v4 + with: + name: OpenAMRobot-v2.0.0-rc.1-candidate + path: dist/first/ diff --git a/RC1_EVIDENCE.md b/RC1_EVIDENCE.md new file mode 100644 index 0000000..7717c89 --- /dev/null +++ b/RC1_EVIDENCE.md @@ -0,0 +1,152 @@ +# OA-05 verification + +Candidate snapshot date: 2026-10-19. +Base: `b9f4a9d2a1845ab50769693240a3050cf0f6d171`. +The PR head is recorded in the PR description after committing. + +## Regression and fault injection + +`python -m unittest discover -s tests -v`: **16 tests, 0 failures** on +Windows with Python 3.14 and Git 2.55.0.windows.5. Tests cover exact SHA fetch +arguments, fetched commit mismatch, submodule rejection, mutable refs, +duplicate keys, changed manifest bytes, changed cache bytes, fetched archive +checksum mismatch, incorrect archive commit, missing checksum coverage, +tampered output, extra output members, unsafe paths, executable/link modes, +ZIP metadata normalization, reproducible output and CLI dispatch. +The legacy v0.0.1 build and validator pass using eight fixture source archives. +Fixture tests establish packaging behavior, not component build acceptance. + +Before-change regression: ran `scripts/build_release.py` from base `b9f4a9d` +against `release-config.rc1.json` in a temporary output directory. It exits 1 +with `KeyError: 'components'`. The updated entry point accepts the same config. + +## Independent real-source builds + +Executed separately, each fetching all 14 repositories into a new temporary +Git repository and using separate source caches: + +```sh +python scripts/build_release.py --config release-config.rc1.json --input-dir input/first --output-dir dist/first --fetch +python scripts/build_release.py --config release-config.rc1.json --input-dir input/second --output-dir dist/second --fetch +python scripts/validate_release.py dist/first/OpenAMRobot-v2.0.0-rc.1-source.zip +``` + +Both builds exit 0, each verifies **1,841 files**, and byte comparison using +Python `filecmp.cmp(..., shallow=False)` passes. The bundle is 231,082,934 bytes. + +SHA-256 for both bundles: + +```text +2f87bf21a99add7fb3aa37b2965943c4ac6fa390054476b5ea5293f6500fed41 +``` + +The SHA inventory and commit provenance are inside each bundle; the external +`.zip.sha256` accompanies it. Generated ZIPs and source caches are not committed. +The new hosted workflow repeats two independent fetches/builds on Ubuntu 24.04 +and uploads the validated candidate, without publishing a GitHub Release. + +## Linux packaging check + +Ran the following with the repository mounted at `/work` in the local +`ros:jazzy-ros-base` image (image ID +`sha256:c3706ef0a0aa45413c07803cf433602f543b22e45b4855f6fca955c2d8ecc4e8`): + +```sh +python3 -m unittest discover -s tests -v +python3 scripts/build_release.py --config release-config.rc1.json --input-dir input/rc1 --output-dir dist/linux +python3 scripts/validate_release.py dist/linux/OpenAMRobot-v2.0.0-rc.1-source.zip +``` + +**16 tests, 0 failures**; build and validation exit 0. The Linux build uses +the verified source cache and verifies 1,841 files. Its bundle has the same +SHA-256 shown above as both independently fetched Windows builds. This proves +cross-platform packaging for this candidate; it is not a Linux source fetch +or ROS package compilation check. + +## Not verified + +Full Ubuntu/Jazzy workspace compilation, third-party build-time dependency +pinning, firmware builds, drivers on physical hardware and rc.1 acceptance +are outside this source-packaging verification. No hardware or safety claim +is made. Hosted CI status is reported separately from local test results. + +## Integration and provenance + +PR #9's navigation-install work and PR #8's release-model documentation were +inspected. This change preserves the legacy builder path and keeps the rc.1 +procedure in a separate document. No assertion is made about inaccessible work. +The release-local manifest will be aligned with OA-39 after review; it includes +13 workspace repositories plus the release-only governance source. + +Prepared with OpenAI Codex: implementation, candidate lock, tests, workflow +and documentation. No safety logic or third-party source was authored or +modified. Third-party source and licence notices remain in their exports; +tooling dependency and upstream licences are described in RC1_RELEASE.md. + +## Rebase validation, 2026-10-24 + +Rebased onto main `c39eda70d8cf2e4f98ae09a51afc3a0750350a11` after navigation gate PR #9 +merged as `0c9442fff722ce56567ebc106dc6ca38afcb9e3d` and release-model PR #8 +merged. Pre-rebase head: `58ce4a24e6a1b9619332a59f44217fb1fd55c70b`. +The original source pins and source checksum lock are unchanged. + +```sh +python -m unittest discover -s tests -v +python scripts/build_release.py --config release-config.rc1.json --input-dir input/rc1 --output-dir dist/rebase-20261024 +python scripts/validate_release.py dist/rebase-20261024/OpenAMRobot-v2.0.0-rc.1-source.zip +``` + +**27 tests, zero failures**, including all 11 merged navigation gate/config/ +provenance tests and 16 pinned-release tests. The real-source build from the +existing verified cache exits 0 and verifies 1,841 files. Its SHA-256 remains +`2f87bf21a99add7fb3aa37b2965943c4ac6fa390054476b5ea5293f6500fed41`. +No fresh network source fetch or physical/ROS package build is claimed by +this cached packaging run. + +Conflict resolution retains main's evidence byte-preservation attributes, +archive ignores and release-model documentation. The merged navigation +manifest, navigation-only config, verifier scripts, workflow and compact +evidence are unchanged relative to main. Only rc.1 dispatch is added to the +archive-input builder; its navigation validation remains intact. + +Fresh hosted check results on the new head are reported in PR #12. This rebase +is not a new rc.1 interfaces/package-version repin or final release selection. + +## Fresh-fetch portability correction, 2026-10-24 + +The first rebased hosted run (40104847352) failed strictly on the interfaces +source checksum. The navigation gate on the same head passed (40104847340). +The cached build above did not exercise a fresh Linux export. + +Investigation reproduced the cause: Windows `core.autocrlf=true` changed text +bytes during `git archive`. With the interfaces SHA unchanged, 59 exported +files differed only by CRLF versus LF. Git export now explicitly uses +`core.autocrlf=false` and `core.eol=lf`. The source lock was regenerated by +freshly fetching the same 14 exact SHAs; no version, component or repository +selection changed. Expected checksum checks remain strict. + +Regression command: + +```sh +python -m unittest discover -s tests -p test_pinned_release.py -k git_line_ending -v +``` + +Before the fix: 1 test fails because opposite host autocrlf defaults produce +different source hashes. After the fix: that test passes. The complete suite +then passes **28 tests, zero failures**, including all merged navigation tests. + +Regenerated the reviewed candidate lock using a new output file: + +```sh +python scripts/lock_release_sources.py --manifest manifests/v2.0.0-rc.1.repos --output release-config.rc1.lf.json --cache-dir input/rc1 --version 2.0.0-rc.1 +``` + +Confirmed all config fields except the 14 source checksums are unchanged, then +replaced the old candidate lock. The rebuilt real-source bundle verifies +1,841 files and has SHA-256: + +`2c494b1d090a3c525379d32585f483d9c685d606943657c1b64991190c11d227`. + +This supersedes the earlier Windows-derived **candidate** bundle checksum; +no published release is changed. Fresh Linux two-fetch reproducibility and +navigation checks are rerun on the final head, with results linked in PR #12. diff --git a/RC1_RELEASE.md b/RC1_RELEASE.md new file mode 100644 index 0000000..610b414 --- /dev/null +++ b/RC1_RELEASE.md @@ -0,0 +1,76 @@ +# Build the rc.1 candidate + +`release-config.rc1.json` locks a release-local snapshot in +`manifests/v2.0.0-rc.1.repos`. Every repository uses a full commit SHA. +The snapshot includes the 13 workspace repositories and a separately pinned +`governance/.github` export, preserving the legacy release's governance input. +The configuration locks both the manifest bytes and each source archive with +SHA-256. The candidate is dated 2026-10-19; final component acceptance is pending. + +## Build and verify + +Requires Python 3.10 or newer, Git and network access to the pinned repositories: + +```sh +python -m pip install -r requirements-release.txt +python -m unittest discover -s tests -v +python scripts/build_release.py --config release-config.rc1.json --input-dir input/first --output-dir dist/first --fetch +python scripts/build_release.py --config release-config.rc1.json --input-dir input/second --output-dir dist/second --fetch +python scripts/validate_release.py dist/first/OpenAMRobot-v2.0.0-rc.1-source.zip +python scripts/validate_release.py dist/second/OpenAMRobot-v2.0.0-rc.1-source.zip +cmp dist/first/OpenAMRobot-v2.0.0-rc.1-source.zip dist/second/OpenAMRobot-v2.0.0-rc.1-source.zip +(cd dist/first && sha256sum -c OpenAMRobot-v2.0.0-rc.1-source.zip.sha256) +``` + +On PowerShell, compare the two files using `Get-FileHash` instead of `cmp`. +`--fetch` uses `git fetch --depth=1 URL FULL_SHA`, verifies the fetched commit, +and exports it using `git archive`. No moving branch or main archive is used. +Omit `--fetch` to rebuild offline from the already verified source cache. +Missing cache entries or any checksum mismatch fail the build. + +Source ZIPs are normalized before hashing: fixed date, sorted files, stored +compression, normalized executable/link modes and the exact commit as ZIP +comment. Git export explicitly sets `core.autocrlf=false` and `core.eol=lf` +so host line-ending defaults cannot change source bytes; repository attributes +still apply. The final ZIP uses the same fixed metadata, retains symlinks as links, +and contains no wall-clock build timestamp or machine-specific paths. +Submodules fail explicitly until represented by separate pinned entries. + +## Outputs + +- `OpenAMRobot-v2.0.0-rc.1-source.zip` and its external `.sha256` file. +- Inside the ZIP: checkout paths from the release manifest, `release.repos`, + `MANIFEST.json` with repository URLs, SHAs and source hashes, and + `checksums.sha256` covering every other file. + +The rc.1 layout follows workspace paths rather than the numbered v0.0.1 layout. +The legacy configuration and its metadata retain their original behavior. +Validation checks every inventory entry and rejects extra or duplicate members +without extracting source symlinks. Checksums detect corruption; they are not +a cryptographic signature or proof of release approval. + +## Refresh a candidate intentionally + +Review the component SHAs first, then create a new manifest and a new lock: + +```sh +python scripts/lock_release_sources.py --manifest manifests/next-candidate.repos --output release-config.next.json --cache-dir input/next --version 2.0.0-rc.1 +``` + +Review both files in the same PR. The lock command refuses to overwrite an +existing configuration. Updating a checksum is an explicit source-selection +change, never an automatic fallback after a build failure. + +## Dependencies and limitations + +The tooling adds PyYAML 6.0.3 (MIT), from https://github.com/yaml/pyyaml. +The source snapshot includes upstream SLAMTEC sllidar_ros2 (package declares +BSD; see its retained upstream licence) and micro-ROS-Agent (Apache-2.0 with +upstream third-party notices). All original repository files and notices are +preserved by Git export; no upstream source is vendored into this builder repo. + +This validates source packaging only. Ubuntu/Jazzy package builds, driver +operation, hardware tests, compiled binary reproducibility and upstream +build-time dependency downloads are outside OA-05 verification. Final rc.1 +selection still depends on the other audit findings. No release is published +by the pull-request workflow; it uploads a candidate artifact for review. diff --git a/README.md b/README.md index 8b7c5aa..bced7ce 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +> For the SHA-pinned **v2.0.0-rc.1 candidate builder**, see [RC1_RELEASE.md](RC1_RELEASE.md). + # OpenAMRobot v0.0.1 For the pending navigation release-install gate, see diff --git a/RELEASE_PROCESS.md b/RELEASE_PROCESS.md index 084593d..e261e48 100644 --- a/RELEASE_PROCESS.md +++ b/RELEASE_PROCESS.md @@ -1,3 +1,5 @@ +> For the SHA-pinned rc.1 candidate build procedure, see [RC1_RELEASE.md](RC1_RELEASE.md). + # OpenAMRobot Release Process For navigation package pinning, clean install evidence, and candidate manifest diff --git a/STATE.md b/STATE.md new file mode 100644 index 0000000..2fa5479 --- /dev/null +++ b/STATE.md @@ -0,0 +1,42 @@ +# Release builder state + +OA-05 adds a candidate v2.0.0-rc.1 source bundle from exact Git SHAs and a +reviewed SHA-256 source lock. The 2026-10-19 snapshot is for reproducibility +validation; it is not the final release selection or acceptance evidence. + +The v0.0.1 archive-input path remains available. The rc.1 path uses the same +build and validation entry points and preserves workspace checkout paths. +Source archive checksums use canonical, uncompressed ZIPs produced from verified +Git commits. Output ZIP metadata and ordering are deterministic. + +Integrated main at `c39eda7` on 2026-10-24 after PR #9 merged navigation +installation validation and PR #8 merged the release model. The existing +navigation gate, test configuration, source provenance and compact evidence +remain intact. The archive-input builder still rejects navigation evidence, +archive, pin, package-version and contract mismatches. + +The original candidate source pins are unchanged by the rebase. Repinning to +the accepted interfaces contract/package version and final rc.1 component +selection remain separate release work. The release-local snapshot still +requires owner alignment with the workspace release manifest. + +Fresh Linux CI exposed host-dependent Git archive line endings in the old +Windows-generated source lock. Export now fixes autocrlf/eol settings; the +source lock is regenerated from the same SHAs. A regression test checks that +opposite host autocrlf defaults produce the same source hash. Checksum +enforcement remains strict; the earlier cached-bundle hash is historical. + +No safety implementation is changed. Source bundling does not validate package +builds, firmware, robot operation, physical safety or release readiness. +Upstream build-time downloads are not locked by this source bundle. + +Prepared 2026-10-28: pinned-release CI now compares the complete repository +mapping with openamrobot-manifest main's releases/v2.0.0-rc.1.repos. Only +governance/.github is exempt; missing files, missing entries and changed URLs +or SHAs fail. The 19 October source pins and checksums are unchanged. +Manifest PR #17 merged at 80161e5 and #18 merged at d9ab9db. The +workspace main now includes all 13 matching entries, including both drivers. +The historical #17-only snapshot correctly failed for both missing drivers. +On 2026-10-30 the old workflow rerun required unavailable repository admin +rights. This state update triggers fresh PR checks with the source pins, +source checksums and builder code unchanged; hosted results remain pending. diff --git a/manifests/v2.0.0-rc.1.repos b/manifests/v2.0.0-rc.1.repos new file mode 100644 index 0000000..a04c89a --- /dev/null +++ b/manifests/v2.0.0-rc.1.repos @@ -0,0 +1,58 @@ +# Candidate snapshot: 2026-10-19. Not release acceptance. +repositories: + src/openamrobot-interfaces: + type: git + url: https://github.com/openAMRobot/openamrobot-interfaces.git + version: 87ab8a936c97ab55ee74b8988ca3d7659160ea6a + src/openamrobot-comm: + type: git + url: https://github.com/openAMRobot/openamrobot-comm.git + version: 1c552921cf05001ee8f43e899aa2f92882766f07 + src/openamrobot-manipulation: + type: git + url: https://github.com/openAMRobot/openamrobot-manipulation.git + version: 095e6c7bbd270db6d21de9664a654bb37919a2eb + src/openamr-platform-sw: + type: git + url: https://github.com/openAMRobot/openamr-platform-sw.git + version: 8a7a5b136bfb3f144b57991455b378f6b3f4113c + src/openamr-upperbody-sw: + type: git + url: https://github.com/openAMRobot/openamr-upperbody-sw.git + version: f57e54cfc9e7f9536e94c52749502cc983fb908e + firmware/openamr-platform-fw: + type: git + url: https://github.com/openAMRobot/openamr-platform-fw.git + version: f96014daae614da4e1bf0c00376f1d9e5d41ae02 + firmware/openamr-upperbody-fw: + type: git + url: https://github.com/openAMRobot/openamr-upperbody-fw.git + version: ba1a53f2a031cdbfbe4fa0f5f4d32ed7cdf4dbbb + hardware/openamr-platform-hw: + type: git + url: https://github.com/openAMRobot/openamr-platform-hw.git + version: 6c32fbabfd08491e74a9a2f601f1e14d07d44938 + hardware/openamr-upperbody-hw: + type: git + url: https://github.com/openAMRobot/openamr-upperbody-hw.git + version: 6349ff2672c9e812655d3ae1d04c67130fb3cac7 + ui/openamrobot-ui: + type: git + url: https://github.com/openAMRobot/openamrobot-ui.git + version: c5558ad5babdab30702c307a08fc94a6408c798c + docs/openamrobot-docs: + type: git + url: https://github.com/openAMRobot/openamrobot-docs.git + version: 95c6140e16866964a7d6fddd37e06cd382b7afae + src/sllidar_ros2: + type: git + url: https://github.com/Slamtec/sllidar_ros2.git + version: 34300099fadfc772965962dec837bf436706188f + src/micro_ros_agent: + type: git + url: https://github.com/micro-ROS/micro-ROS-Agent.git + version: 7c932329ad5591ef23942ef1962534258c16b000 + governance/.github: + type: git + url: https://github.com/openAMRobot/.github.git + version: 9109d3e1a8fc07151e500e83b32b7ce6ccab6af6 diff --git a/release-config.rc1.json b/release-config.rc1.json new file mode 100644 index 0000000..f2be49e --- /dev/null +++ b/release-config.rc1.json @@ -0,0 +1,25 @@ +{ + "product": { + "name": "OpenAMRobot", + "version": "2.0.0-rc.1", + "status": "candidate snapshot; not release acceptance" + }, + "manifest": "manifests/v2.0.0-rc.1.repos", + "manifest_sha256": "fa3bfb59dcede1db2d275945916a57dd52d3ff496f2db344376b7e291edc3658", + "sources": { + "src/openamrobot-interfaces": "44791ae3f40a73ce14310a16bd2cb65b6686b72e291754bb8797acb784a65c9e", + "src/openamrobot-comm": "ba5bb57f0975d56e1d36d37a9ed9cb390f52c6f3fcf038e22a005fae5fd0359e", + "src/openamrobot-manipulation": "aaf5fb531566756c881110d8f5c7e04c07b958e5a3ca01eeaaa359594335a5f5", + "src/openamr-platform-sw": "165e3a179c6450af5b7c9b5efe1dc492ee78322da4ee0e6d822a1bdd6a520a85", + "src/openamr-upperbody-sw": "0089765ce8427c3285c585a151acb53f56bed295a1bc49d150f71a6369e84f16", + "firmware/openamr-platform-fw": "c8c891b9265663b993f385b6350a523d217c0cb2fb24e5dd82ae6b656796ae1b", + "firmware/openamr-upperbody-fw": "1bdac6a86e857281fbb9f9f02f29c7109086f5f9a8820306432e0a79df73bd49", + "hardware/openamr-platform-hw": "9da8c7476f0d9d97a97ccc2b790478d8e007176aba36b03036736dca3d08a5d5", + "hardware/openamr-upperbody-hw": "6d25f2befd773cbe3f3eba38089657fd800b9021c6c50713d3bde74c70bcd25a", + "ui/openamrobot-ui": "b308ca0b2362380058b65d06fb76daa98a3d1eff4d4d2d4c61f0e75a18bdc791", + "docs/openamrobot-docs": "0e5e954788ff25ce8790dd85fe7f422eeb9926890615c67aa6486acc875d7c05", + "src/sllidar_ros2": "970b3978319bd95318d1d074de931d75283b36e96982d5c07cd59f43ac934bb7", + "src/micro_ros_agent": "81c1ff84991841682efd4beab8fe826b9164a3527c24a10d257d2e6fce18bc55", + "governance/.github": "a9f0ff5c5f03bfcc079e7518e1d650a1cb3f464fedbc2a5ab1ffcb9bfdba2698" + } +} diff --git a/requirements-release.txt b/requirements-release.txt new file mode 100644 index 0000000..f62ce0c --- /dev/null +++ b/requirements-release.txt @@ -0,0 +1 @@ +PyYAML==6.0.3 diff --git a/scripts/build_pinned_release.py b/scripts/build_pinned_release.py new file mode 100644 index 0000000..170671f --- /dev/null +++ b/scripts/build_pinned_release.py @@ -0,0 +1,203 @@ +"""Build an immutable source bundle; legacy v0.0.1 builder remains available.""" +import argparse +import hashlib +import io +import json +import re +import subprocess +import tempfile +import zipfile +from pathlib import Path, PurePosixPath + +import yaml + + +class StrictLoader(yaml.SafeLoader): + pass + + +def mapping(loader, node): + result = {} + for key, value in node.value: + key = loader.construct_object(key) + if key in result: + raise ValueError(f'duplicate key: {key}') + result[key] = loader.construct_object(value) + return result + + +StrictLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, mapping) + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def safe_path(value): + path = PurePosixPath(value) + if not value or any(ord(c) < 32 for c in value) or path.is_absolute() or '..' in path.parts or '\\' in value or ':' in value or str(path) != value: + raise ValueError(f'unsafe path: {value}') + return value + + +def load_manifest(path): + data = yaml.load(path.read_text(encoding='utf-8'), Loader=StrictLoader) + if not isinstance(data, dict) or set(data) != {'repositories'}: + raise ValueError('expected repositories mapping') + repos = data['repositories'] + if not isinstance(repos, dict) or not repos: + raise ValueError('empty repositories') + for name, entry in repos.items(): + safe_path(name) + if set(entry) != {'type', 'url', 'version'} or entry['type'] != 'git': + raise ValueError(f'{name}: expected git type, url, version') + if not re.fullmatch(r'https://github.com/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+(?:\.git)?', entry['url']): + raise ValueError(f'{name}: expected GitHub HTTPS URL') + if not re.fullmatch(r'[0-9a-f]{40}', entry['version']): + raise ValueError(f'{name}: full commit SHA required') + if any(other != name and other.startswith(name + '/') for other in repos): + raise ValueError('nested repository paths') + return repos + + +def source_archive(entry): + # Export directly from a verified tree, with conversion settings pinned below. + # Archive bytes are generated from a verified Git commit, never main.zip. + with tempfile.TemporaryDirectory() as tmp: + subprocess.run(['git', 'init', '-q', tmp], check=True) + subprocess.run(['git', '-C', tmp, 'fetch', '--depth=1', entry['url'], entry['version']], check=True) + actual = subprocess.check_output(['git', '-C', tmp, 'rev-parse', 'FETCH_HEAD^{commit}'], text=True).strip() + if actual != entry['version']: + raise ValueError('fetched SHA mismatch') + tree = subprocess.check_output(['git', '-C', tmp, 'ls-tree', '-r', actual], text=True) + if any(line.startswith('160000 ') for line in tree.splitlines()): + raise ValueError('submodules need explicit manifest entries') + # Git archive applies checkout line-ending conversion. Pin these + # settings so a Windows user's autocrlf does not change source bytes. + data = subprocess.check_output([ + 'git', '-C', tmp, '-c', 'core.autocrlf=false', '-c', 'core.eol=lf', + 'archive', '--format=zip', actual]) + return canonical_source(data) + + +def canonical_source(data): + """Remove Git/zlib/platform ZIP metadata from the source checksum input.""" + output = io.BytesIO() + with zipfile.ZipFile(io.BytesIO(data)) as source, zipfile.ZipFile(output, 'w', compression=zipfile.ZIP_STORED) as dest: + dest.comment = source.comment + seen = set() + for item in sorted(source.infolist(), key=lambda item: item.filename): + if item.is_dir(): + continue + name = safe_path(item.filename) + if name in seen: + raise ValueError('duplicate source member') + seen.add(name) + mode = item.external_attr >> 16 + mode = 0o120777 if mode & 0o170000 == 0o120000 else (0o100755 if mode & 0o111 else 0o100644) + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.create_system = 3 + info.external_attr = mode << 16 + dest.writestr(info, source.read(item)) + return output.getvalue() + + +def write_zip(path, files): + with zipfile.ZipFile(path, 'w', compression=zipfile.ZIP_STORED) as archive: + for name, (data, mode) in sorted(files.items()): + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.create_system = 3 + info.external_attr = mode << 16 + archive.writestr(info, data) + + +def validate(path): + """Verify inventory coverage and every member without extracting symlinks.""" + with zipfile.ZipFile(path) as archive: + names = archive.namelist() + if len(names) != len(set(names)): + raise ValueError('duplicate archive member') + for name in names: + safe_path(name) + expected = {} + for line in archive.read('checksums.sha256').decode().splitlines(): + checksum, name = line.split(' ', 1) + safe_path(name) + if name in expected or not re.fullmatch(r'[0-9a-f]{64}', checksum): + raise ValueError('invalid checksum inventory') + expected[name] = checksum + if set(names) != set(expected) | {'checksums.sha256'}: + raise ValueError('inventory coverage mismatch') + for name, checksum in expected.items(): + if digest(archive.read(name)) != checksum: + raise ValueError(f'checksum mismatch: {name}') + metadata = json.loads(archive.read('MANIFEST.json')) + if digest(archive.read('release.repos')) != metadata['manifest_sha256']: + raise ValueError('manifest checksum mismatch') + print(f'PASS: {len(expected)} files verified') + + +def build(config_path, output, cache, fetch=False): + config = json.loads(config_path.read_text(encoding='utf-8')) + manifest_path = config_path.parent / config['manifest'] + manifest_bytes = manifest_path.read_bytes() + if digest(manifest_bytes) != config['manifest_sha256']: + raise ValueError('manifest checksum mismatch') + repos = load_manifest(manifest_path) + if set(config['sources']) != set(repos): + raise ValueError('source checksum coverage mismatch') + files = {'release.repos': (manifest_bytes, 0o100644)} + components = [] + cache.mkdir(parents=True, exist_ok=True) + for name, entry in repos.items(): + expected = config['sources'][name] + if not re.fullmatch(r'[0-9a-f]{64}', expected): + raise ValueError(f'{name}: SHA256 required') + cached = cache / (expected + '.zip') + if fetch: + data = source_archive(entry) + if digest(data) != expected: + raise ValueError(f'{name}: source checksum mismatch: expected {expected}, got {digest(data)}') + cached.write_bytes(data) + data = cached.read_bytes() + if digest(data) != expected: + raise ValueError(f'{name}: source checksum mismatch') + with zipfile.ZipFile(io.BytesIO(data)) as archive: + if archive.comment.decode('ascii') != entry['version']: + raise ValueError(f'{name}: archive commit mismatch') + for item in archive.infolist(): + if item.is_dir(): + continue + rel = safe_path(item.filename) + mode = (item.external_attr >> 16) & 0o170777 + if mode & 0o170000 == 0o120000: + # Preserve link bytes and mode without following the target. + mode = 0o120777 + else: + mode = 0o100755 if mode & 0o111 else 0o100644 + files[name + '/' + rel] = (archive.read(item), mode) + components.append(dict(path=name, **entry, archive_sha256=expected)) + metadata = {'product': config['product'], 'manifest_sha256': digest(manifest_bytes), 'components': components} + files['MANIFEST.json'] = ((json.dumps(metadata, indent=2, sort_keys=True) + '\n').encode(), 0o100644) + checksums = ''.join(f'{digest(data)} {name}\n' for name, (data, _) in sorted(files.items())) + files['checksums.sha256'] = (checksums.encode(), 0o100644) + output.mkdir(parents=True, exist_ok=True) + version = config['product']['version'] + if not re.fullmatch(r'[0-9A-Za-z.-]+', version): + raise ValueError('invalid product version') + target = output / f'OpenAMRobot-v{version}-source.zip' + write_zip(target, files) + validate(target) + target.with_suffix('.zip.sha256').write_text(f'{digest(target.read_bytes())} {target.name}\n', encoding='ascii') + print(f'{target}: {digest(target.read_bytes())}') + return target + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--config', type=Path, required=True) + parser.add_argument('--output-dir', type=Path, required=True) + parser.add_argument('--cache-dir', type=Path, required=True) + parser.add_argument('--fetch', action='store_true', help='fetch and verify every source by SHA') + args = parser.parse_args() + build(args.config, args.output_dir, args.cache_dir, args.fetch) diff --git a/scripts/build_release.py b/scripts/build_release.py index dd07dae..ce3c711 100644 --- a/scripts/build_release.py +++ b/scripts/build_release.py @@ -34,9 +34,14 @@ def main() -> int: p.add_argument('--output-dir', required=True, type=Path) p.add_argument('--metadata-dir', type=Path, default=Path('release-metadata')) p.add_argument('--keep-staging', action='store_true') + p.add_argument('--fetch', action='store_true', help='Fetch pinned rc.1 sources by SHA before building') args = p.parse_args() cfg = json.loads(args.config.read_text()) + if 'manifest' in cfg: + from build_pinned_release import build + build(args.config, args.output_dir, args.input_dir, args.fetch) + return 0 product = cfg['product'] version = product['version'] package_name = f"{product['name']}-v{version}" diff --git a/scripts/check_manifest_sync.py b/scripts/check_manifest_sync.py new file mode 100644 index 0000000..981780a --- /dev/null +++ b/scripts/check_manifest_sync.py @@ -0,0 +1,40 @@ +"""Require the release snapshot to match the workspace rc.1 manifest.""" +import argparse +from pathlib import Path + +from build_pinned_release import load_manifest + + +def check_sync(release_path, workspace_path): + release = load_manifest(release_path) + workspace = load_manifest(workspace_path) + # Only this release-specific checkout is exempt, on either side. + release.pop('governance/.github', None) + workspace.pop('governance/.github', None) + differences = [] + for name in sorted(release.keys() | workspace.keys()): + if name not in release: + differences.append(f'{name}: missing from release manifest') + elif name not in workspace: + differences.append(f'{name}: missing from workspace manifest') + elif release[name] != workspace[name]: + differences.append(f'{name}: type, URL or SHA differs') + if differences: + raise ValueError('manifest drift:\n' + '\n'.join(differences)) + return len(release) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('release', type=Path) + parser.add_argument('workspace', type=Path) + args = parser.parse_args() + try: + count = check_sync(args.release, args.workspace) + except (ValueError, OSError) as error: + parser.exit(1, f'{error}\n') + print(f'Matched {count} repositories (governance/.github excepted)') + + +if __name__ == '__main__': + main() diff --git a/scripts/lock_release_sources.py b/scripts/lock_release_sources.py new file mode 100644 index 0000000..633dccf --- /dev/null +++ b/scripts/lock_release_sources.py @@ -0,0 +1,29 @@ +"""Prepare a candidate checksum lock from a reviewed SHA-only manifest.""" +import argparse +import json +from pathlib import Path +from build_pinned_release import digest, load_manifest, source_archive + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--manifest', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--cache-dir', type=Path, required=True) + parser.add_argument('--version', required=True) + args = parser.parse_args() + # Keep review of a new lock explicit: never replace an existing lock file. + if args.output.exists(): + raise SystemExit('Choose a new output file; existing locks are not overwritten') + relative = args.manifest.resolve().relative_to(args.output.parent.resolve()).as_posix() + repos = load_manifest(args.manifest) + config = {'product': {'name': 'OpenAMRobot', 'version': args.version, + 'status': 'candidate snapshot; not release acceptance'}, + 'manifest': relative, 'manifest_sha256': digest(args.manifest.read_bytes()), 'sources': {}} + args.cache_dir.mkdir(parents=True, exist_ok=True) + for name, entry in repos.items(): + data = source_archive(entry) + checksum = digest(data) + (args.cache_dir / (checksum + '.zip')).write_bytes(data) + config['sources'][name] = checksum + print(f'{name}: {entry["version"]} {checksum}', flush=True) + args.output.write_text(json.dumps(config, indent=2) + '\n', encoding='utf-8', newline='\n') diff --git a/scripts/validate_pinned_release.py b/scripts/validate_pinned_release.py new file mode 100644 index 0000000..fa8e138 --- /dev/null +++ b/scripts/validate_pinned_release.py @@ -0,0 +1,9 @@ +"""Validate a pinned source bundle without extracting it.""" +import argparse +from pathlib import Path +from build_pinned_release import validate + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('archive', type=Path) + validate(parser.parse_args().archive) diff --git a/scripts/validate_release.py b/scripts/validate_release.py index 99f80c4..93582ce 100644 --- a/scripts/validate_release.py +++ b/scripts/validate_release.py @@ -14,6 +14,11 @@ def sha256(path: Path) -> str: p.add_argument('archive', type=Path) a=p.parse_args() if not a.archive.is_file(): raise SystemExit('Archive not found') +with zipfile.ZipFile(a.archive) as archive: + if 'release.repos' in archive.namelist(): + from build_pinned_release import validate + validate(a.archive) + raise SystemExit(0) with tempfile.TemporaryDirectory() as td: td=Path(td) with zipfile.ZipFile(a.archive) as z: diff --git a/tests/test_manifest_sync.py b/tests/test_manifest_sync.py new file mode 100644 index 0000000..6b981d3 --- /dev/null +++ b/tests/test_manifest_sync.py @@ -0,0 +1,69 @@ +"""Parity must cover every workspace source, including the drivers.""" +import copy +from pathlib import Path +import sys +import tempfile +import unittest + +import yaml + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts')) +from check_manifest_sync import check_sync + + +class ManifestSyncTests(unittest.TestCase): + def setUp(self): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + self.root = Path(temp.name) + self.entry = {'type': 'git', 'url': 'https://github.com/example/source.git', + 'version': 'a' * 40} + self.repos = {'src/example': self.entry, 'src/sllidar_ros2': self.entry} + + def write(self, name, repos): + path = self.root / name + path.write_text(yaml.safe_dump({'repositories': repos}), encoding='utf-8') + return path + + def compare(self, release, workspace): + return check_sync(self.write('release.repos', release), + self.write('workspace.repos', workspace)) + + def test_order_and_comments_do_not_matter(self): + release = self.write('release.repos', self.repos) + workspace = self.root / 'workspace.repos' + workspace.write_text('# workspace\n' + yaml.safe_dump( + {'repositories': dict(reversed(list(self.repos.items())))}, + sort_keys=False), encoding='utf-8') + self.assertEqual(check_sync(release, workspace), 2) + + def test_only_governance_is_exempt(self): + release = dict(self.repos, **{'governance/.github': self.entry}) + self.assertEqual(self.compare(release, self.repos), 2) + + def test_missing_driver_fails(self): + with self.assertRaisesRegex(ValueError, 'src/sllidar_ros2: missing'): + self.compare(self.repos, {'src/example': self.entry}) + + def test_missing_release_entry_fails(self): + with self.assertRaisesRegex(ValueError, 'missing from release'): + self.compare({'src/example': self.entry}, self.repos) + + def test_changed_sha_or_url_fails(self): + for field, value in [('version', 'b' * 40), + ('url', 'https://github.com/example/other.git')]: + with self.subTest(field=field): + changed = copy.deepcopy(self.repos) + changed['src/example'][field] = value + with self.assertRaisesRegex(ValueError, 'differs'): + self.compare(self.repos, changed) + + def test_missing_reference_fails(self): + with self.assertRaises(FileNotFoundError): + check_sync(self.write('release.repos', self.repos), self.root / 'absent') + + def test_mutable_reference_fails(self): + changed = copy.deepcopy(self.repos) + changed['src/example']['version'] = 'main' + with self.assertRaisesRegex(ValueError, 'full commit SHA required'): + self.compare(self.repos, changed) diff --git a/tests/test_pinned_release.py b/tests/test_pinned_release.py new file mode 100644 index 0000000..4b9c0d6 --- /dev/null +++ b/tests/test_pinned_release.py @@ -0,0 +1,204 @@ +"""Offline fault injection and reproducibility tests for rc.1 packaging.""" +import io +import json +import os +import subprocess +import sys +import tempfile +import unittest +import zipfile +from pathlib import Path +from unittest.mock import patch + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'scripts')) +import build_pinned_release as builder + + +class PinnedReleaseTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.sha = 'a' * 40 + self.entry = {'type': 'git', 'url': 'https://github.com/example/source.git', 'version': self.sha} + self.manifest = self.root / 'release.repos' + self.manifest.write_text(yaml.safe_dump({'repositories': {'src/example': self.entry}})) + data = io.BytesIO() + with zipfile.ZipFile(data, 'w') as archive: + archive.comment = self.sha.encode() + archive.writestr('README.md', b'source content\n') + self.source = data.getvalue() + self.checksum = builder.digest(self.source) + self.cache = self.root / 'cache' + self.cache.mkdir() + (self.cache / (self.checksum + '.zip')).write_bytes(self.source) + self.config = {'product': {'name': 'OpenAMRobot', 'version': '2.0.0-rc.1'}, + 'manifest': 'release.repos', 'manifest_sha256': builder.digest(self.manifest.read_bytes()), + 'sources': {'src/example': self.checksum}} + self.config_path = self.root / 'config.json' + self.save_config() + + def save_config(self): + self.config_path.write_text(json.dumps(self.config)) + + def build(self, name='out', fetch=False): + return builder.build(self.config_path, self.root / name, self.cache, fetch) + + def test_two_builds_identical(self): + first = self.build('first') + second = self.build('second') + self.assertEqual(first.read_bytes(), second.read_bytes()) + builder.validate(first) + with zipfile.ZipFile(first) as archive: + component = json.loads(archive.read('MANIFEST.json'))['components'][0] + self.assertEqual(component['version'], self.sha) + self.assertEqual(component['archive_sha256'], self.checksum) + + def test_source_normalization_ignores_zip_metadata(self): + variants = [] + for date, compression in (((2001, 1, 1, 0, 0, 0), zipfile.ZIP_STORED), + ((2026, 1, 1, 0, 0, 0), zipfile.ZIP_DEFLATED)): + stream = io.BytesIO() + with zipfile.ZipFile(stream, 'w', compression=compression) as archive: + archive.comment = self.sha.encode() + item = zipfile.ZipInfo('README.md', date_time=date) + item.compress_type = compression + item.external_attr = 0o100644 << 16 + archive.writestr(item, b'same bytes') + variants.append(builder.canonical_source(stream.getvalue())) + self.assertEqual(*variants) + + def test_source_hash_ignores_git_line_ending_defaults(self): + revision = subprocess.check_output(['git', '-C', str(ROOT), 'rev-parse', 'HEAD'], text=True).strip() + entry = {'url': str(ROOT), 'version': revision} + hashes = [] + for autocrlf in ('true', 'false'): + with patch.dict(os.environ, {'GIT_CONFIG_COUNT': '1', + 'GIT_CONFIG_KEY_0': 'core.autocrlf', + 'GIT_CONFIG_VALUE_0': autocrlf}): + hashes.append(builder.digest(builder.source_archive(entry))) + self.assertEqual(hashes[0], hashes[1]) + + def test_executable_and_symlink_modes_preserved(self): + data = io.BytesIO() + with zipfile.ZipFile(data, 'w') as archive: + for name, mode, content in [('tool.sh', 0o100755, b'echo test'), + ('link', 0o120777, b'tool.sh')]: + item = zipfile.ZipInfo(name) + item.external_attr = mode << 16 + archive.writestr(item, content) + normalized = builder.canonical_source(data.getvalue()) + with zipfile.ZipFile(io.BytesIO(normalized)) as archive: + self.assertEqual(archive.getinfo('tool.sh').external_attr >> 16, 0o100755) + self.assertEqual(archive.getinfo('link').external_attr >> 16, 0o120777) + self.assertEqual(archive.read('link'), b'tool.sh') + + def test_fetch_uses_full_sha_and_verifies_commit(self): + outputs = [self.sha + '\n', '', self.source] + with patch.object(builder.subprocess, 'run') as run, patch.object(builder.subprocess, 'check_output', side_effect=outputs): + self.assertEqual(builder.source_archive(self.entry), builder.canonical_source(self.source)) + fetch_args = run.call_args_list[1].args[0] + self.assertEqual(fetch_args[-2:], [self.entry['url'], self.sha]) + + def test_fetched_wrong_sha_and_submodules_rejected(self): + with patch.object(builder.subprocess, 'run'), patch.object(builder.subprocess, 'check_output', return_value='b' * 40): + with self.assertRaisesRegex(ValueError, 'fetched SHA mismatch'): + builder.source_archive(self.entry) + with patch.object(builder.subprocess, 'run'), patch.object(builder.subprocess, 'check_output', side_effect=[self.sha, '160000 commit deadbeef\tsubmodule']): + with self.assertRaisesRegex(ValueError, 'submodules'): + builder.source_archive(self.entry) + + def test_changed_manifest_rejected(self): + self.manifest.write_text(self.manifest.read_text() + '# changed\n') + with self.assertRaisesRegex(ValueError, 'manifest checksum mismatch'): + self.build() + + def test_branch_tag_and_short_sha_rejected(self): + for version in ('main', 'v2.0.0-rc.1', self.sha[:7]): + with self.subTest(version=version): + self.entry['version'] = version + self.manifest.write_text(yaml.safe_dump({'repositories': {'src/example': self.entry}})) + with self.assertRaisesRegex(ValueError, 'full commit SHA'): + builder.load_manifest(self.manifest) + + def test_duplicate_manifest_keys_rejected(self): + self.manifest.write_text('repositories: {}\nrepositories: {}\n') + with self.assertRaisesRegex(ValueError, 'duplicate key'): + builder.load_manifest(self.manifest) + + def test_tampered_cached_source_rejected(self): + (self.cache / (self.checksum + '.zip')).write_bytes(b'tampered') + with self.assertRaisesRegex(ValueError, 'source checksum mismatch'): + self.build() + + def test_fetch_mismatch_rejected(self): + with patch.object(builder, 'source_archive', return_value=b'wrong source'): + with self.assertRaisesRegex(ValueError, 'source checksum mismatch'): + self.build(fetch=True) + + def test_wrong_archive_commit_rejected(self): + data = io.BytesIO() + with zipfile.ZipFile(data, 'w') as archive: + archive.comment = ('b' * 40).encode() + archive.writestr('README.md', 'wrong commit') + raw = data.getvalue() + checksum = builder.digest(raw) + (self.cache / (checksum + '.zip')).write_bytes(raw) + self.config['sources']['src/example'] = checksum + self.save_config() + with self.assertRaisesRegex(ValueError, 'archive commit mismatch'): + self.build() + + def test_missing_checksum_rejected(self): + self.config['sources'] = {} + self.save_config() + with self.assertRaisesRegex(ValueError, 'coverage mismatch'): + self.build() + + def test_output_tampering_and_extra_file_rejected(self): + original = self.build() + for mutation in ('content', 'extra'): + with self.subTest(mutation=mutation): + target = self.root / (mutation + '.zip') + with zipfile.ZipFile(original) as source, zipfile.ZipFile(target, 'w') as dest: + for name in source.namelist(): + data = source.read(name) + if mutation == 'content' and name == 'src/example/README.md': + data = b'tampered' + dest.writestr(name, data) + if mutation == 'extra': + dest.writestr('extra.txt', b'extra') + with self.assertRaisesRegex(ValueError, 'checksum mismatch|coverage mismatch'): + builder.validate(target) + + def test_unsafe_paths_rejected(self): + for path in ('../escape', '/absolute', 'C:/escape', 'a\\b', 'a\nb', 'a/../b'): + with self.subTest(path=path), self.assertRaisesRegex(ValueError, 'unsafe path'): + builder.safe_path(path) + + def test_existing_cli_dispatches_rc1(self): + subprocess.run([sys.executable, str(ROOT / 'scripts/build_release.py'), '--config', str(self.config_path), + '--input-dir', str(self.cache), '--output-dir', str(self.root / 'cli')], check=True) + subprocess.run([sys.executable, str(ROOT / 'scripts/validate_release.py'), + str(self.root / 'cli/OpenAMRobot-v2.0.0-rc.1-source.zip')], check=True) + + def test_legacy_v001_build_and_validation_unchanged(self): + config = json.loads((ROOT / 'release-config.json').read_text()) + inputs = self.root / 'legacy-input' + inputs.mkdir() + for component in config['components']: + with zipfile.ZipFile(inputs / component['archive'], 'w') as archive: + archive.writestr(component['expected_root'] + '/README.md', 'legacy fixture') + output = self.root / 'legacy-output' + subprocess.run([sys.executable, str(ROOT / 'scripts/build_release.py'), + '--config', str(ROOT / 'release-config.json'), '--input-dir', str(inputs), + '--output-dir', str(output), '--metadata-dir', str(ROOT / 'release-metadata')], check=True) + subprocess.run([sys.executable, str(ROOT / 'scripts/validate_release.py'), + str(output / 'OpenAMRobot-v0.0.1-source.zip')], check=True) + + +if __name__ == '__main__': + unittest.main()