diff --git a/.github/workflows/c10-dataset-quality.yml b/.github/workflows/c10-dataset-quality.yml new file mode 100644 index 0000000..b16e0fc --- /dev/null +++ b/.github/workflows/c10-dataset-quality.yml @@ -0,0 +1,73 @@ +name: C10 reference quality gate + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + c10-reference: + runs-on: ubuntu-24.04 + defaults: + run: + shell: bash + env: + C10_CHECKER_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + C10_EVIDENCE_DIR: .verification/c10-evidence + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - name: Prepare evidence directory + run: | + mkdir -p "$C10_EVIDENCE_DIR" + printf '%s\n' "$C10_CHECKER_SHA" > "$C10_EVIDENCE_DIR/expected-checker-sha.txt" + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install scoped C10 verification dependencies + run: | + python -m pip install -r tools/requirements-c10-native.txt + sudo apt-get update + sudo apt-get install -y --no-install-recommends ffmpeg + - name: Run nonempty C10 core suite + if: ${{ !cancelled() }} + run: | + set -o pipefail + python tools/run_verification_tests.py tests test_c10_dataset.py 2>&1 | tee "$C10_EVIDENCE_DIR/core-suite.log" + - name: Run nonempty C10 native suite + if: ${{ !cancelled() }} + run: | + set -o pipefail + python tools/run_verification_tests.py tests test_c10_native.py 2>&1 | tee "$C10_EVIDENCE_DIR/native-suite.log" + - name: Run nonempty C10 evidence suite + if: ${{ !cancelled() }} + run: | + set -o pipefail + python tools/run_verification_tests.py tests test_c10_evidence.py 2>&1 | tee "$C10_EVIDENCE_DIR/evidence-suite.log" + - name: Prove reverted arithmetic check is detected + if: ${{ !cancelled() }} + run: | + set -o pipefail + python tools/prove_c10_negative.py 2>&1 | tee "$C10_EVIDENCE_DIR/revert-negative.log" + - name: Generate reproducible synthetic CLI evidence + if: ${{ !cancelled() }} + run: | + set -o pipefail + python tools/generate_c10_evidence.py \ + --output-dir "$C10_EVIDENCE_DIR/synthetic" \ + --expected-checker-sha "$C10_CHECKER_SHA" \ + 2>&1 | tee "$C10_EVIDENCE_DIR/generation.log" + - name: Upload C10 report and test evidence + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: c10-evidence-${{ github.event.pull_request.head.sha || github.sha }}-${{ github.run_attempt }} + path: .verification/c10-evidence/ + include-hidden-files: true + if-no-files-found: error + retention-days: 30 diff --git a/docs/c10-check-fixture-matrix.md b/docs/c10-check-fixture-matrix.md new file mode 100644 index 0000000..27af8fa --- /dev/null +++ b/docs/c10-check-fixture-matrix.md @@ -0,0 +1,101 @@ + +# C10 check-to-corrupted-fixture acceptance matrix + +Generated from the named cases in `tests/test_c10_dataset.py` and +`tests/test_c10_native.py`. Core corruption cases remain valid against the +unchanged C9 Draft 2020-12 schema. Wrapper cases deliberately corrupt native +artifacts or exercise missing/contradictory evidence. No real recording is altered. + +| Stable diagnostic | Deliberate corruption / missing-evidence case | Expected | Named unittest | +| --- | --- | --- | --- | +| `C10_ACTION_BOUNDS` | `action_range` | REJECT | `C10SemanticTests.test_reject_action_range` | +| `C10_ACTION_LATENCY` | `action_timestamp_semantics` | REJECT | `C10SemanticTests.test_reject_action_timestamp_semantics` | +| `C10_ACTION_LATENCY` | `action_value_semantics` | REJECT | `C10SemanticTests.test_reject_action_value_semantics` | +| `C10_ACTION_LATENCY` | `command_latency` | REJECT | `C10SemanticTests.test_reject_command_latency` | +| `C10_BINDING` | `binding_revision` | REJECT | `C10SemanticTests.test_reject_binding_revision` | +| `C10_BINDING` | `episode_length` | REJECT | `C10SemanticTests.test_reject_episode_length` | +| `C10_BINDING` | `external_native_identity_contradiction` | REJECT | `C10SemanticTests.test_reject_external_native_identity_contradiction` | +| `C10_BINDING` | `format_version` | REJECT | `C10SemanticTests.test_reject_format_version` | +| `C10_BINDING` | `orphan_native_episode` | REJECT | `C10SemanticTests.test_reject_orphan_native_episode` | +| `C10_COMPLETENESS` | `interrupted_recording` | REJECT | `C10SemanticTests.test_reject_interrupted_recording` | +| `C10_COMPLETENESS` | `success_with_abort_event` | REJECT | `C10SemanticTests.test_reject_success_with_abort_event` | +| `C10_COVERAGE` | `duplicate_frame` | REJECT | `C10SemanticTests.test_reject_duplicate_frame` | +| `C10_COVERAGE` | `frame_coverage` | REJECT | `C10SemanticTests.test_reject_frame_coverage` | +| `C10_COVERAGE` | `native_frame_index` | REJECT | `C10SemanticTests.test_reject_native_frame_index` | +| `C10_EVENTS` | `duplicate_event_id` | REJECT | `C10SemanticTests.test_reject_duplicate_event_id` | +| `C10_EVENTS` | `event_missing_frame` | REJECT | `C10SemanticTests.test_reject_event_missing_frame` | +| `C10_EVENTS` | `event_outside_recording` | REJECT | `C10SemanticTests.test_reject_event_outside_recording` | +| `C10_EVENTS` | `inverted_event` | REJECT | `C10SemanticTests.test_reject_inverted_event` | +| `C10_FEATURES` | `camera_fps_descriptor` | REJECT | `C10SemanticTests.test_reject_camera_fps_descriptor` | +| `C10_FEATURES` | `feature_shape` | REJECT | `C10SemanticTests.test_reject_feature_shape` | +| `C10_FEATURES` | `legacy_camera_aliases` | REJECT | `C10SemanticTests.test_reject_legacy_camera_aliases` | +| `C10_FEATURES` | `native_vector_name_order` | REJECT | `C10SemanticTests.test_reject_native_vector_name_order` | +| `C10_FEATURES` | `native_vector_shape` | REJECT | `C10SemanticTests.test_reject_native_vector_shape` | +| `C10_HEALTH` | `fresh_reused_sample` | REJECT | `C10SemanticTests.test_reject_fresh_reused_sample` | +| `C10_HEALTH` | `fresh_without_receipt` | REJECT | `C10SemanticTests.test_reject_fresh_without_receipt` | +| `C10_MAPPING` | `affine_arithmetic` | REJECT | `C10SemanticTests.test_reject_affine_arithmetic` | +| `C10_MAPPING` | `mapping_validity_interval` | REJECT | `C10SemanticTests.test_reject_mapping_validity_interval` | +| `C10_PROFILE` | `profile_dimension_order` | REJECT | `C10SemanticTests.test_reject_profile_dimension_order` | +| `C10_PROFILE` | `profile_units` | REJECT | `C10SemanticTests.test_reject_profile_units` | +| `C10_SELECTION` | `selected_time_disagrees_with_mapping` | REJECT | `C10SemanticTests.test_reject_selected_time_disagrees_with_mapping` | +| `C10_SELECTION` | `selection_source` | REJECT | `C10SemanticTests.test_reject_selection_source` | +| `C10_SEQUENCE` | `nonconsecutive_source_id_reuse` | REJECT | `C10SemanticTests.test_reject_nonconsecutive_source_id_reuse` | +| `C10_SEQUENCE` | `sequence_reset` | REJECT | `C10SemanticTests.test_reject_sequence_reset` | +| `C10_SKEW` | `reported_skew` | REJECT | `C10SemanticTests.test_reject_reported_skew` | +| `C10_STREAM_QUALITY` | `measured_source_fps` | REJECT | `C10SemanticTests.test_reject_measured_source_fps` | +| `C10_STREAM_QUALITY` | `sequence_gap` | REJECT | `C10SemanticTests.test_reject_sequence_gap` | +| `C10_STREAM_QUALITY` | `stale_observation_age` | REJECT | `C10SemanticTests.test_reject_stale_observation_age` | +| `C10_SYNC` | `declared_policy_tolerance_mismatch` | REJECT | `C10SemanticTests.test_reject_declared_policy_tolerance_mismatch` | +| `C10_SYNC` | `inter_observation_skew` | REJECT | `C10SemanticTests.test_reject_inter_observation_skew` | +| `C10_TASK` | `orphan_native_task` | REJECT | `C10SemanticTests.test_reject_orphan_native_task` | +| `C10_TASK` | `task_text` | REJECT | `C10SemanticTests.test_reject_task_text` | +| `C10_TIME_ORDER` | `descending_authoritative_time` | REJECT | `C10SemanticTests.test_reject_descending_authoritative_time` | +| `C10_TIME_ORDER` | `timestamp_order` | REJECT | `C10SemanticTests.test_reject_timestamp_order` | +| `C10_VECTOR_VALUES` | `vector_infinity` | REJECT | `C10SemanticTests.test_reject_vector_infinity` | +| `C10_VECTOR_VALUES` | `vector_nonfinite` | REJECT | `C10SemanticTests.test_reject_vector_nonfinite` | +| `C10_VECTOR_VALUES` | `vector_width` | REJECT | `C10SemanticTests.test_reject_vector_width` | +| `C10_ARTIFACT` | `absent_configuration_artifact` | NOT_VERIFIED | `C10WrapperTests.test_wrapper_absent_configuration_artifact` | +| `C10_ARTIFACT` | `artifact_bytes_changed` | REJECT | `C10WrapperTests.test_wrapper_artifact_bytes_changed` | +| `C10_ARTIFACT` | `configuration_declared_hash` | REJECT | `C10WrapperTests.test_wrapper_configuration_declared_hash` | +| `C10_ARTIFACT` | `configuration_symlink_escape` | REJECT | `C10WrapperTests.test_wrapper_configuration_symlink_escape` | +| `C10_CALIBRATION` | `calibration_identity` | REJECT | `C10WrapperTests.test_wrapper_calibration_identity` | +| `C10_CALIBRATION` | `calibration_residual_exceeds` | REJECT | `C10WrapperTests.test_wrapper_calibration_residual_exceeds` | +| `C10_CALIBRATION` | `unapproved_calibration` | NOT_VERIFIED | `C10WrapperTests.test_wrapper_unapproved_calibration` | +| `C10_INPUT` | `duplicate_json_key` | REJECT | `C10WrapperTests.test_wrapper_duplicate_json_key` | +| `C10_INPUT` | `malformed_nested_policy` | REJECT | `C10WrapperTests.test_wrapper_malformed_nested_policy` | +| `C10_NATIVE_DATA` | `native_row_range` | REJECT | `C10WrapperTests.test_wrapper_native_row_range` | +| `C10_NATIVE_DATA` | `orphan_native_episode_rows` | REJECT | `C10WrapperTests.test_wrapper_orphan_native_episode_rows` | +| `C10_NATIVE_DATA` | `truncated_parquet` | REJECT | `C10WrapperTests.test_wrapper_truncated_parquet` | +| `C10_NATIVE_METADATA` | `duplicate_native_episode` | REJECT | `C10WrapperTests.test_wrapper_duplicate_native_episode` | +| `C10_PARQUET_DTYPE` | `native_double_vectors` | REJECT | `C10WrapperTests.test_wrapper_native_double_vectors` | +| `C10_PROFILE` | `source_profile_identity` | REJECT | `C10WrapperTests.test_wrapper_source_profile_identity` | +| `C10_REAL_EPISODE` | `synthetic_is_not_real_evidence` | NOT_VERIFIED | `C10WrapperTests.test_wrapper_synthetic_is_not_real_evidence` | +| `C10_RELEASE` | `recording_release_commit` | REJECT | `C10WrapperTests.test_wrapper_recording_release_commit` | +| `C10_RELEASE` | `unapproved_release` | NOT_VERIFIED | `C10WrapperTests.test_wrapper_unapproved_release` | +| `C10_SCHEMA` | `invalid_c9_schema` | REJECT | `C10WrapperTests.test_wrapper_invalid_c9_schema` | +| `C10_VIDEO` | `decoded_video_frame_count` | REJECT | `C10WrapperTests.test_wrapper_decoded_video_frame_count` | +| `C10_VIDEO` | `frozen_decoded_pixels` | REJECT | `C10WrapperTests.test_wrapper_frozen_decoded_pixels` | + +Additional explicit tests cover clean semantic PASS, independent fixture objects, +inclusive action/synchronization/clock-map boundaries, ties-to-even mapping, +exact JSON/YAML fractional clock lexemes, pending tolerance with an approved +policy, pending/unmapped clocks, honest missing/dropped slots, stale producer +labels requiring review, missing episode identity, source-profile identity, +explicit action reference offsets, unavailable native data, native metadata +totals, strict JSON and nonfinite vectors, CLI exits and unchanged dataset bytes. + +`test_every_core_diagnostic_has_named_corruption` fails if a semantic diagnostic +is added without a named corruption. All CLI diagnostic families also appear +above. The acceptance matrix proves diagnostic-specific fault detection, not +independent correctness of every algorithm or physical acceptance. + +`test_real_video_decode_or_explicit_unavailable` decodes a temporary synthetic +MP4 and corrupts its bytes when ffmpeg/ffprobe are available. Without them it +explicitly asserts NOT_VERIFIED and prints which branch was exercised. It does +not use unittest skip. The decoded-frame-count and frozen-pixel tests use a +deterministic subprocess double and are distinguished from actual decoding. + +The revert demonstration uses `python tools/prove_c10_negative.py`: one unchanged +negative test passes on current code, then fails with exactly one assertion +failure after the arithmetic comparison is reverted in a temporary source copy. +The original checkout and fixtures remain intact. diff --git a/docs/c10-compatibility-matrix.md b/docs/c10-compatibility-matrix.md new file mode 100644 index 0000000..db3a0d9 --- /dev/null +++ b/docs/c10-compatibility-matrix.md @@ -0,0 +1,42 @@ +# C10 compatibility review + +Status: **Draft conformance reference, pending architecture review.** This review +does not approve a runtime home, source profile, release pin or numeric threshold. +The reference tooling under `tools/` and its synthetic `tests/` exercise the +accepted C9 contract; recorder, ingestion and training implementation ownership +remains unresolved. + +The C9 v1 shape was accepted in [issue #18](https://github.com/openAMRobot/openamrobot-interfaces/issues/18#issuecomment-5919324379) +and [PR #16](https://github.com/openAMRobot/openamrobot-interfaces/pull/16) +merged on 2 October 2026 as `fa7c438e33bd807c174fa0747119d7d627faa3bf`. +The older Proposed label in the C9 document does not override that decision. +Comparison baseline: [dataset tooling](https://github.com/4u-79/openamrobot-c-baseline/tree/a29ce7c89b7c42c9147ad81e6a751de9dad045c9), +commit `a29ce7c89b7c42c9147ad81e6a751de9dad045c9`. + +| Area | Inspected baseline behavior | Accepted C9 v1 / reconciliation needed | +| --- | --- | --- | +| Native format | Calls its layout v3.0; expects JSONL task/episode registries and one Parquet/video file per episode. | Native LeRobot v3.0 remains primary; sidecar at `meta/openamrobot/episodes/episode-{episode_index:06d}.json`. Resolve actual native metadata/file bindings using the chosen writer; the label alone proves no loader compatibility. | +| Cameras | Wrist keys agree; requires `observation.images.camera_scene` and `observation.images.camera_base_nav` with fixed shapes. | Requires `observation.images.head` and `observation.images.base` plus wrists; additional explicitly bound cameras allowed. An approved source mapping must resolve the differing keys and physical/stereo identity; no silent aliases. | +| State/action | Exactly 19 channels; historical fixed-mast rev18.2 profile, mixed base velocity, joint radians and normalized grippers; zero-limit torso channels. Both validator constants are 19; README and validator docstring retain stale 21-channel wording. | Named ordered dimensions, units, frames and action semantics control interpretation. The supplied OpenArm development profile has 16 positions in degrees. Neither vector length nor latest URDF authorizes conversion. The historical profile must remain distinct from the current lift/arm configuration. | +| Source profiles | Enum `ROBOT`, `GAZEBO`, `ISAAC`, `CALIBRATION`, `DIAGNOSTICS`; defaults to requiring `ROBOT`. | Source-profile identity and version must describe the actual configuration, serialization, clocks and units. The enum alone establishes none of these facts. Synthetic or simulation evidence cannot establish physical acceptance. | +| Clock representation | Two frame-level float64 seconds columns: host monotonic `timestamp` and `timestamp_device`. Integrity gate assumes epoch alignment; contract validator instead removes the median host/device offset. | Recording-relative integer nanoseconds; each stream retains raw device/host receipt evidence and explicit affine mapping, unit, source domain and validity interval. A host receipt is an arrival proxy. Reconstructing absent acquisition evidence from native FPS is invalid. | +| Skew | Integrity gate: maximum absolute host/device difference. Contract validator: maximum deviation of that difference from its median. These measure different quantities. | Recompute selected time using the declared mapping and ties-to-even nanosecond rounding. Recompute per-stream absolute skew against observation reference; inter-observation skew is max minus min across cameras/state. Assess action latency separately. Missing/unmapped timing is unassessable, never zero. | +| Thresholds | Defaults: 30 ms skew, 30 frames minimum, camera container FPS at least 95% of configured FPS; frozen run at least 2 s and phase lag beyond 3 frames produce nonblocking flags. | These are baseline parameters, not accepted C9 release limits. C9 pending tolerance is null; a declared value plus evidence does not prove approval. An explicit reviewed policy must supply numeric acceptance values. Container FPS cannot establish acquisition freshness. | +| Range checks | Hard-coded 19-channel limits, 5% of span margin; zero-span channels use absolute `1e-4` tolerance. `FIX-OOB-STATS` injects NaN/Inf rather than a finite range violation. | Check finite values and profile-specific bounds separately, by dimension name/unit and immutable configuration evidence. Do not apply radians/normalized-gripper limits to the degrees profile. | +| Calibration/configuration | README describes hashes; inspected gate attaches source profile and declared length from the episode registry, without checking C9 calibration/configuration artifacts. | Each camera needs calibration ID/version and structured measured residual or explicit not-measured reason. Configuration hashes identify exact artifact bytes and serialization. Checking identifier shape/equality does not verify artifact existence, contents or physical accuracy. | +| Provenance/pins | `requirements.txt` pins `lerobot==0.4.1`; README describes a future release manifest. | Inspected upstream snapshot is v0.6.1, `7e241bd630a3719a56157a497ce5d08f244784f1`; this is not a validated release pin. Record actual recorder version/commit separately; retain unknowns and null validated pin until release-owner approval. | +| Completeness/events | Eight fault classes cover jitter, missing video frames, phase lag, repeated images, broken Parquet, vector width, nonfinite values and orphan task. Most operate on in-memory synthetic records. | Bind every native frame/task/episode and stream source sample; verify recording boundaries/completeness, independent outcome and event linkage. Synthetic fixtures do not demonstrate native dataset loading, media decoding or real capture. | + +The [13 October integration decision](https://github.com/openAMRobot/openamrobot-interfaces/issues/18#issuecomment-6152412257) +requires camera mappings, configuration/action profiles, pins and timestamps to +be reconciled before a real-episode gate, training or another schema freeze. +The accepted additive schema shape remains intact. The +[organization decisions register](https://github.com/openAMRobot/.github/blob/539da1982bccf8e2315483f41ac16734e0bed328/decisions.yaml) +records the current lift/arm and configuration distinctions; it supplies no +C10 runtime-home or dataset synchronization acceptance decision. + +Acceptance remains blocked on an approved runtime owner/location and policy, +an approved source profile with immutable configuration/calibration evidence, +a validated LeRobot release pin, and one actual native episode plus matching C9 +sidecar and end-to-end report. A passing conformance fixture alone closes none +of those items. diff --git a/docs/c10-evidence-handoff.md b/docs/c10-evidence-handoff.md new file mode 100644 index 0000000..1ce76ab --- /dev/null +++ b/docs/c10-evidence-handoff.md @@ -0,0 +1,162 @@ + +# C10 evidence handoff + +**SYNTHETIC reporting evidence only. REAL_EPISODE_NOT_VERIFIED. +PRODUCTION ACCEPTANCE BLOCKED. PR #25 stays draft.** + +This addresses the scoped review of commit +`8440f4b818f4bd0e6bbbbcc1538eee0555e0012a` on +[PR #25](https://github.com/openAMRobot/openamrobot-interfaces/pull/25). +The review recommends keeping this as a conformance-reference consumer under +`tools/` and `tests/` while runtime ownership is agreed separately. It does not +approve a runtime repository, numeric policy, source profile or release pin. + +## Owner decisions + +The work-package owner's explicit **adopt / adapt / reject** decision has not +been recorded. Instructions to prepare this evidence are not substituted for +that decision. The following checklist remains open: + +- [ ] SatyaA27 records adopt, adapt or reject in the PR thread. +- [ ] Architecture/runtime owners confirm the runtime home and adoption scope. +- [ ] Release owner supplies the validated LeRobot release/commit and evidence. +- [ ] Recording/profile owners approve timing/FPS/freshness policy and action + dimensions, order, units, bounds and observation/timestamp relationship. +- [ ] Calibration/configuration owners provide versioned source-profile and + calibration/configuration artifacts, exact-byte hashes and residual criteria. +- [ ] Recording owner supplies an actual native episode revision, matching C9 + sidecar and immutable artifacts for the next real-episode acceptance report. + +No baseline defaults fill these decisions. Existing synthetic test-policy values +are reused only to exercise reporting and remain labelled synthetic. Contributor +agreement and other rights-holder authorization records remain unverified. + +## Reproduce the synthetic CLI report + +Check out the exact `checker.git_head` found in the downloaded evidence manifest, +then install the existing scoped native-test dependencies. Run from that checkout: + +```bash +python3.12 -m venv /tmp/openamrobot-c10-handoff +/tmp/openamrobot-c10-handoff/bin/python -m pip install -r tools/requirements-c10-native.txt +/tmp/openamrobot-c10-handoff/bin/python tools/generate_c10_evidence.py \ + --output-dir .verification/c10-evidence/synthetic \ + --expected-checker-sha "$(git rev-parse HEAD)" +``` + +Use a fresh output directory for each invocation. The helper refuses to overwrite +an existing directory; it writes generated dataset/report artifacts only inside +its new bundle. It reuses +the C9-derived two-frame fixture and native Parquet writer in +`tests/c10_fixture.py`, invokes the actual checker CLI, and retains the unmodified +checker report. No real recording is discovered, downloaded or modified. + +For byte comparisons with an earlier run, use the Python/package versions in +`evidence.json` under `environment`, especially its `pyarrow` version: the +dependency ranges alone are not an environment lock. Absolute command paths and +manifest creation time describe each invocation and can differ; the raw fixture +report and dataset hashes reproduce across output directories in the tested +environment. No new project release pin is created by recording test versions. + +The helper records the exact subprocess argument vector, shell-readable command +and working directory. Repeating that captured command in the original checkout +re-runs the checker against the retained generated dataset. When moving an +artifact to another machine, regenerate the bundle at the recorded checker SHA +or adjust only the local paths to the downloaded dataset/policy/report. Absolute +interpreter and workspace paths in the manifest describe the original run. + +`--expected-checker-sha` requires the actual checkout to match that full SHA and +the source tree to be clean. Without that flag a local dirty checkout is recorded +transparently; it is not immutable exact-SHA evidence. CI always supplies the +expected PR-head SHA. Source-file hashes further identify the checker, semantic +implementation, loader, C9 schema/example and fixture/evidence generator bytes. + +The fixture identity is dataset +`openamrobot/synthetic-c9-contract-fixture`, revision `synthetic-fixture-1`, +episode `synthetic-episode-000000`, index 0. Its timing, configuration and +calibration numbers are synthetic inputs, not hardware measurements or approvals. + +## Downloadable workflow evidence + +Open the **C10 reference quality gate** run linked from PR #25's checks, verify +its head SHA, and download: + +```text +c10-evidence-- +``` + +GitHub retains this artifact for **30 days**; authenticated artifact-download +access may be required. The PR handoff records the exact successful run and +artifact links after CI finishes. Local ignored paths are supplemental evidence, +not the reviewer handoff. When retention expires, the command above recreates +the bundle from the recorded source revision and dependency versions. + +The artifact contains: + +| Path | Evidence | +| --- | --- | +| `expected-checker-sha.txt` | Requested immutable PR-head source revision. | +| `core-suite.log` | Complete core-suite output and executed test count. | +| `native-suite.log` | Complete native-suite output, including media capability/result branch. | +| `evidence-suite.log` | Evidence-helper and workflow failure-retention regression output. | +| `revert-negative.log` | Current-code positive test and intended failure with a reverted arithmetic rule. | +| `generation.log` | Evidence-generation command output, including failure diagnostics. | +| `synthetic/evidence.json` | Actual checker SHA/source state, exact CLI command, dataset revision, environment, source/dataset/report hashes, actual gate verdict and separate reporting-path result. | +| `synthetic/c10-report.json` | Original strict JSON report from the actual C10 CLI. | +| `synthetic/cli-stdout.log`, `synthetic/cli-stderr.log` | Original CLI process output. | +| `synthetic/dataset/` | Generated native Parquet metadata/data, matching C9 sidecar and explicitly synthetic policy for reproduction. | + +The report is expected to have `status: NOT_VERIFIED`, `exit_code: 2`, +`training_ready: false` and `production_acceptance: NOT_VERIFIED`. Native fixture +frame coverage and selected-time skew are measured only from those synthetic +inputs. The generator exits zero only when the expected blocked reporting path +was observed and the generated dataset remained unchanged. Its success does +**not** convert the checker verdict to PASS. Unexpected PASS, REJECT, a missing +or inconsistent report, checker revision mismatch, or changed input bytes fails +the generator and retains available diagnostics. + +All test/evidence steps preserve nonzero results through `tee` using `pipefail`. +They continue after an earlier check failure unless the run is cancelled. The +upload step uses `if: always()` and includes hidden `.verification` files, so +available logs, raw rejected reports and partial manifests remain downloadable +when checks fail. There is no `continue-on-error` and no blanket acceptance of +exit 2 for arbitrary checker invocations. Upload success cannot erase a failed +test or generation step. Missing evidence files are reported rather than +invented; cancellation or runner failure may prevent their creation. + +## Verification and scope + +```bash +/tmp/openamrobot-c10-handoff/bin/python tools/run_verification_tests.py tests 'test_c10*.py' +/tmp/openamrobot-c10-handoff/bin/python tools/run_verification_tests.py tests test_compatibility.py +/tmp/openamrobot-c10-handoff/bin/python -m unittest discover -s tests -p test_schema_validation.py -k SchemaTests -v +/tmp/openamrobot-c10-handoff/bin/python tools/prove_c10_negative.py +``` + +The new evidence tests preserve the existing core/native corruption suites and +exercise raw blocked reports, exact command/revision/hash binding, deterministic +report reproduction, output-directory protection, unexpected gate outcomes, +missing/malformed evidence and workflow failure/pipe behavior. Exact final counts, +commands, head/base SHAs and downloadable artifacts are recorded on PR #25 after +verification. [Earlier execution evidence](c10-verification.md) remains historical +and is not substituted for the latest exact-head CI result. + +Local follow-up on macOS with the existing Python 3.12.13 environment passed +**117 C10 tests** (63 core, 31 native, 23 evidence), **14 compatibility tests**, +**10 schema-only tests**, and the unchanged registered C9 validation (1 schema, +10 assets). The arithmetic revert proof still passed: its single negative test +passes on current code and fails once under the temporary reverted rule. +The evidence guard was also reverted in a temporary module copy by replacing +`if not expected_gate_result(report, manifest["gate"]["process_exit_code"]):` +with `if False:`. Unchanged +`C10EvidenceTests.test_rejects_unexpected_gate_pass` then failed exactly once at +`assertNotEqual(helper_exit, 0)`, with no import errors or skipped tests. Neither +experiment changed tracked source or fixture expectations. Actual video decoding +and full ROS Jazzy integration remain hosted checks because this Mac lacks +ffmpeg/ffprobe and ROS Jazzy/colcon; explicit unavailable branches are not media +or ROS execution evidence. + +No dependency manifest, checker verdict logic, accepted C9 schema, frozen +profile or runtime policy is changed by this follow-up. The only generated +recordings are temporary synthetic fixtures. Full real-episode and production +acceptance remain unresolved regardless of a successful evidence handoff. diff --git a/docs/c10-provenance.md b/docs/c10-provenance.md new file mode 100644 index 0000000..b668923 --- /dev/null +++ b/docs/c10-provenance.md @@ -0,0 +1,86 @@ +# C10 provenance and evidence limits + +Audit date: 16 October 2026. This is documentation for a draft C9 conformance +reference pending architecture review, not a claim of runtime or physical +acceptance. + +| Inspected source | Revision and scope | +| --- | --- | +| [Interfaces repository](https://github.com/openAMRobot/openamrobot-interfaces/tree/87ab8a936c97ab55ee74b8988ca3d7659160ea6a) | Parent `87ab8a936c97ab55ee74b8988ca3d7659160ea6a`; README boundaries, C9 schema, contract document and synthetic fixture. | +| [Accepted C9 PR #16](https://github.com/openAMRobot/openamrobot-interfaces/pull/16) | Merged 2 October 2026 as `fa7c438e33bd807c174fa0747119d7d627faa3bf`. [Acceptance](https://github.com/openAMRobot/openamrobot-interfaces/issues/18#issuecomment-5919324379) covers the v1 sidecar shape only. | +| [Dataset QA baseline](https://github.com/4u-79/openamrobot-c-baseline/tree/a29ce7c89b7c42c9147ad81e6a751de9dad045c9) | `a29ce7c89b7c42c9147ad81e6a751de9dad045c9`; read-only inspection of `scripts/check_dataset_integrity.py`, `scripts/validate_dataset_contract.py`, `tests/test_c10_fixtures.py`, `meta/info.json`, requirements and README. No baseline test run or real-episode pass is claimed. | +| [Organization rules](https://github.com/openAMRobot/.github/blob/539da1982bccf8e2315483f41ac16734e0bed328/AGENTS.md) and [decisions](https://github.com/openAMRobot/.github/blob/539da1982bccf8e2315483f41ac16734e0bed328/decisions.yaml) | `539da1982bccf8e2315483f41ac16734e0bed328`; source documents remain provenance, approved technical values belong in the register. Historical fixed-mast and current lift profiles must remain distinct. | +| [LeRobot source snapshot](https://github.com/huggingface/lerobot/tree/7e241bd630a3719a56157a497ce5d08f244784f1) | C9's previously inspected v0.6.1 snapshot, Apache-2.0; not a newly verified runtime dependency or project release pin. | + +## Reuse and licensing + +The baseline provides useful fault classes and comparison points: temporal +regression, frame loss, phase lag, frozen images, corrupt storage, malformed +dimensions, nonfinite data and orphan tasks. These ideas inform independent +C9-specific checks; baseline implementation and test source are not copied or +vendored. + +The positive fixture is derived at test time from the accepted C9 YAML using a +deep copy. It supplies explicitly synthetic reviewed-policy inputs and adjusts +the copied event interval to contain its linked frame's observation timestamp. +The canonical C9 example remains unchanged: structural validity alone does not +promise a positive C10 semantic verdict. + +## Dependencies + +| Scoped dependency | Source / licence | Change | +| --- | --- | --- | +| jsonschema `>=4.18,<5` | [PyPI](https://pypi.org/project/jsonschema/), MIT | Declares the existing schema-validator dependency for standalone C10 installs. | +| PyYAML `>=6,<7` | [PyPI](https://pypi.org/project/PyYAML/), MIT | Declares the existing strict YAML-loader dependency. | +| pyarrow `>=17,<27` | [Apache Arrow/PyPI](https://pypi.org/project/pyarrow/), Apache-2.0 | Optional native Parquet inspection/integration manifest, not a training dependency. | +| ffmpeg / ffprobe | [FFmpeg](https://ffmpeg.org/legal.html), LGPL-2.1-or-later or GPL depending on build | Optional external media inspection; hosted test job installs the distribution package. No binary vendored or runtime release pin selected. | + +Only C10 install manifests and its new verification job declare dependencies. +Existing packages and training requirements are not upgraded. Standard-library +code handles reports, exact decimal mapping, SHA-256 and process invocation. +These dependency ranges are tool-install constraints, not LeRobot release-owner +approval. Exact locally tested versions are in the execution evidence. + +The inspected baseline commit contains no tracked `LICENSE` or `LICENSING.md` +and its README states no source license. Its public availability therefore +does not establish permission to copy its code. The license ambiguity remains +open; no license grant or ownership assignment is inferred. This audit does +not resolve third-party rights. + +Repository licensing follows [LICENSING.md](../LICENSING.md): software MIT, +documentation CC BY 4.0. The [contributor agreement process](https://github.com/openAMRobot/.github/blob/539da1982bccf8e2315483f41ac16734e0bed328/CLA.md) +requires DCO and the applicable agreement, including authorization where other +rights holders control the work. The old C9 PR reports checks for its own head; +that does not attest agreement coverage or institutional authorization for this +new contribution. Those private legal records were not available in this audit +and no attestation is made on the contributor's behalf. + +## Search evidence and open ownership + +Read-only GitHub API searches covered the accessible organization repository +list, `org:openAMRobot C10`, `org:openAMRobot "dataset"`, C10 references in +the organization policy repository, issue #18 and its comments, and PR #16. +The accessible repository list contains interfaces, platform, manipulation, +upperbody, documentation, manifest and release repositories, with no dedicated +AI/dataset repository. No approved C10 runtime home was found. Issue #18 +explicitly leaves recorder ownership open; its +[13 October comment](https://github.com/openAMRobot/openamrobot-interfaces/issues/18#issuecomment-6152412257) +requires a compatibility matrix and one real episode report. Search absence +does not establish that private or unpublished decisions do not exist. + +The local search was limited to the authorized project workspace. It used +`rg --files --hidden` with `.git`, `node_modules`, `.venv` and `.pixi` +excluded, looking for `meta/info.json`, sidecar, roster, agreement and agent-rule +filenames. It found no actual native recording metadata/sidecar pair or team +roster; the existing C9 fixture is synthetic. Ignored files and remote/private +storage were not exhaustively searched. The public organization contributor +list is not the workstream roster. No real dataset, calibration artifact or +private agreement record was supplied through these sources. + +The interfaces README reserves this repository for shared contracts and puts +application implementations elsewhere. Keeping this draft under existing +`tools/` and `tests/` is a reference conformance check while the runtime-home +decision remains open; it does not approve ingestion, replay or training code +here. Runtime adoption requires the work-package owner's explicit decision +and review. No real-episode, physical calibration, training readiness or +release acceptance claim follows from the synthetic results. diff --git a/docs/c10-readme.md b/docs/c10-readme.md new file mode 100644 index 0000000..ac247ec --- /dev/null +++ b/docs/c10-readme.md @@ -0,0 +1,194 @@ + +# C10 Dataset Quality Gate reference + +**Draft placement pending architecture review. SYNTHETIC QA is not REAL DATA +VALIDATION or PRODUCTION ACCEPTANCE.** Owner: SatyaA27. Reviewer: +panthera-momagdii. Target date: 16 October 2026. + +This read-only Python consumer checks the accepted C9 v1 sidecar against native +LeRobot metadata and available artifacts. It creates no dataset format, writes +no recording, and requires no ROS, Gazebo, LeRobot installation or GPU. The +canonical schema and native writer remain unchanged. See the +[compatibility matrix](c10-compatibility-matrix.md) and +[Integration Gate/provenance audit](c10-provenance.md). + +## Install and run on macOS + +From the repository root, with Python 3.12 or later: + +```bash +python3.12 -m venv /tmp/openamrobot-c10 +/tmp/openamrobot-c10/bin/python -m pip install -r tools/requirements-c10.txt +/tmp/openamrobot-c10/bin/python tools/run_verification_tests.py tests test_c10_dataset.py +# Optional native Parquet inspection and integration tests: +/tmp/openamrobot-c10/bin/python -m pip install -r tools/requirements-c10-native.txt +/tmp/openamrobot-c10/bin/python tools/run_verification_tests.py tests 'test_c10*.py' +/tmp/openamrobot-c10/bin/python tools/check_c10_dataset.py \ + --dataset-path /absolute/path/to/native-dataset \ + --policy /absolute/path/to/reviewed-c10-policy.json \ + --report /tmp/c10-report.json +``` + +`--policy` is optional: omitting it leaves evidence-dependent decisions +`NOT_VERIFIED`. Repeat `--episode 0 --episode 7` to inspect selected episodes; +omitting selection requires a sidecar for every native episode. Report output +must be outside the dataset. No missing test dependencies are silently skipped. + +Native v3 layout: `meta/info.json`, `meta/tasks.parquet`, +`meta/episodes/chunk-*/file-*.parquet`, native `data_path` and `video_path` +templates, and C9 sidecars at +`meta/openamrobot/episodes/episode-{episode_index:06d}.json` (YAML also accepted). +Legacy JSONL metadata is read when Parquet metadata is absent, without calling +that layout a verified v3 writer. Shared native video files require explicit +`videos//{chunk_index,file_index,from_timestamp,to_timestamp}` episode +bindings; video segments use a half-open timestamp range. Native data rows use +`data/chunk_index`, `data/file_index` and their episode index, with native global +row ranges checked when present. File paths must stay inside the dataset root. + +Parquet inspection requires `pyarrow`. Video frame inspection requires `ffprobe`; +exact decoded pixel hashes additionally require `ffmpeg` and a reviewed repeated +image policy. These executables can be installed through the user's normal +package manager. Absence is reported explicitly, including when the integration +test exercises the unavailable-tool branch. Frame count/dimension checks decode +frame metadata, rather than trusting a container's nominal frame-count field. + +## Verdicts and report + +| Exit | Verdict | Consumer behavior | +| --- | --- | --- | +| 0 | PASS | All implemented required conformance/evidence checks completed. Subject to human adoption and release acceptance. | +| 1 | REJECT | Contradictory, malformed or corrupt input. Block ingestion into training. | +| 2 | NOT_VERIFIED / BLOCKED | Required evidence or inspection capability missing. Block training. Also used for CLI usage/report-output failure. | + +REJECT takes precedence over NOT_VERIFIED. Each finding has a stable check ID, +severity, status, episode/frame/stream context, observed/expected evidence, +units, threshold, policy provenance and suggested fix. Metrics are emitted only +for evidence actually inspected: native frame coverage, selected-time intervals, +source sequence gaps/reuse, stale runs, measured source FPS, inter-observation +skew and separate action latency. Nominal native output FPS and configured +camera FPS remain distinct from measured FPS. Clock decimal lexemes are retained +for exact affine arithmetic; output represents exact fractional clock values as +decimal strings. Nonfinite vector evidence is represented as strings in strict +JSON while still producing REJECT. + +An abbreviated synthetic report looks like: + +```json +{ + "report_version": 1, + "status": "NOT_VERIFIED", + "exit_code": 2, + "training_ready": false, + "production_acceptance": "NOT_VERIFIED", + "findings": [{ + "check_id": "C10_REAL_EPISODE", + "status": "NOT_VERIFIED", + "severity": "BLOCKER", + "episode": 0, + "frame": null, + "stream": null, + "observed": "synthetic", + "expected": "actual recording with provenance reviewed externally", + "units": null, + "threshold": null, + "policy": null, + "fix": "Provide a real recorded episode; synthetic fixtures establish QA behavior only." + }] +} +``` + +The [evidence handoff](c10-evidence-handoff.md) documents a single command that +generates the complete synthetic CLI report, exact checker SHA/command and +reproduction inputs. The C10 workflow uploads these and native-suite output as +downloadable artifacts, including on failure. The report stays blocked even +when synthetic semantic checks pass. Earlier local results remain in the +[execution evidence](c10-verification.md). + +## Reviewed policy input + +Policy JSON is a local, proposed checker configuration, not an additional native +LeRobot or C9 contract. `approved: true`, a nonempty `id` and `evidence` identify +an approval supplied by the caller; the tool cannot grant or authenticate human +approval. Release owners must review these artifacts before deployment. There +are **no built-in hardware thresholds, action bounds or universal vector widths**. + +The complete executable **synthetic-only** example is +[`synthetic_policy()`](../tests/c10_fixture.py). Never promote its numbers, +hashes, profiles or approval flag into recording policy. A real reviewed policy +supplies: + +- `profiles`: exact state/action feature-key mapping to versioned `profile`, + ordered `dimensions` (name/unit/meaning) and `reference_frame`. +- `source_profile`: exact C9 provenance `id` and an `artifact` with `path` and + `sha256`. No unit coercion or camera aliases are applied. +- `synchronization.max_skew_ns`: accepted inter-observation bound. C9 must + declare the matching bound. Pending C9 tolerance stays NOT_VERIFIED even when + a policy supplies a number. Action timing is excluded from observation skew. +- `action_latency`: inclusive `min_ns`/`max_ns`, exact declared `values`, + `timestamp`, `observation_relationship` and an explicit + `reference_frame_offset` identifying the linked observation. Missing reference + evidence remains unverified; free-text relationships are not inferred. +- `action_bounds`: every action dimension name mapped to authoritative `min`, + `max` and exact `unit`. Limits are inclusive. +- `stream_quality`: each stream maps to `min_fps`, `max_freshness_ns`, + `max_sequence_gap`, `max_reuse`. Partial policy leaves the decision unverified. + This assesses observed timing/IDs; a host-arrival timestamp remains an arrival + proxy, never upgraded to physical exposure evidence. +- `video_content`: per-camera `max_identical_run` in decoded frames. This + requires explicit review because a static scene can produce identical images. +- `artifacts`: records with `kind` (`robot_configuration`, + `stream_configuration` or `calibration`), `path`, `sha256`, and `stream` except + for the robot configuration. Calibration also supplies `id` and `version`. + Hashes use `sha256:` followed by the digest of exact bytes; no normalization. +- `calibration`: per-camera `name`, `unit`, `statistic`, `method`, `max_value` + for a measured residual. Matching IDs and bytes cannot prove physical accuracy. +- `release`: `pin` equal to the C9 validated release object, and `version` and + `commit` equal to that pin and the actual `lerobot` software provenance entry. + Inspected baselines and `unknown` versions do not qualify. +- `native_identity`: reviewed `dataset_id`, `dataset_revision`, + `episode_ids` mapping decimal episode index strings to IDs, and `evidence` + identifying the immutable native snapshot. This supplies bindings absent from + standard native metadata without injecting fields into `meta/info.json`. + Explicit native contradictions reject the data. +- `real_episode`: full `dataset_id`, `dataset_revision`, `episode_id`, + `episode_index`, plus external recording-review `evidence`. A producer's + `data_origin: actual` claim alone cannot qualify. + +All referenced artifact paths are dataset-relative, immutable inputs. A policy +is provenance supplied to the checker, not a replacement for organization +change control. Repository and release acceptance remain human decisions. + +## Recorder and C11 invocation + +Finalize the recording and native metadata, retain the exact source artifacts, +and export the matching C9 sidecar. Invoke this checker with the reviewed policy +and persist the report outside the recording. A consumer may start training +only after checking **both** exit code 0 and `status == "PASS"`; exit 1 or 2 +must block it. Capture reports against an immutable dataset revision and include +the checker revision and reviewed policy ID in the training job provenance. +Do not rewrite data or manufacture timing/calibration evidence to open the gate. +Safety events are recorded observations, never safety authorization. + +## Acceptance and limitations + +The [check-to-corruption matrix](c10-check-fixture-matrix.md) lists the named +tests for every diagnostic family. C10-only mutations are validated against the +unchanged C9 schema before checking the expected diagnostic. Native integration +tests use temporary synthetic artifacts; no real recording is corrupted. + +REAL_EPISODE_NOT_VERIFIED: no real native episode with matching sidecar was +found in the authorized workspace search. The two-frame C9 example and generated +Parquet are synthetic. A claimed source/profile hash cannot establish hardware +authenticity. Exact pixel equality detects repeated images, not every frozen, +miscalibrated or semantically wrong recording. No training or policy evaluation +was executed. The checker reads the referenced episode artifacts in memory; +large-scale streaming ingestion remains runtime work. No remote dataset is +downloaded and no remote release pin is authenticated automatically. + +Runtime repository ownership, release pin, timing/action/source profiles, +calibration criteria and the next real-episode evidence require reviewer/owner +confirmation. The interfaces location is a draft conformance reference under +the existing verification structure. No recorder or training implementation is +introduced. Full October 16 production acceptance is **not satisfied** by this +reference implementation. ROS Jazzy integration and hosted CI results must be +reported separately from Mac Python tests. diff --git a/docs/c10-verification.md b/docs/c10-verification.md new file mode 100644 index 0000000..f6f1601 --- /dev/null +++ b/docs/c10-verification.md @@ -0,0 +1,136 @@ + +# C10 execution evidence — 16 October 2026 + +**SYNTHETIC QA PASS. REAL_EPISODE_NOT_VERIFIED. PRODUCTION ACCEPTANCE BLOCKED.** + +Repository: `openAMRobot/openamrobot-interfaces`. Branch: +`c10-dataset-quality-gate`. Parent/base: +`87ab8a936c97ab55ee74b8988ca3d7659160ea6a`, freshly fetched from upstream and +confirmed unchanged after verification. The draft PR records its actual head SHA; +embedding a commit's own SHA in its tracked content is not possible. + +The initial clean checkout was `c9-lerobot-dataset-contract` at +`989b963f324b2e57c644edaa59e312a786b747e4`. A fresh scoped branch was created +from upstream main. Existing files/worktrees and the inspected baseline checkout +were preserved. No C9 schema/fixture, ROS interface, frozen contract, governance, +safety-path, release pin or baseline source was changed. + +Environment: macOS 26.5.1, Apple arm64; Python 3.12.13; jsonschema 4.26.0, +PyYAML 6.0.3, pyarrow 26.0.0. Dependencies were installed only in an isolated +temporary venv. No ROS/Gazebo/GPU dependencies were installed for C10. + +## Commands and results + +Commands below ran from the repository root. Python suite output and full JSON +report are retained locally under ignored `.verification/c10-evidence/`. + +```bash +git fetch upstream main +git switch -c c10-dataset-quality-gate upstream/main +uv venv --python python3.12 /private/tmp/openamrobot-c10-venv +UV_CACHE_DIR=/private/tmp/openamrobot-c10-uv-cache uv pip install \ + --python /private/tmp/openamrobot-c10-venv/bin/python jsonschema PyYAML pyarrow + +/private/tmp/openamrobot-c10-venv/bin/python tools/run_verification_tests.py tests test_c10_dataset.py +/private/tmp/openamrobot-c10-venv/bin/python tools/run_verification_tests.py tests test_c10_native.py +/private/tmp/openamrobot-c10-venv/bin/python tools/run_verification_tests.py tests test_compatibility.py +/private/tmp/openamrobot-c10-venv/bin/python -m unittest discover -s tests -p test_schema_validation.py -k SchemaTests -v +/private/tmp/openamrobot-c10-venv/bin/python tools/prove_c10_negative.py +/private/tmp/openamrobot-c10-venv/bin/python -c 'from pathlib import Path; import sys; sys.path.insert(0,"tools"); from lint_schemas import validate_schemas; print(validate_schemas(Path.cwd()))' +git diff --check +``` + +| Verification | Result / executed count | +| --- | --- | +| C10 core suite | PASS, 63 tests, zero skips | +| C10 native-artifact suite | PASS, 31 tests, zero skips; includes actual temporary Parquet reads/corruption and orphan rows in shared files. Video-unavailable branch explicitly exercised locally. | +| Existing compatibility suite | PASS, 14 tests | +| Existing schema-only regression class | PASS, 10 tests | +| Existing registered schema validation | PASS, 1 schema / 10 registered assets (schema plus 1 valid and 8 invalid fixtures) | +| Reverted affine rule | Current code: 1 unchanged negative test PASS. Temporary reverted rule: that same test FAILED with exactly 1 assertion failure and no errors/skips. Proof command exit 0. | +| Whitespace/scope review | PASS; canonical C9 assets untouched | + +The C10 matrices contain 46 C9-valid semantic corruption cases and 21 named +wrapper corruption/missing-evidence cases across all 28 diagnostic families. +Other tests exercise clean PASS, boundaries, missing approval and environmental +evidence. Counts are unittest test methods; subcases are not inflated into +additional tests. See the [acceptance matrix](c10-check-fixture-matrix.md). + +The exact reverted comparison is +`calculated == stamp.get("mapped_authoritative_ns")`. The experiment replaces +only that expression in a temporary module copy. The unchanged +`test_reject_affine_arithmetic` fails with +`Expected C10_MAPPING/REJECT` because the bad declared mapping is accepted. +An import failure, wrong test count, skip or unexpected error fails the proof. + +Also executed: + +```bash +/private/tmp/openamrobot-c10-venv/bin/python tools/run_verification_tests.py tests test_schema_validation.py +``` + +This full existing suite executed **15 tests and failed with 3 errors**, each +`ModuleNotFoundError: No module named 'catkin_pkg'` in ROS-interface verification. +It is not recorded as a pass. The schema-only class and registered C9 validation +were then verified independently as shown above. The Mac lacks +`/opt/ros/jazzy/setup.bash` and `colcon`, so `bash tools/verify.sh` was not run. +It requires the documented Ubuntu/Jazzy environment. Existing ROS stages remain +intact; hosted `quality/build` must complete them on the PR head. The added C10 +workflow installs scoped Python/native inspection dependencies and ffmpeg and +runs the nonempty C10 suites. Hosted results are separate evidence. + +## Synthetic native CLI report + +A new temporary directory was created using `tempfile.mkdtemp`; the fixture +generator wrote only there. Reproduce the generation from the repository root: + +```python +from pathlib import Path +import sys +import tempfile +sys.path.insert(0, "tests") +from c10_fixture import fixture, write_dataset +sidecar, native, policy = fixture() +dataset = Path(tempfile.mkdtemp(prefix="c10-synthetic-native-")) +write_dataset(dataset, sidecar, native, policy) +print(dataset) +``` + +The actual CLI command used the printed path: + +```bash +/private/tmp/openamrobot-c10-venv/bin/python tools/check_c10_dataset.py \ + --dataset-path /var/folders/6y/841vvczx5g50bx_3cgjnvtg40000gn/T/c10-synthetic-native-rcy0er3d \ + --policy /var/folders/6y/841vvczx5g50bx_3cgjnvtg40000gn/T/c10-synthetic-native-rcy0er3d/synthetic-policy.json \ + --report .verification/c10-evidence/synthetic-report.json +``` + +Dataset ID: `openamrobot/synthetic-c9-contract-fixture`; revision: +`synthetic-fixture-1`; episode: `synthetic-episode-000000`, index 0. +Result: exit **2**, **NOT_VERIFIED**, `training_ready: false` and +`production_acceptance: NOT_VERIFIED`. Native frame coverage: 2/2; +inter-observation skew measured from selected fixture times: 0 ns for both +frames. These values are deterministic fixture arithmetic, not recording +measurements. Full findings distinguish the completed synthetic checks from +unavailable calibration/config/profile bytes, release and real-episode evidence, +and media inspection. No real recordings were used or changed. + +## Not verified + +- **REAL_EPISODE_NOT_VERIFIED:** no actual native episode plus matching sidecar + found in the bounded authorized workspace search. No dataset revision exists + here that can be reported as real-data validation. +- Actual MP4 decoding/frozen-image analysis was not run locally: ffmpeg and + ffprobe are absent. Mocked decoder metadata/content checks do not count as + actual decoding. The hosted workflow supplies these tools for the real + synthetic-media branch, pending its result. +- Runtime repository ownership/adoption, release-owner pin, timing/action/source + profiles, calibration/configuration source bytes and physical acceptance. +- Full ROS Jazzy integration on this Mac and hosted CI on the final PR head. +- Private contributor-agreement/third-party authorization records. The draft PR + makes no agreement-acceptance or institutional-authorization attestation. + +The October 16 deliverable is prepared as a reviewable conformance implementation +with synthetic evidence. Its real-episode and production acceptance obligations +remain unsatisfied. The reviewer must confirm runtime ownership, approved +profiles/pin and the next real-recording evidence before adoption. diff --git a/tests/c10_fixture.py b/tests/c10_fixture.py new file mode 100644 index 0000000..8d9a104 --- /dev/null +++ b/tests/c10_fixture.py @@ -0,0 +1,138 @@ +# SPDX-License-Identifier: MIT +"""Deterministic, explicitly synthetic C10 inputs derived from the C9 example. + +No value here constitutes an approved recording threshold, calibration, device +specification, configuration identity, or production release pin. +""" + +from copy import deepcopy +import json +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +SYNTHETIC_HASH = "sha256:" + "1" * 64 +CAMERAS = ("camera_wrist_left", "camera_wrist_right", "head", "base") + + +def fixture(): + """Return independent sidecar/native/policy objects for a two-frame episode.""" + sidecar = deepcopy(yaml.safe_load( + (ROOT / "schemas/yaml/c9-lerobot-dataset-valid.yaml").read_text() + )) + capture = sidecar["openamrobot"] + capture["synchronization"] = { + "status": "declared", + "tolerance_ns": 100, + "basis": "SYNTHETIC TEST ONLY: no physical evidence or production approval.", + } + for name, stream in capture["streams"].items(): + stream["configuration_hash"] = SYNTHETIC_HASH + if name in CAMERAS: + stream["calibration"]["residual"] = { + "status": "measured", "name": "synthetic_reprojection_error", + "value": 0.25, "unit": "pixel", "statistic": "rms", + "method": "synthetic deterministic test fixture; no measurement", + } + capture["provenance"]["robot_configuration_hash"] = SYNTHETIC_HASH + # The C9 example is structural only: its marker starts after linked frame 0. + # Include that observation time to make this a positive C10 linkage fixture. + capture["events"][0]["start_timestamp_ns"] = 0 + manifest = sidecar["native_lerobot"] + features = { + key: {"dtype": descriptor["dtype"], "shape": deepcopy(descriptor["shape"])} + for key, descriptor in manifest["features"].items() + } + # Native LeRobot vector names carry the same ordering as C9 dimensions. + for key in ("observation.state", "action"): + features[key]["names"] = [d["name"] for d in manifest["features"][key]["dimensions"]] + for key, descriptor in features.items(): + if descriptor["dtype"] == "video": + descriptor["names"] = ["height", "width", "channels"] + native = { + "info": { + "codebase_version": manifest["format_version"], + "dataset_id": manifest["dataset_id"], + "dataset_revision": manifest["dataset_revision"], + "fps": manifest["fps"], "features": features, + "total_episodes": 1, "total_frames": 2, "total_tasks": 1, + "data_path": "data/chunk-{chunk_index:03d}/file-{file_index:03d}.parquet", + }, + "episode": { + "episode_index": 0, "episode_id": manifest["episode_id"], + "length": 2, "tasks": [manifest["task"]["instruction"]], + "data/chunk_index": 0, "data/file_index": 0, + "dataset_from_index": 0, "dataset_to_index": 2, + }, + "tasks": {0: manifest["task"]["instruction"]}, + "rows": [ + {"episode_index": 0, "frame_index": i, "task_index": 0, + "timestamp": i / 30, "index": i, + "observation.state": [float(i)] * 16, + "action": [float(i + 1)] * 16} + for i in range(2) + ], + } + return sidecar, native, synthetic_policy(sidecar) + + +def synthetic_policy(sidecar): + """Explicit test thresholds; ``approved`` is only the checker input flag.""" + features = sidecar["native_lerobot"]["features"] + return { + "id": "SYNTHETIC-TEST-ONLY-never-production-approval", + "evidence": "Deterministic synthetic unit tests; no physical validation.", + "approved": True, + "profiles": { + key: {"profile": features[key]["profile"], + "dimensions": deepcopy(features[key]["dimensions"]), + "reference_frame": None} + for key in ("observation.state", "action") + }, + "action_bounds": { + d["name"]: {"min": -180, "max": 180, "unit": d["unit"]} + for d in features["action"]["dimensions"] + }, + "synchronization": {"max_skew_ns": 100}, + "stream_quality": { + name: {"min_fps": 29, "max_freshness_ns": 100, + "max_sequence_gap": 0, "max_reuse": 0} + for name in sidecar["openamrobot"]["streams"] + }, + "action_latency": { + "min_ns": 0, "max_ns": 2000, + "reference_frame_offset": 0, + "values": sidecar["openamrobot"]["action_semantics"]["values"], + "timestamp": sidecar["openamrobot"]["action_semantics"]["timestamp"], + "observation_relationship": sidecar["openamrobot"]["action_semantics"][ + "observation_relationship"], + }, + } + + +def write_dataset(path, sidecar, native, policy=None): + """Write actual Parquet metadata/data, keeping the dependency explicit.""" + import pyarrow as pa + import pyarrow.parquet as pq + + path = Path(path) + (path / "meta/episodes/chunk-000").mkdir(parents=True, exist_ok=True) + (path / "data/chunk-000").mkdir(parents=True, exist_ok=True) + (path / "meta/openamrobot/episodes").mkdir(parents=True, exist_ok=True) + (path / "meta/info.json").write_text(json.dumps(native["info"])) + tasks = [{"task_index": i, "task": task} for i, task in native["tasks"].items()] + pq.write_table(pa.Table.from_pylist(tasks), path / "meta/tasks.parquet") + pq.write_table(pa.Table.from_pylist([native["episode"]]), + path / "meta/episodes/chunk-000/file-000.parquet") + table = pa.Table.from_pylist(native["rows"]) + for key in ("observation.state", "action"): + column_index = table.column_names.index(key) + values = [row[key] for row in native["rows"]] + table = table.set_column(column_index, key, pa.array(values, type=pa.list_(pa.float32()))) + pq.write_table(table, path / "data/chunk-000/file-000.parquet") + (path / "meta/openamrobot/episodes/episode-000000.json").write_text(json.dumps(sidecar)) + if policy is not None: + (path / "synthetic-policy.json").write_text(json.dumps(policy)) + return path diff --git a/tests/test_c10_dataset.py b/tests/test_c10_dataset.py new file mode 100644 index 0000000..b39815f --- /dev/null +++ b/tests/test_c10_dataset.py @@ -0,0 +1,433 @@ +# SPDX-License-Identifier: MIT +"""C10 fault injection: every mutation is valid against the unchanged C9 schema.""" + +from copy import deepcopy +import json +from pathlib import Path +import sys +import unittest + +from jsonschema import Draft202012Validator + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools")) +sys.path.insert(0, str(ROOT / "tests")) + +from c10_fixture import fixture +from c10_semantics import CHECK_IDS, check_episode + + +# These names also form the machine-readable acceptance-matrix source. +NEGATIVE_CASES = { + "camera_fps_descriptor": "C10_FEATURES", + "success_with_abort_event": "C10_COMPLETENESS", + "binding_revision": "C10_BINDING", + "feature_shape": "C10_FEATURES", + "task_text": "C10_TASK", + "frame_coverage": "C10_COVERAGE", + "timestamp_order": "C10_TIME_ORDER", + "affine_arithmetic": "C10_MAPPING", + "selection_source": "C10_SELECTION", + "fresh_reused_sample": "C10_HEALTH", + "sequence_reset": "C10_SEQUENCE", + "reported_skew": "C10_SKEW", + "inter_observation_skew": "C10_SYNC", + "sequence_gap": "C10_STREAM_QUALITY", + "vector_width": "C10_VECTOR_VALUES", + "profile_units": "C10_PROFILE", + "action_range": "C10_ACTION_BOUNDS", + "command_latency": "C10_ACTION_LATENCY", + "event_outside_recording": "C10_EVENTS", + "interrupted_recording": "C10_COMPLETENESS", + "mapping_validity_interval": "C10_MAPPING", + "event_missing_frame": "C10_EVENTS", + "vector_nonfinite": "C10_VECTOR_VALUES", + "profile_dimension_order": "C10_PROFILE", + "native_frame_index": "C10_COVERAGE", + "format_version": "C10_BINDING", + "stale_observation_age": "C10_STREAM_QUALITY", + "measured_source_fps": "C10_STREAM_QUALITY", + "action_value_semantics": "C10_ACTION_LATENCY", + "action_timestamp_semantics": "C10_ACTION_LATENCY", + "declared_policy_tolerance_mismatch": "C10_SYNC", + "nonconsecutive_source_id_reuse": "C10_SEQUENCE", + "duplicate_frame": "C10_COVERAGE", + "episode_length": "C10_BINDING", + "fresh_without_receipt": "C10_HEALTH", + "descending_authoritative_time": "C10_TIME_ORDER", + "selected_time_disagrees_with_mapping": "C10_SELECTION", + "legacy_camera_aliases": "C10_FEATURES", + "native_vector_shape": "C10_FEATURES", + "native_vector_name_order": "C10_FEATURES", + "vector_infinity": "C10_VECTOR_VALUES", + "orphan_native_task": "C10_TASK", + "orphan_native_episode": "C10_BINDING", + "inverted_event": "C10_EVENTS", + "duplicate_event_id": "C10_EVENTS", + "external_native_identity_contradiction": "C10_BINDING", +} + + + + +def capture(sidecar, native, policy): + findings = [] + + def emit(check_id, status, observed=None, expected=None, units=None, + threshold=None, policy=None, fix="", frame=None, stream=None): + findings.append({ + "check_id": check_id, "status": status, "observed": observed, + "expected": expected, "units": units, "threshold": threshold, + "policy": policy, "fix": fix, "frame": frame, "stream": stream, + }) + + metrics = check_episode(sidecar, native, policy, emit) + return findings, metrics + + +def set_time(sidecar, frame, stream, timestamp): + """Keep mapping, selection and stored skew consistent for one mutation.""" + sample = sidecar["openamrobot"]["samples"][frame] + item = sample["streams"][stream] + source = item["selection_timestamp_source"] + metadata = sidecar["openamrobot"]["streams"][stream]["timestamps"][source] + mapping = metadata["mapping_to_authoritative"] + stamp = item["device_timestamp" if source == "device" else "host_receipt"] + stamp["raw"] = (mapping["source_origin"] + + (timestamp - mapping["authoritative_origin_ns"]) / + mapping["scale_ns_per_source_unit"]) + stamp["mapped_authoritative_ns"] = timestamp + item["selected_timestamp_ns"] = timestamp + item["measured_skew_ns"] = abs(timestamp - sample["authoritative_timestamp_ns"]) + + +def mutate(name, sidecar, native, policy): + manifest = sidecar["native_lerobot"] + capture = sidecar["openamrobot"] + first, second = capture["samples"] + if name == "camera_fps_descriptor": + native["info"]["features"]["observation.images.head"]["info"] = {"fps":15} + elif name == "success_with_abort_event": + capture["outcome"]["status"] = "success" + capture["events"][0]["type"] = "abort" + elif name == "binding_revision": + native["info"]["dataset_revision"] = "different-synthetic-revision" + elif name == "feature_shape": + native["info"]["features"]["observation.images.head"]["shape"][0] += 1 + elif name == "task_text": + native["tasks"][0] = "A different synthetic task." + elif name == "frame_coverage": + second["frame_index"] = 2 + elif name == "timestamp_order": + second["authoritative_timestamp_ns"] = first["authoritative_timestamp_ns"] + elif name == "affine_arithmetic": + first["streams"]["head"]["device_timestamp"]["mapped_authoritative_ns"] = 1 + elif name == "selection_source": + first["streams"]["head"]["selection_timestamp_source"] = "host_receipt" + elif name == "fresh_reused_sample": + second["streams"]["base"] = deepcopy(first["streams"]["base"]) + second["streams"]["base"]["measured_skew_ns"] = 33333333 + elif name == "sequence_reset": + first["streams"]["base"]["sequence_id"] = 2 + elif name == "reported_skew": + second["streams"]["base"]["measured_skew_ns"] = 1 + elif name == "inter_observation_skew": + set_time(sidecar, 1, "base", second["authoritative_timestamp_ns"] - 101) + elif name == "sequence_gap": + second["streams"]["base"]["sequence_id"] = 2 + elif name == "vector_width": + native["rows"][1]["action"] = [0.0] * 15 + elif name == "profile_units": + manifest["features"]["action"]["dimensions"][0]["unit"] = "radian" + elif name == "action_range": + native["rows"][1]["action"][0] = 181.0 + elif name == "command_latency": + set_time(sidecar, 1, "action", second["authoritative_timestamp_ns"] + 2001) + elif name == "event_outside_recording": + capture["events"][0]["end_timestamp_ns"] = capture["recording"]["end_timestamp_ns"] + 1 + elif name == "interrupted_recording": + capture["recording"]["status"] = "interrupted" + elif name == "mapping_validity_interval": + capture["streams"]["head"]["timestamps"]["device"]["mapping_to_authoritative"][ + "validity"]["source_start"] = 1000001 + elif name == "event_missing_frame": + capture["events"][0]["linked_frame_indices"] = [999] + elif name == "vector_nonfinite": + native["rows"][0]["observation.state"][0] = float("nan") + elif name == "profile_dimension_order": + dimensions = manifest["features"]["action"]["dimensions"] + dimensions[0], dimensions[1] = dimensions[1], dimensions[0] + elif name == "native_frame_index": + native["rows"][1]["frame_index"] = 2 + elif name == "format_version": + native["info"]["codebase_version"] = "v2.1" + elif name == "producer_reports_stale": + second["streams"]["base"]["health"] = "stale" + elif name == "stale_observation_age": + set_time(sidecar, 1, "base", second["authoritative_timestamp_ns"] - 101) + elif name == "measured_source_fps": + capture["recording"]["end_timestamp_ns"] = 50000000 + set_time(sidecar, 1, "base", 40000000) + elif name == "action_value_semantics": + capture["action_semantics"]["values"] = "sent_post_clip" + elif name == "action_timestamp_semantics": + capture["action_semantics"]["timestamp"] = "command_dispatched" + elif name == "declared_policy_tolerance_mismatch": + capture["synchronization"]["tolerance_ns"] = 101 + elif name == "nonconsecutive_source_id_reuse": + third = deepcopy(second) + third["frame_index"] = 2 + third["authoritative_timestamp_ns"] = 66666666 + capture["samples"].append(third) + capture["recording"]["end_timestamp_ns"] = 70000000 + manifest["length"] = native["episode"]["length"] = 3 + native["episode"]["dataset_to_index"] = native["info"]["total_frames"] = 3 + row = deepcopy(native["rows"][1]) + row.update(frame_index=2, index=2, timestamp=2 / 30) + native["rows"].append(row) + for stream, item in third["streams"].items(): + item["sequence_id"] = 2 + item["source_frame_id"] = "synthetic-2" + set_time(sidecar, 2, stream, 66666666 + (1000 if stream == "action" else 0)) + third["streams"]["base"]["source_frame_id"] = first["streams"]["base"]["source_frame_id"] + elif name == "duplicate_frame": + second["frame_index"] = 0 + elif name == "episode_length": + native["episode"]["length"] = 3 + elif name == "fresh_without_receipt": + second["streams"]["base"]["host_receipt"] = { + "status": "unavailable", "reason": "Synthetic receipt evidence missing."} + elif name == "descending_authoritative_time": + first["authoritative_timestamp_ns"] = 33333333 + second["authoritative_timestamp_ns"] = 0 + elif name == "selected_time_disagrees_with_mapping": + second["streams"]["base"]["selected_timestamp_ns"] += 1 + second["streams"]["base"]["measured_skew_ns"] = 1 + elif name == "legacy_camera_aliases": + for key in ("head", "base"): + features = native["info"]["features"] + features[f"observation.images.{key}_camera"] = features.pop(f"observation.images.{key}") + elif name == "native_vector_shape": + native["info"]["features"]["observation.state"]["shape"] = [15] + elif name == "native_vector_name_order": + names = native["info"]["features"]["action"]["names"] + names[0], names[1] = names[1], names[0] + elif name == "vector_infinity": + native["rows"][0]["action"][0] = float("inf") + elif name == "orphan_native_task": + native["rows"][1]["task_index"] = 999 + elif name == "orphan_native_episode": + native["episode"]["episode_index"] = 999 + elif name == "inverted_event": + capture["events"][0].update(start_timestamp_ns=2000, end_timestamp_ns=1000) + elif name == "duplicate_event_id": + capture["events"].append(deepcopy(capture["events"][0])) + elif name == "external_native_identity_contradiction": + policy["native_identity"] = { + "dataset_id": manifest["dataset_id"], "dataset_revision": "a-contradictory-snapshot", + "episode_ids": {"0": manifest["episode_id"]}, "evidence": "Synthetic snapshot evidence."} + else: + raise AssertionError(f"Unnamed C10 fault: {name}") + + +class C10SemanticTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.validator = Draft202012Validator(json.loads( + (ROOT / "schemas/json/c9-lerobot-dataset.schema.json").read_text())) + + def assert_c9_valid(self, sidecar): + errors = sorted(self.validator.iter_errors(sidecar), key=lambda e: str(e.path)) + self.assertEqual([], [f"{list(e.path)}: {e.message}" for e in errors]) + + def assert_finding(self, findings, check_id, status="REJECT"): + matching = [f for f in findings if f["check_id"] == check_id and f["status"] == status] + diagnostic = [{key: f.get(key) for key in ("check_id", "status", "frame", "stream")} + for f in findings if f["check_id"] == check_id] + self.assertTrue(matching, f"Expected {check_id}/{status}; received {diagnostic}") + return matching + + def test_positive_fixture_is_derived_and_c9_valid(self): + sidecar, native, policy = fixture() + self.assert_c9_valid(sidecar) + findings, metrics = capture(sidecar, native, policy) + unverified = [f for f in findings if f["status"] in ("REJECT", "ERROR", "NOT_VERIFIED")] + self.assertFalse(unverified, unverified) + self.assertIsInstance(metrics, dict) + self.assertIn("SYNTHETIC", policy["id"]) + + def test_every_core_diagnostic_has_named_corruption(self): + self.assertEqual(set(CHECK_IDS), set(NEGATIVE_CASES.values())) + + def test_fixture_objects_are_independent(self): + sidecar, native, policy = fixture() + sidecar["native_lerobot"]["features"]["action"]["shape"][0] = 1 + native["rows"][0]["action"][0] = 42 + policy["action_bounds"].clear() + clean_sidecar, clean_native, clean_policy = fixture() + self.assertEqual(clean_sidecar["native_lerobot"]["features"]["action"]["shape"], [16]) + self.assertEqual(clean_native["rows"][0]["action"][0], 1) + self.assertEqual(len(clean_policy["action_bounds"]), 16) + + def test_pending_tolerance_is_not_a_zero_skew_pass(self): + sidecar, native, policy = fixture() + sidecar["openamrobot"]["synchronization"] = { + "status": "pending_evidence", "tolerance_ns": None, + "basis": "Synthetic unresolved tolerance.", + } + self.assert_c9_valid(sidecar) + findings, _ = capture(sidecar, native, policy) + self.assert_finding(findings, "C10_SYNC", "NOT_VERIFIED") + + def test_partial_quality_policy_does_not_verify_stream(self): + sidecar, native, policy = fixture() + del policy["stream_quality"]["base"]["max_freshness_ns"] + findings, _ = capture(sidecar, native, policy) + matching = self.assert_finding(findings, "C10_STREAM_QUALITY", "NOT_VERIFIED") + self.assertTrue(any(f["stream"] == "base" for f in matching)) + + def test_unmapped_device_preserves_raw_and_does_not_pass_sync(self): + sidecar, native, policy = fixture() + sidecar["openamrobot"]["streams"]["head"]["timestamps"]["device"][ + "mapping_to_authoritative"] = {"status": "pending", "reason": "Synthetic clock evidence pending."} + for sample in sidecar["openamrobot"]["samples"]: + item = sample["streams"]["head"] + raw = item["device_timestamp"]["raw"] + item.update(health="unmapped", selected_timestamp_ns=None, measured_skew_ns=None) + item["device_timestamp"] = {"status": "pending", "raw": raw, "reason": "Synthetic pending mapping."} + self.assert_c9_valid(sidecar) + findings, _ = capture(sidecar, native, policy) + self.assert_finding(findings, "C10_SYNC", "NOT_VERIFIED") + + def test_affine_rounds_nearest_ties_to_even_and_validity_is_inclusive(self): + sidecar, native, policy = fixture() + mapping = sidecar["openamrobot"]["streams"]["head"]["timestamps"]["device"]["mapping_to_authoritative"] + mapping.update(source_origin=0, authoritative_origin_ns=0, + scale_ns_per_source_unit=0.5, + validity={"source_start": 1, "source_end": 3}) + for frame, (raw, mapped) in enumerate(((1, 0), (3, 2))): + item = sidecar["openamrobot"]["samples"][frame]["streams"]["head"] + item["device_timestamp"].update(raw=raw, mapped_authoritative_ns=mapped) + item["selected_timestamp_ns"] = mapped + item["measured_skew_ns"] = abs(mapped - sidecar["openamrobot"]["samples"][frame]["authoritative_timestamp_ns"]) + self.assert_c9_valid(sidecar) + findings, _ = capture(sidecar, native, policy) + self.assertFalse([f for f in findings if f["check_id"] == "C10_MAPPING" and f["status"] == "REJECT"], findings) + sidecar["openamrobot"]["samples"][1]["streams"]["head"]["device_timestamp"]["raw"] = 3.01 + findings, _ = capture(sidecar, native, policy) + self.assert_finding(findings, "C10_MAPPING") + + def test_action_bounds_include_both_endpoints(self): + sidecar, native, policy = fixture() + native["rows"][0]["action"] = [-180.0] * 16 + native["rows"][1]["action"] = [180.0] * 16 + findings, _ = capture(sidecar, native, policy) + self.assertFalse([f for f in findings if f["check_id"] == "C10_ACTION_BOUNDS" and f["status"] == "REJECT"], findings) + + def test_command_latency_includes_both_endpoints(self): + sidecar, native, policy = fixture() + set_time(sidecar, 0, "action", 0) + set_time(sidecar, 1, "action", 33333333 + 2000) + self.assert_c9_valid(sidecar) + findings, _ = capture(sidecar, native, policy) + self.assertFalse([f for f in findings if f["check_id"] == "C10_ACTION_LATENCY" and f["status"] == "REJECT"], findings) + + def test_sync_equal_to_threshold_is_accepted(self): + sidecar, native, policy = fixture() + set_time(sidecar, 1, "base", 33333333 - 100) + findings, _ = capture(sidecar, native, policy) + self.assertFalse([f for f in findings if f["check_id"] == "C10_SYNC" and f["status"] == "REJECT"], findings) + + def test_unapproved_numeric_policy_cannot_approve_ranges(self): + sidecar, native, policy = fixture() + policy["approved"] = False + findings, _ = capture(sidecar, native, policy) + self.assert_finding(findings, "C10_ACTION_BOUNDS", "NOT_VERIFIED") + + def test_native_identity_uses_external_snapshot_evidence(self): + sidecar, native, policy = fixture() + manifest = sidecar["native_lerobot"] + for key in ("dataset_id", "dataset_revision"): + del native["info"][key] + del native["episode"]["episode_id"] + policy["native_identity"] = { + "dataset_id": manifest["dataset_id"], "dataset_revision": manifest["dataset_revision"], + "episode_ids": {"0": manifest["episode_id"]}, + "evidence": "Synthetic immutable snapshot identity; not a production attestation.", + } + findings, _ = capture(sidecar, native, policy) + self.assert_finding(findings, "C10_BINDING", "PASS") + policy["native_identity"]["dataset_revision"] = "different-snapshot" + findings, _ = capture(sidecar, native, policy) + self.assert_finding(findings, "C10_BINDING") + + def test_episode_id_requires_native_or_external_snapshot_evidence(self): + sidecar, native, policy = fixture() + del native["episode"]["episode_id"] + findings, _ = capture(sidecar, native, policy) + self.assert_finding(findings, "C10_BINDING", "NOT_VERIFIED") + + def test_action_reference_offset_requires_explicit_policy(self): + sidecar, native, policy = fixture() + del policy["action_latency"]["reference_frame_offset"] + findings, _ = capture(sidecar, native, policy) + self.assert_finding(findings, "C10_ACTION_LATENCY", "NOT_VERIFIED") + self.assertFalse([f for f in findings if f["check_id"] == "C10_ACTION_LATENCY" and f["status"] == "PASS"]) + + def test_action_reference_offset_outside_episode_is_not_verified(self): + sidecar, native, policy = fixture() + policy["action_latency"]["reference_frame_offset"] = 999 + findings, _ = capture(sidecar, native, policy) + self.assert_finding(findings, "C10_ACTION_LATENCY", "NOT_VERIFIED") + self.assertFalse([f for f in findings if f["check_id"] == "C10_ACTION_LATENCY" and f["status"] == "PASS"]) + + def test_honest_missing_and_dropped_slots_remain_not_verified(self): + for health in ("missing", "dropped"): + with self.subTest(health=health): + sidecar, native, policy = fixture() + sidecar["openamrobot"]["samples"][1]["streams"]["base"] = { + "sequence_id": None, "source_frame_id": None, "health": health, + "selection_timestamp_source": "none", "selected_timestamp_ns": None, + "host_receipt": {"status": "unavailable", "reason": "Synthetic absent slot."}, + "device_timestamp": {"status": "unavailable", "reason": "Synthetic absent slot."}, + "measured_skew_ns": None, + } + self.assert_c9_valid(sidecar) + findings, metrics = capture(sidecar, native, policy) + self.assert_finding(findings, "C10_SYNC", "NOT_VERIFIED") + self.assertEqual(metrics["streams"]["base"][health + "_count"], 1) + self.assertFalse([f for f in findings if f["check_id"] == "C10_SYNC" and + f["frame"] == 1 and f["status"] == "PASS"]) + + def test_stale_producer_label_requires_review_without_fabricated_limit(self): + sidecar,native,policy = fixture() + mutate("producer_reports_stale",sidecar,native,policy) + self.assert_c9_valid(sidecar) + findings,metrics = capture(sidecar,native,policy) + self.assert_finding(findings,"C10_HEALTH","NOT_VERIFIED") + self.assertFalse([f for f in findings if f["check_id"] == "C10_HEALTH" and f["status"] == "REJECT"]) + self.assertEqual(metrics["streams"]["base"]["max_stale_run"],1) + + + + +def _negative_test(case, check_id): + def test(self): + sidecar, native, policy = fixture() + mutate(case, sidecar, native, policy) + self.assert_c9_valid(sidecar) + findings, _ = capture(sidecar, native, policy) + self.assert_finding(findings, check_id) + test.__name__ = f"test_reject_{case}" + test.__doc__ = f"C9-valid {case} must produce {check_id}/REJECT." + return test + + +for _case, _check_id in NEGATIVE_CASES.items(): + setattr(C10SemanticTests, f"test_reject_{_case}", _negative_test(_case, _check_id)) + + + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_c10_evidence.py b/tests/test_c10_evidence.py new file mode 100644 index 0000000..c24e90b --- /dev/null +++ b/tests/test_c10_evidence.py @@ -0,0 +1,371 @@ +# SPDX-License-Identifier: MIT +"""C10 evidence handoff integration; native Parquet dependency is explicit.""" + +import hashlib +import json +import os +from pathlib import Path +import shlex +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +import pyarrow # Native evidence tests fail visibly if their dependency is absent. +import yaml + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools")) +sys.path.insert(0, str(ROOT / "tests")) +import generate_c10_evidence as evidence +from c10_fixture import fixture, write_dataset + + +def file_hashes(directory): + return {str(path.relative_to(directory)): hashlib.sha256(path.read_bytes()).hexdigest() + for path in sorted(directory.rglob("*")) if path.is_file()} + + +class C10EvidenceTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.parent = Path(self.temporary.name) + self.output = self.parent / "evidence" + + def invoke_cli(self, output=None, *extra): + return subprocess.run([ + sys.executable, str(ROOT / "tools/generate_c10_evidence.py"), + "--output-dir", str(output or self.output), *extra], + capture_output=True, text=True, timeout=60) + + def manifest(self, output=None): + return json.loads(((output or self.output) / "evidence.json").read_text()) + + def test_actual_cli_keeps_not_verified_report_and_exact_provenance(self): + process = self.invoke_cli() + self.assertEqual(process.returncode, 0, process.stdout + process.stderr) + report_path = self.output / "c10-report.json" + raw_report = report_path.read_bytes() + report = json.loads(raw_report) + self.assertEqual(report["status"], "NOT_VERIFIED") + self.assertEqual(report["exit_code"], 2) + self.assertFalse(report["training_ready"]) + manifest = self.manifest() + self.assertEqual(manifest["report_sha256"], hashlib.sha256(raw_report).hexdigest()) + self.assertEqual(manifest["expected_synthetic_result"], evidence.EXPECTED_SYNTHETIC_RESULT) + self.assertTrue(manifest["reporting_path_success"]) + self.assertEqual(manifest["helper_exit_code"], 0) + self.assertEqual(manifest["errors"], []) + self.assertEqual(manifest["gate"]["process_exit_code"], 2) + self.assertEqual(manifest["gate"]["report_exit_code"], 2) + self.assertEqual(manifest["gate"]["status"], "NOT_VERIFIED") + self.assertFalse(manifest["gate"]["training_ready"]) + self.assertEqual(manifest["gate"]["production_acceptance"], "NOT_VERIFIED") + head = subprocess.run(["git", "rev-parse", "HEAD"], cwd=ROOT, + capture_output=True, text=True, check=True).stdout.strip() + self.assertEqual(manifest["checker"]["git_head"], head) + self.assertIsNone(manifest["checker"]["expected_checker_sha"]) + self.assertFalse(manifest["checker"]["immutable_checkout_verified"]) + argv = manifest["invocation"]["argv"] + self.assertEqual(Path(argv[0]).resolve(), Path(sys.executable).resolve()) + self.assertEqual(Path(argv[1]).resolve(), ROOT / "tools/check_c10_dataset.py") + self.assertEqual(manifest["invocation"]["shell_command"], shlex.join(argv)) + self.assertEqual(Path(manifest["invocation"]["cwd"]).resolve(), ROOT) + for option, path in (("--dataset-path", self.output / "dataset"), + ("--report", report_path), + ("--policy", self.output / "dataset/synthetic-policy.json")): + self.assertEqual(Path(argv[argv.index(option) + 1]).resolve(), path.resolve()) + self.assertIn("C10 NOT_VERIFIED", (self.output / "cli-stdout.log").read_text()) + self.assertTrue((self.output / "cli-stderr.log").is_file()) + self.assertTrue(manifest["source_files_sha256"]) + self.assertEqual(manifest["source_files_sha256_after"], manifest["source_files_sha256"]) + for relative, digest in manifest["source_files_sha256"].items(): + self.assertEqual(hashlib.sha256((ROOT / relative).read_bytes()).hexdigest(), digest, relative) + actual_hashes = file_hashes(self.output / "dataset") + self.assertEqual(manifest["dataset_files_sha256_before"], actual_hashes) + self.assertEqual(manifest["dataset_files_sha256_after"], actual_hashes) + self.assertTrue(manifest["dataset_unchanged"]) + sidecar, native, policy = fixture() + for key in ("dataset_id", "dataset_revision", "episode_id", "episode_index"): + self.assertEqual(manifest["dataset"][key], sidecar["native_lerobot"][key]) + self.assertEqual(manifest["dataset"]["data_origin"], "synthetic") + self.assertEqual(Path(manifest["dataset"]["path"]).resolve(), (self.output / "dataset").resolve()) + self.assertEqual(manifest["environment"]["python_executable"], sys.executable) + self.assertEqual(manifest["environment"]["packages"]["pyarrow"], pyarrow.__version__) + independent = self.parent / "independent-fixture" + write_dataset(independent, sidecar, native, policy) + self.assertEqual(file_hashes(independent), actual_hashes) + + def test_report_bytes_reproduce_across_independent_output_directories(self): + first = self.invoke_cli(self.output) + second_dir = self.parent / "another-output" + second = self.invoke_cli(second_dir) + self.assertEqual((first.returncode, second.returncode), (0, 0), first.stderr + second.stderr) + self.assertEqual((self.output / "c10-report.json").read_bytes(), + (second_dir / "c10-report.json").read_bytes()) + self.assertEqual(self.manifest()["report_sha256"], self.manifest(second_dir)["report_sha256"]) + self.assertEqual(file_hashes(self.output / "dataset"), file_hashes(second_dir / "dataset")) + + def test_preexisting_output_is_refused_without_changing_any_bytes(self): + self.output.mkdir() + (self.output / "sentinel").write_bytes(b"existing evidence must remain intact\x00") + nested = self.output / "nested" + nested.mkdir() + (nested / "raw-report.json").write_bytes(b'{"untouched":true}\n') + before = file_hashes(self.output) + process = self.invoke_cli() + self.assertNotEqual(process.returncode, 0) + self.assertIn("exist", (process.stdout + process.stderr).lower()) + self.assertEqual(before, file_hashes(self.output)) + self.assertFalse((self.output / "evidence.json").exists()) + + def test_existing_empty_output_is_also_refused(self): + self.output.mkdir() + process = self.invoke_cli() + self.assertNotEqual(process.returncode, 0) + self.assertIn("exist", (process.stdout + process.stderr).lower()) + self.assertEqual(list(self.output.iterdir()), []) + + def simulate_checker(self, report_bytes, return_code): + def run(argv, cwd, stdout_path, stderr_path): + Path(stdout_path).write_bytes(b"raw synthetic gate stdout\n") + Path(stderr_path).write_bytes(b"raw synthetic gate stderr\n") + if report_bytes is not None: + Path(argv[argv.index("--report") + 1]).write_bytes(report_bytes) + return return_code + return run + + def raw_gate_report(self, **overrides): + report = { + "status": "NOT_VERIFIED", "exit_code": 2, "training_ready": False, + "production_acceptance": "NOT_VERIFIED", + "findings": [{"check_id": "C10_REAL_EPISODE", "status": "NOT_VERIFIED", "observed": "synthetic"}], + } + report.update(overrides) + return json.dumps(report, sort_keys=True, indent=1).encode() + b"\n" + + def check_failed_gate_artifacts(self, report_bytes, return_code, error_term): + with patch.object(evidence, "run_checker", side_effect=self.simulate_checker(report_bytes, return_code)): + helper_exit = evidence.generate_evidence(self.output) + self.assertNotEqual(helper_exit, 0) + manifest = self.manifest() + self.assertFalse(manifest["reporting_path_success"]) + self.assertEqual(manifest["expected_synthetic_result"], evidence.EXPECTED_SYNTHETIC_RESULT) + self.assertFalse(manifest["checker"]["immutable_checkout_verified"]) + self.assertEqual(manifest["gate"]["process_exit_code"], return_code) + self.assertEqual(manifest["helper_exit_code"], helper_exit) + self.assertIn(error_term, " ".join(manifest["errors"]).lower()) + self.assertEqual((self.output / "cli-stdout.log").read_bytes(), b"raw synthetic gate stdout\n") + self.assertEqual((self.output / "cli-stderr.log").read_bytes(), b"raw synthetic gate stderr\n") + report_path = self.output / "c10-report.json" + if report_bytes is None: + self.assertFalse(report_path.exists()) + self.assertIsNone(manifest["report_sha256"]) + else: + self.assertEqual(report_path.read_bytes(), report_bytes) + self.assertEqual(manifest["report_sha256"], hashlib.sha256(report_bytes).hexdigest()) + self.assertTrue(manifest["dataset_unchanged"]) + + def test_rejects_unexpected_gate_pass(self): + raw = self.raw_gate_report(status="PASS", exit_code=0, training_ready=True, + production_acceptance="PASS") + self.check_failed_gate_artifacts(raw, 0, "expected") + + def test_rejects_unexpected_gate_reject(self): + raw = self.raw_gate_report(status="REJECT", exit_code=1, production_acceptance="REJECT") + self.check_failed_gate_artifacts(raw, 1, "expected") + + def test_rejects_nonmatching_gate_process_exit(self): + raw = self.raw_gate_report() + self.check_failed_gate_artifacts(raw, 17, "expected") + + def test_rejects_gate_report_with_wrong_report_exit(self): + raw = self.raw_gate_report(exit_code=0) + self.check_failed_gate_artifacts(raw, 2, "expected") + + def test_rejects_noninteger_report_exit_even_when_numerically_two(self): + self.check_failed_gate_artifacts(self.raw_gate_report(exit_code=2.0), 2, "expected") + + def test_rejects_report_without_explicit_false_readiness(self): + report = json.loads(self.raw_gate_report()) + del report["training_ready"] + self.check_failed_gate_artifacts(json.dumps(report).encode(), 2, "expected") + + def test_rejects_report_without_synthetic_origin_finding(self): + self.check_failed_gate_artifacts(self.raw_gate_report(findings=[]), 2, "expected") + + def test_rejects_absent_report_and_preserves_logs(self): + self.check_failed_gate_artifacts(None, 2, "report") + + def test_rejects_malformed_report_and_preserves_exact_bytes(self): + self.check_failed_gate_artifacts(b'{"status": "NOT_VERIFIED", BROKEN\n', 2, "jsondecodeerror") + + def test_checker_sha_mismatch_never_claims_immutable_checkout(self): + process = self.invoke_cli(None, "--expected-checker-sha", "0" * 40) + self.assertNotEqual(process.returncode, 0) + manifest = self.manifest() + self.assertFalse(manifest["checker"]["immutable_checkout_verified"]) + self.assertEqual(manifest["checker"]["expected_checker_sha"], "0" * 40) + self.assertFalse(manifest["reporting_path_success"]) + self.assertTrue(any("sha" in str(error).lower() for error in manifest["errors"])) + + def test_matching_sha_with_mocked_clean_identity_records_verification(self): + mocked_head = "a" * 40 + with patch.object(evidence, "git_state", return_value=(mocked_head, True)): + result = evidence.generate_evidence(self.output, mocked_head) + self.assertEqual(result, 0) + manifest = self.manifest() + self.assertEqual(manifest["checker"]["git_head"], mocked_head) + self.assertEqual(manifest["checker"]["git_head_after"], mocked_head) + self.assertTrue(manifest["checker"]["immutable_checkout_verified"]) + self.assertTrue(manifest["reporting_path_success"]) + self.assertEqual(manifest["gate"]["process_exit_code"], 2) + + def test_matching_sha_with_mocked_dirty_tree_is_refused(self): + mocked_head = "a" * 40 + with patch.object(evidence, "git_state", return_value=(mocked_head, False)): + result = evidence.generate_evidence(self.output, mocked_head) + self.assertEqual(result, 1) + manifest = self.manifest() + self.assertFalse(manifest["checker"]["immutable_checkout_verified"]) + self.assertFalse(manifest["reporting_path_success"]) + self.assertTrue(any("clean" in error.lower() for error in manifest["errors"])) + self.assertIsNone(manifest["invocation"]) + self.assertFalse((self.output / "dataset").exists()) + + def test_checker_mutating_generated_dataset_fails_and_retains_raw_report(self): + real_run = evidence.run_checker + def mutate_after_gate(argv, cwd, stdout_path, stderr_path): + result = real_run(argv, cwd, stdout_path, stderr_path) + dataset = Path(argv[argv.index("--dataset-path") + 1]) + (dataset / "unexpected-mutation.bin").write_bytes(b"deliberate read-only contract violation") + return result + with patch.object(evidence, "run_checker", side_effect=mutate_after_gate): + result = evidence.generate_evidence(self.output) + self.assertEqual(result, 1) + manifest = self.manifest() + self.assertFalse(manifest["dataset_unchanged"]) + self.assertFalse(manifest["reporting_path_success"]) + self.assertIn("unexpected-mutation.bin", manifest["dataset_files_sha256_after"]) + self.assertNotIn("unexpected-mutation.bin", manifest["dataset_files_sha256_before"]) + self.assertTrue(any("dataset" in error.lower() and ("changed" in error.lower() or "differs" in error.lower()) + for error in manifest["errors"])) + report = (self.output / "c10-report.json").read_bytes() + self.assertEqual(json.loads(report)["status"], "NOT_VERIFIED") + self.assertEqual(manifest["report_sha256"], hashlib.sha256(report).hexdigest()) + + def test_head_change_during_generation_cannot_claim_immutable_checkout(self): + before, after = "a" * 40, "b" * 40 + with patch.object(evidence, "git_state", side_effect=[(before, True), (after, True)]): + result = evidence.generate_evidence(self.output, before) + self.assertEqual(result, 1) + manifest = self.manifest() + self.assertFalse(manifest["checker"]["immutable_checkout_verified"]) + self.assertEqual(manifest["checker"]["git_head"], before) + self.assertEqual(manifest["checker"]["git_head_after"], after) + self.assertFalse(manifest["reporting_path_success"]) + self.assertTrue(any("source" in error.lower() and "changed" in error.lower() for error in manifest["errors"])) + + def test_source_bytes_drift_with_unchanged_head_invalidates_evidence(self): + head = "a" * 40 + relative = evidence.SOURCE_FILES[0] + source = (ROOT / relative).resolve() + real_hash = evidence.sha256_file + reads = 0 + def drifted_hash(path): + nonlocal reads + if Path(path).resolve() == source: + reads += 1 + if reads > 1: + return "f" * 64 + return real_hash(path) + with patch.object(evidence, "git_state", return_value=(head, True)), patch.object( + evidence, "sha256_file", side_effect=drifted_hash): + result = evidence.generate_evidence(self.output, head) + self.assertEqual(result, 1) + manifest = self.manifest() + self.assertEqual(manifest["checker"]["git_head"], manifest["checker"]["git_head_after"]) + self.assertEqual(manifest["source_files_sha256"][relative], real_hash(source)) + self.assertEqual(manifest["source_files_sha256_after"][relative], "f" * 64) + self.assertFalse(manifest["checker"]["immutable_checkout_verified"]) + self.assertFalse(manifest["reporting_path_success"]) + self.assertTrue(any("source" in error.lower() and "changed" in error.lower() for error in manifest["errors"])) + self.assertEqual(json.loads((self.output / "c10-report.json").read_bytes())["status"], "NOT_VERIFIED") + self.assertIn("C10 NOT_VERIFIED", (self.output / "cli-stdout.log").read_text()) + + def test_clean_checkout_becoming_dirty_during_generation_is_refused(self): + head = "a" * 40 + with patch.object(evidence, "git_state", side_effect=[(head, True), (head, False)]): + result = evidence.generate_evidence(self.output, head) + self.assertEqual(result, 1) + manifest = self.manifest() + self.assertTrue(manifest["checker"]["source_tree_clean"]) + self.assertFalse(manifest["checker"]["source_tree_clean_after"]) + self.assertFalse(manifest["checker"]["immutable_checkout_verified"]) + self.assertFalse(manifest["reporting_path_success"]) + self.assertTrue(any("dirty" in error.lower() for error in manifest["errors"])) + + +class C10EvidenceWorkflowTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + workflow = yaml.safe_load((ROOT / ".github/workflows/c10-dataset-quality.yml").read_text()) + cls.job = workflow["jobs"]["c10-reference"] + cls.steps = cls.job["steps"] + + def test_upload_runs_after_failures_and_retains_hidden_evidence(self): + upload = [step for step in self.steps if step.get("uses", "").startswith("actions/upload-artifact@")] + self.assertEqual(len(upload), 1) + upload = upload[0] + self.assertEqual(upload["if"], "${{ always() }}") + self.assertTrue(upload["with"]["include-hidden-files"]) + self.assertEqual(upload["with"]["if-no-files-found"], "error") + self.assertEqual(upload["with"]["path"].rstrip("/"), self.job["env"]["C10_EVIDENCE_DIR"]) + self.assertGreaterEqual(upload["with"]["retention-days"], 1) + names = [step.get("name") for step in self.steps] + self.assertGreater(names.index(upload["name"]), names.index("Generate reproducible synthetic CLI evidence")) + self.assertLess(names.index("Prepare evidence directory"), names.index("Install scoped C10 verification dependencies")) + + def test_checkout_and_generator_bind_the_same_explicit_checker_sha(self): + checkout = next(step for step in self.steps if step.get("uses", "").startswith("actions/checkout@")) + self.assertEqual(self.job["env"]["C10_CHECKER_SHA"], "${{ github.event.pull_request.head.sha || github.sha }}") + self.assertEqual(checkout["with"]["ref"], self.job["env"]["C10_CHECKER_SHA"]) + generator = next(step for step in self.steps if step.get("name") == "Generate reproducible synthetic CLI evidence") + self.assertIn('--expected-checker-sha "$C10_CHECKER_SHA"', generator["run"]) + self.assertEqual(generator["if"], "${{ !cancelled() }}") + + def test_each_captured_command_preserves_failure_status_through_tee(self): + suites = { + "Run nonempty C10 core suite": ("test_c10_dataset.py", "core-suite.log"), + "Run nonempty C10 native suite": ("test_c10_native.py", "native-suite.log"), + "Run nonempty C10 evidence suite": ("test_c10_evidence.py", "evidence-suite.log"), + "Prove reverted arithmetic check is detected": ("tools/prove_c10_negative.py", "revert-negative.log"), + "Generate reproducible synthetic CLI evidence": ("tools/generate_c10_evidence.py", "generation.log"), + } + self.assertEqual(self.job["defaults"]["run"]["shell"], "bash") + for name, (pattern, log_name) in suites.items(): + with self.subTest(suite=name), tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + binaries = root / "bin" + binaries.mkdir() + python = binaries / "python" + python.write_text("#!/bin/sh\nprintf '%s\\n' 'injected suite failure output'\nexit 23\n") + python.chmod(0o755) + output = root / "captured" + output.mkdir() + step = next(step for step in self.steps if step.get("name") == name) + self.assertEqual(step["if"], "${{ !cancelled() }}") + self.assertIn(pattern, step["run"]) + self.assertIn("set -o pipefail", step["run"]) + environment = dict(os.environ, PATH=str(binaries) + os.pathsep + os.environ["PATH"], + C10_EVIDENCE_DIR=str(output)) + process = subprocess.run(["bash", "-e", "-c", step["run"]], cwd=root, + env=environment, capture_output=True, text=True) + self.assertEqual(process.returncode, 23, process.stdout + process.stderr) + self.assertEqual((output / log_name).read_text(), "injected suite failure output\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_c10_native.py b/tests/test_c10_native.py new file mode 100644 index 0000000..fe2d746 --- /dev/null +++ b/tests/test_c10_native.py @@ -0,0 +1,311 @@ +# SPDX-License-Identifier: MIT +"""Explicit native-Parquet C10 integration suite; requires pyarrow (no skips).""" + +from copy import deepcopy +import hashlib +import json +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +# This suite intentionally fails import when the optional native dependency is absent. +import pyarrow as pa +import pyarrow.parquet as pq + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools")) +sys.path.insert(0, str(ROOT / "tests")) +from c10_fixture import CAMERAS, fixture, write_dataset +import test_c10_dataset as semantics +from check_c10_dataset import check_dataset + +WRAPPER_CASES = { + "orphan_native_episode_rows": ("C10_NATIVE_DATA", "REJECT"), + "duplicate_json_key": ("C10_INPUT", "REJECT"), + "invalid_c9_schema": ("C10_SCHEMA", "REJECT"), + "duplicate_native_episode": ("C10_NATIVE_METADATA", "REJECT"), + "native_row_range": ("C10_NATIVE_DATA", "REJECT"), + "native_double_vectors": ("C10_PARQUET_DTYPE", "REJECT"), + "decoded_video_frame_count": ("C10_VIDEO", "REJECT"), + "artifact_bytes_changed": ("C10_ARTIFACT", "REJECT"), + "calibration_identity": ("C10_CALIBRATION", "REJECT"), + "recording_release_commit": ("C10_RELEASE", "REJECT"), + "synthetic_is_not_real_evidence": ("C10_REAL_EPISODE", "NOT_VERIFIED"), + "frozen_decoded_pixels": ("C10_VIDEO", "REJECT"), + "configuration_declared_hash": ("C10_ARTIFACT", "REJECT"), + "absent_configuration_artifact": ("C10_ARTIFACT", "NOT_VERIFIED"), + "configuration_symlink_escape": ("C10_ARTIFACT", "REJECT"), + "malformed_nested_policy": ("C10_INPUT", "REJECT"), + "truncated_parquet": ("C10_NATIVE_DATA", "REJECT"), + "unapproved_calibration": ("C10_CALIBRATION", "NOT_VERIFIED"), + "unapproved_release": ("C10_RELEASE", "NOT_VERIFIED"), + "source_profile_identity": ("C10_PROFILE", "REJECT"), + "calibration_residual_exceeds": ("C10_CALIBRATION", "REJECT"), +} + + +class C10WrapperTests(unittest.TestCase): + setUpClass = classmethod(semantics.C10SemanticTests.setUpClass.__func__) + assert_c9_valid = semantics.C10SemanticTests.assert_c9_valid + assert_finding = semantics.C10SemanticTests.assert_finding + + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.path = Path(self.temporary.name) / "dataset" + self.sidecar, self.native, self.policy = fixture() + + def write(self): + self.assert_c9_valid(self.sidecar) + write_dataset(self.path, self.sidecar, self.native, self.policy) + + def run_gate(self): + from check_c10_dataset import check_dataset + return check_dataset(self.path, self.path / "synthetic-policy.json") + + def test_synthetic_native_parquet_is_not_training_ready(self): + self.write() + report = self.run_gate() + self.assertEqual(report["status"], "NOT_VERIFIED", report) + self.assertEqual(report["exit_code"], 2) + self.assertFalse(report["training_ready"]) + for check_id in ("C10_NATIVE_METADATA", "C10_NATIVE_DATA", "C10_PARQUET_DTYPE"): + self.assert_finding(report["findings"], check_id, "PASS") + self.assert_finding(report["findings"], "C10_VIDEO", "NOT_VERIFIED") + self.assert_finding(report["findings"], "C10_REAL_EPISODE", "NOT_VERIFIED") + + def test_native_missing_parquet_is_not_verified(self): + self.write() + (self.path / "data/chunk-000/file-000.parquet").unlink() + self.assert_finding(self.run_gate()["findings"], "C10_NATIVE_DATA", "NOT_VERIFIED") + + def test_cli_report_exit_and_read_only_dataset(self): + self.write() + before = {str(p.relative_to(self.path)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in self.path.rglob("*") if p.is_file()} + report_path = Path(self.temporary.name) / "report.json" + process = subprocess.run([ + sys.executable, str(ROOT / "tools/check_c10_dataset.py"), + "--dataset-path", str(self.path), "--policy", str(self.path / "synthetic-policy.json"), + "--report", str(report_path)], capture_output=True, text=True) + self.assertEqual(process.returncode, 2, process.stdout + process.stderr) + report = json.loads(report_path.read_text()) + self.assertEqual(report["exit_code"], process.returncode) + self.assertEqual(report["status"], "NOT_VERIFIED") + after = {str(p.relative_to(self.path)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in self.path.rglob("*") if p.is_file()} + self.assertEqual(before, after) + json.dumps(report, allow_nan=False) + required = {"check_id", "severity", "status", "episode", "frame", "stream", + "observed", "expected", "units", "threshold", "policy", "fix"} + self.assertTrue(report["findings"]) + for finding in report["findings"]: + self.assertTrue(required <= set(finding), finding) + + def test_selection_rejects_an_unavailable_episode(self): + from check_c10_dataset import check_dataset + self.write() + report = check_dataset(self.path, self.path / "synthetic-policy.json", [999]) + self.assertEqual(report["status"], "REJECT") + self.assert_finding(report["findings"], "C10_NATIVE_METADATA") + + def test_native_metadata_totals_cannot_contradict_episode_registry(self): + self.native["info"]["total_frames"] = 99 + self.write() + self.assert_finding(self.run_gate()["findings"],"C10_NATIVE_METADATA") + + def test_missing_dataset_is_environmental_not_verified(self): + report = check_dataset(self.path) + self.assertEqual(report["exit_code"],2) + self.assert_finding(report["findings"],"C10_INPUT","NOT_VERIFIED") + + def test_nonfinite_parquet_values_produce_strict_json_rejection(self): + self.native["rows"][0]["action"][0] = float("nan") + self.write() + report = self.run_gate() + self.assert_finding(report["findings"], "C10_VECTOR_VALUES") + self.assertEqual(report["exit_code"], 1) + json.dumps(report, allow_nan=False) + + def test_actual_claim_without_external_review_stays_not_verified(self): + self.sidecar["openamrobot"]["provenance"]["data_origin"] = "actual" + self.write() + self.assert_finding(self.run_gate()["findings"], "C10_REAL_EPISODE", "NOT_VERIFIED") + + def test_real_video_decode_or_explicit_unavailable(self): + self.native["info"]["video_path"] = "videos/{video_key}/episode-{episode_index:06d}.mp4" + self.policy["video_content"] = {name: {"max_identical_run": 1} for name in CAMERAS} + self.write() + if not shutil.which("ffmpeg") or not shutil.which("ffprobe"): + report = self.run_gate() + missing = self.assert_finding(report["findings"], "C10_VIDEO", "NOT_VERIFIED") + self.assertEqual({f["stream"] for f in missing}, set(CAMERAS)) + print("C10 real video decode branch unavailable: ffmpeg/ffprobe absent; explicit NOT_VERIFIED verified.") + return + seed = Path(self.temporary.name) / "two-frame-synthetic.mp4" + process = subprocess.run([ + "ffmpeg", "-v", "error", "-f", "lavfi", "-i", "testsrc=size=64x48:rate=30", + "-frames:v", "2", "-pix_fmt", "yuv420p", "-c:v", "mpeg4", "-y", str(seed)], + capture_output=True, text=True) + self.assertEqual(process.returncode, 0, process.stderr) + videos = [] + for stream in CAMERAS: + feature = self.sidecar["openamrobot"]["streams"][stream]["native_feature_key"] + path = self.path / f"videos/{feature}/episode-000000.mp4" + path.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(seed, path) + videos.append(path) + report = self.run_gate() + successful = self.assert_finding(report["findings"], "C10_VIDEO", "PASS") + self.assertEqual({f["stream"] for f in successful}, set(CAMERAS)) + videos[0].write_bytes(b"deliberately corrupt synthetic mp4") + self.assert_finding(self.run_gate()["findings"], "C10_VIDEO") + + def test_fractional_clock_lexemes_survive_json_and_yaml_loading(self): + import yaml + from check_c10_dataset import preserve_clock_precision + from c10_semantics import _mapped + mapping = self.sidecar["openamrobot"]["streams"]["head"]["timestamps"]["device"]["mapping_to_authoritative"] + mapping.update(source_origin=0, scale_ns_per_source_unit=0.5, + validity={"source_start": 0, "source_end": 100000000}) + first = self.sidecar["openamrobot"]["samples"][0]["streams"]["head"] + first["device_timestamp"].update(raw="EXACT_RAW", mapped_authoritative_ns=1) + first.update(selected_timestamp_ns=1, measured_skew_ns=1) + second = self.sidecar["openamrobot"]["samples"][1]["streams"]["head"] + second["device_timestamp"]["raw"] = 66666666 + for extension in ("json", "yaml"): + with self.subTest(format=extension): + path = Path(self.temporary.name) / f"lexical-tie.{extension}" + text = json.dumps(self.sidecar) if extension == "json" else yaml.safe_dump(self.sidecar) + text = text.replace('"EXACT_RAW"' if extension == "json" else "EXACT_RAW", "1.0000000000000000001") + path.write_text(text) + parsed = json.loads(text) if extension == "json" else yaml.safe_load(text) + self.assert_c9_valid(parsed) + precise = preserve_clock_precision(path, parsed) + if precise is None: + precise = parsed + raw = precise["openamrobot"]["samples"][0]["streams"]["head"]["device_timestamp"]["raw"] + precise_mapping = precise["openamrobot"]["streams"]["head"]["timestamps"]["device"]["mapping_to_authoritative"] + self.assertEqual(str(raw), "1.0000000000000000001") + self.assertEqual(_mapped(raw, precise_mapping), 1) + + def run_named_fault(self, case): + import pyarrow as pa + import pyarrow.parquet as pq + + if case == "orphan_native_episode_rows": + orphan = deepcopy(self.native["rows"][0]) + orphan["episode_index"] = 99 + self.native["rows"].append(orphan) + elif case == "native_row_range": + self.native["rows"][1]["index"] = 42 + elif case in ("recording_release_commit", "unapproved_release"): + pin = {"release": "synthetic-test-release", "commit": "1" * 40, + "evidence": "Synthetic release-owner fixture; not a production pin."} + self.sidecar["openamrobot"]["provenance"]["lerobot_compatibility"]["validated_release_pin"] = pin + self.sidecar["openamrobot"]["provenance"]["software"].append({ + "name": "lerobot", "version": pin["release"], "commit": ("2" if case == "recording_release_commit" else "1") * 40, + "repository": "https://github.com/huggingface/lerobot"}) + self.policy["release"] = {"pin": deepcopy(pin), "version": pin["release"], "commit": pin["commit"]} + elif case in ("decoded_video_frame_count", "frozen_decoded_pixels"): + self.native["info"]["video_path"] = "videos/{video_key}/episode-{episode_index:06d}.mp4" + if case == "frozen_decoded_pixels": + self.policy["video_content"] = {name: {"max_identical_run": 1} for name in CAMERAS} + elif case == "unapproved_calibration": + residual = self.sidecar["openamrobot"]["streams"]["base"]["calibration"]["residual"] + self.policy["calibration"] = {"base": { + key: residual[key] for key in ("name", "unit", "statistic", "method")}} + self.policy["calibration"]["base"]["max_value"] = 0.3 + elif case == "calibration_residual_exceeds": + residual = self.sidecar["openamrobot"]["streams"]["base"]["calibration"]["residual"] + self.policy["calibration"] = {"base": { + key: residual[key] for key in ("name", "unit", "statistic", "method")}} + self.policy["calibration"]["base"]["max_value"] = 0.2 + self.write() + if case == "duplicate_json_key": + path = self.path / "meta/openamrobot/episodes/episode-000000.json" + path.write_text('{"contract_version":1,"contract_version":1}') + elif case == "invalid_c9_schema": + del self.sidecar["contract_version"] + (self.path / "meta/openamrobot/episodes/episode-000000.json").write_text(json.dumps(self.sidecar)) + elif case == "duplicate_native_episode": + pq.write_table(pa.Table.from_pylist([self.native["episode"], self.native["episode"]]), + self.path / "meta/episodes/chunk-000/file-000.parquet") + elif case == "native_double_vectors": + pq.write_table(pa.Table.from_pylist(self.native["rows"]), self.path / "data/chunk-000/file-000.parquet") + elif case == "artifact_bytes_changed": + artifact = self.path / "synthetic-config.bin" + artifact.write_bytes(b"mutated synthetic bytes") + self.policy["artifacts"] = [{"kind": "robot_configuration", "path": artifact.name, + "sha256": self.sidecar["openamrobot"]["provenance"]["robot_configuration_hash"]}] + elif case in ("configuration_declared_hash", "configuration_symlink_escape", "absent_configuration_artifact"): + artifact = self.path / "synthetic-config.bin" + if case == "configuration_symlink_escape": + external = Path(self.temporary.name) / "outside-config.bin" + external.write_bytes(b"synthetic bytes outside dataset") + artifact.symlink_to(external) + elif case != "absent_configuration_artifact": + artifact.write_bytes(b"synthetic bytes with a different declared hash") + digest = "sha256:" + hashlib.sha256(artifact.read_bytes()).hexdigest() if artifact.exists() else "sha256:" + "1" * 64 + self.policy["artifacts"] = [{"kind": "robot_configuration", "path": artifact.name, "sha256": digest}] + elif case == "malformed_nested_policy": + self.policy["profiles"]["action"] = [] + elif case == "source_profile_identity": + artifact = self.path / "synthetic-source-profile.json" + artifact.write_text('{"synthetic_only":true}') + self.policy["source_profile"] = { + "id": "a-different-synthetic-source-profile", "artifact": { + "path": artifact.name, "sha256": "sha256:" + hashlib.sha256(artifact.read_bytes()).hexdigest()}} + elif case == "truncated_parquet": + path = self.path / "data/chunk-000/file-000.parquet" + path.write_bytes(path.read_bytes()[:64]) + elif case == "calibration_identity": + artifact = self.path / "synthetic-calibration.bin" + artifact.write_bytes(b"synthetic calibration") + self.policy["artifacts"] = [{"kind": "calibration", "stream": "base", "path": artifact.name, + "sha256": "sha256:" + hashlib.sha256(artifact.read_bytes()).hexdigest(), + "id": "a-different-synthetic-calibration", "version": "fixture-1"}] + elif case in ("decoded_video_frame_count", "frozen_decoded_pixels"): + for stream in CAMERAS: + feature = self.sidecar["openamrobot"]["streams"][stream]["native_feature_key"] + path = self.path / f"videos/{feature}/episode-000000.mp4" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"synthetic probe input") + frames = [{"width": 64, "height": 48, "best_effort_timestamp_time": "0"}] + if case == "frozen_decoded_pixels": + frames.append({"width": 64, "height": 48, "best_effort_timestamp_time": "0.033333"}) + output = json.dumps({"frames": frames}) + def fake_process(command, **kwargs): + value = output if command[0] == "ffprobe" else "# synthetic framemd5\n0,0,0,1,9216,samehash\n0,1,1,1,9216,samehash\n" + return subprocess.CompletedProcess(command, 0, value, "") + with patch("check_c10_dataset.shutil.which", return_value="synthetic-ffprobe"), patch( + "check_c10_dataset.subprocess.run", side_effect=fake_process): + return self.run_gate() + elif case in ("unapproved_calibration", "unapproved_release"): + check_id = "C10_CALIBRATION" if case == "unapproved_calibration" else "C10_RELEASE" + self.assert_finding(self.run_gate()["findings"], check_id, "PASS") + self.policy["approved"] = False + (self.path / "synthetic-policy.json").write_text(json.dumps(self.policy)) + return self.run_gate() + + + +def _wrapper_test(case, check_id, status): + def test(self): + report = self.run_named_fault(case) + self.assert_finding(report["findings"], check_id, status) + self.assertEqual(report["exit_code"], 1 if status == "REJECT" else 2) + test.__name__ = f"test_wrapper_{case}" + return test + + +for _case, (_check_id, _status) in WRAPPER_CASES.items(): + setattr(C10WrapperTests, f"test_wrapper_{_case}", _wrapper_test(_case, _check_id, _status)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/c10_semantics.py b/tools/c10_semantics.py new file mode 100644 index 0000000..443ff40 --- /dev/null +++ b/tools/c10_semantics.py @@ -0,0 +1,433 @@ +# SPDX-License-Identifier: MIT +"""Read-only C9 cross-field checks. No physical acceptance thresholds are built in. + +The caller structurally validates the sidecar first and supplies decoded native +records. An explicitly supplied policy uses ``approved: true``, ``id`` and +``evidence``; these identify the caller's reviewed policy, not an approval by +this program. Missing policy/evidence is NOT_VERIFIED, never an implicit pass. +""" + +from decimal import Decimal, InvalidOperation, ROUND_HALF_EVEN, localcontext +import math +import struct + + +CHECK_IDS = ( + "C10_BINDING", "C10_FEATURES", "C10_TASK", "C10_COVERAGE", + "C10_TIME_ORDER", "C10_MAPPING", "C10_SELECTION", "C10_HEALTH", + "C10_SEQUENCE", "C10_SKEW", "C10_SYNC", "C10_STREAM_QUALITY", + "C10_VECTOR_VALUES", "C10_PROFILE", "C10_ACTION_BOUNDS", + "C10_ACTION_LATENCY", "C10_EVENTS", "C10_COMPLETENESS", +) + + +def _finite(value): + return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) + + +def _decimal(value): + result = Decimal(str(value)) + if not result.is_finite(): + raise ValueError("nonfinite decimal") + return result + + +def _mapped(raw, mapping): + """Use decimal input values, sufficient exact precision, ties to even.""" + raw, origin, scale, offset, start, end = ( + _decimal(raw), _decimal(mapping["source_origin"]), + _decimal(mapping["scale_ns_per_source_unit"]), + _decimal(mapping["authoritative_origin_ns"]), + _decimal(mapping["validity"]["source_start"]), + _decimal(mapping["validity"]["source_end"]), + ) + if not start <= raw <= end or scale <= 0: + raise ValueError("outside inclusive validity or nonpositive scale") + # Decimal's default 28 digits can lose nanoseconds at large source origins. + with localcontext() as ctx: + ctx.prec = sum(len(x.as_tuple().digits) + abs(x.as_tuple().exponent) + for x in (raw, origin, scale, offset)) + 20 + value = (offset + scale * (raw - origin)).to_integral_value(rounding=ROUND_HALF_EVEN) + if value < 0: + raise ValueError("negative recording time") + return int(value) + + +def check_episode(sidecar, native, policy, emit): + """Check a schema-valid sidecar against native metadata and optional rows. + + ``native`` contains info, episode, tasks (index -> instruction), and rows + (list or None). Returns measured metrics only; diagnostic verdicts go to + ``emit(check_id, status, observed=..., expected=..., ...)``. + """ + binding, robot = sidecar["native_lerobot"], sidecar["openamrobot"] + info, episode = native.get("info", {}), native.get("episode", {}) + rows, tasks = native.get("rows"), native.get("tasks", {}) + streams, samples = robot["streams"], robot["samples"] + features, task = binding["features"], binding["task"] + policy = policy or {} + approved = (policy.get("approved") is True and bool(policy.get("id")) + and bool(policy.get("evidence"))) + policy_ref = {"id": policy.get("id"), "evidence": policy.get("evidence")} + + def check(code, valid, observed, expected, **context): + emit(code, "PASS" if valid else "REJECT", observed=observed, + expected=expected, fix="Correct the source evidence and regenerate the recording sidecar.", **context) + + def pending(code, observed, expected, **context): + emit(code, "NOT_VERIFIED", observed=observed, expected=expected, + fix="Supply the missing native evidence or an explicitly reviewed policy with evidence.", **context) + + expected_binding = {"dataset_id": binding["dataset_id"], + "dataset_revision": binding["dataset_revision"], + "format_version": binding["format_version"], + "episode_id": binding["episode_id"], + "episode_index": binding["episode_index"], "length": binding["length"]} + observed_binding = {"dataset_id": info.get("repo_id", info.get("dataset_id")), + "dataset_revision": info.get("dataset_revision"), + "format_version": info.get("codebase_version"), + "episode_id": episode.get("episode_id"), + "episode_index": episode.get("episode_index"), "length": episode.get("length")} + identity = policy.get("native_identity", {}) + if approved and isinstance(identity, dict) and identity.get("evidence"): + for key in ("dataset_id", "dataset_revision"): + if key in identity: + check("C10_BINDING", identity[key] == expected_binding[key], identity[key], + expected_binding[key], policy=policy_ref) + if observed_binding[key] is None: + observed_binding[key] = identity.get(key) + episode_identity = identity.get("episode_ids", {}).get(str(binding["episode_index"])) + if episode_identity is not None: + check("C10_BINDING", episode_identity == binding["episode_id"], episode_identity, + binding["episode_id"], policy=policy_ref) + observed_binding["episode_id"] = episode.get("episode_id", episode_identity) + if "episode_id" in episode: + expected_binding["episode_id"] = binding["episode_id"] + observed_binding["episode_id"] = episode["episode_id"] + missing_binding = [key for key, value in observed_binding.items() if value is None] + known_mismatch = any(value is not None and value != expected_binding[key] + for key, value in observed_binding.items()) + if missing_binding and not known_mismatch: + pending("C10_BINDING", observed_binding, expected_binding) + else: + check("C10_BINDING", observed_binding == expected_binding, observed_binding, expected_binding) + + check("C10_FEATURES", info.get("fps") == binding["fps"], info.get("fps"), binding["fps"], units="Hz") + native_features = info.get("features", {}) + for key, descriptor in features.items(): + actual = native_features.get(key, {}) + expected = {"dtype": descriptor["dtype"], "shape": descriptor["shape"]} + observed = {"dtype": actual.get("dtype"), "shape": actual.get("shape")} + if "dimensions" in descriptor: + names = [item["name"] for item in descriptor["dimensions"]] + expected["names"] = names + observed["names"] = actual.get("names") + dimensions_valid = (len(names) == descriptor["shape"][0] and len(names) == len(set(names))) + else: + dimensions_valid = descriptor["fps"] == binding["fps"] + if "fps" in actual: + dimensions_valid &= actual["fps"] == descriptor["fps"] + video_info = actual.get("info", {}) + if "video.fps" in video_info: + dimensions_valid &= video_info["video.fps"] == descriptor["fps"] + if "fps" in video_info: + dimensions_valid &= video_info["fps"] == descriptor["fps"] + check("C10_FEATURES", expected == observed and dimensions_valid, observed, expected, stream=key) + feature_keys = [stream["native_feature_key"] for stream in streams.values()] + check("C10_FEATURES", set(feature_keys) == set(features) and len(feature_keys) == len(set(feature_keys)), + feature_keys, sorted(features)) + for name, stream in streams.items(): + if "frame" in stream: + frame, descriptor = stream["frame"], features.get(stream["native_feature_key"], {}) + observed = {"shape": [frame["height"], frame["width"], 3], "encoding": frame["encoding"]} + expected = {"shape": descriptor.get("shape"), "encoding": descriptor.get("encoding")} + check("C10_FEATURES", observed == expected, observed, expected, stream=name) + + native_task = tasks.get(task["task_index"], tasks.get(str(task["task_index"]))) + check("C10_TASK", native_task == task["instruction"] and episode.get("tasks") == [task["instruction"]], + {"instruction": native_task, "episode_tasks": episode.get("tasks")}, + {"instruction": task["instruction"], "episode_tasks": [task["instruction"]]}) + frame_indices = [sample["frame_index"] for sample in samples] + check("C10_COVERAGE", frame_indices == list(range(binding["length"])), frame_indices, + list(range(binding["length"]))) + start, end = robot["recording"]["start_timestamp_ns"], robot["recording"]["end_timestamp_ns"] + references = [sample["authoritative_timestamp_ns"] for sample in samples] + check("C10_TIME_ORDER", start == 0 and start <= end and all(start <= t <= end for t in references) + and all(a < b for a, b in zip(references, references[1:])), + {"start": start, "end": end, "timestamps": references}, + "recording starts at zero; strictly increasing frame references within inclusive recording bounds", units="ns") + + metrics = {"frame_count": len(samples), "streams": {}, "inter_observation_skew_ns": [], "action_latency_ns": []} + previous = {} + source_ids = {name: {} for name in streams} + runs = {name: {"stale": 0, "reuse": 0} for name in streams} + frame_times = {sample["frame_index"]: sample["authoritative_timestamp_ns"] for sample in samples} + observations = [name for name in streams if name != "action"] + for name in streams: + metrics["streams"][name] = {"available_count": 0, "missing_count": 0, "dropped_count": 0, + "unmapped_count": 0, "stale_count": 0, "reuse_count": 0, + "sequence_gap_count": 0, "max_sequence_gap": 0, + "max_stale_run": 0, "max_reuse_run": 0, + "configured_fps": streams[name].get("frame", {}).get("fps"), + "nominal_fps": binding["fps"], + "selected_timestamps_ns": [], "freshness_ns": []} + for sample in samples: + frame, reference = sample["frame_index"], sample["authoritative_timestamp_ns"] + check("C10_TASK", sample["episode_index"] == binding["episode_index"] and sample["task_index"] == task["task_index"], + {"episode_index": sample["episode_index"], "task_index": sample["task_index"]}, + {"episode_index": binding["episode_index"], "task_index": task["task_index"]}, frame=frame) + check("C10_COVERAGE", set(sample["streams"]) == set(streams), sorted(sample["streams"]), sorted(streams), frame=frame) + usable = {} + for name, metadata in streams.items(): + if name not in sample["streams"]: + continue + item = sample["streams"][name] + context = {"frame": frame, "stream": name} + health = item["health"] + selected = item["selected_timestamp_ns"] + source = item["selection_timestamp_source"] + timestamps = metadata["timestamps"] + stats = metrics["streams"][name] + runs[name]["stale"] = runs[name]["stale"] + 1 if health == "stale" else 0 + stats["max_stale_run"] = max(stats["max_stale_run"], runs[name]["stale"]) + if health in ("missing", "dropped", "unmapped", "stale"): + stats[health + "_count"] += 1 + absent = health in ("missing", "dropped") + if absent: + runs[name]["reuse"] = 0 + host, device = item["host_receipt"], item["device_timestamp"] + health_valid = True + if absent: + health_valid = (item["sequence_id"] is None and item["source_frame_id"] is None + and selected is None and item["measured_skew_ns"] is None and source == "none" + and host["status"] == "unavailable" and device["status"] == "unavailable") + else: + health_valid = (item["sequence_id"] is not None and bool(item["source_frame_id"]) + and host["status"] in ("valid", "pending") and source == timestamps["capture_source"]) + chosen = device if source == "device" else host + if health == "unmapped": + health_valid &= (selected is None and item["measured_skew_ns"] is None + and chosen["status"] == "pending" and "raw" in chosen and bool(chosen.get("reason"))) + else: + health_valid &= selected is not None and chosen["status"] == "valid" + check("C10_HEALTH", health_valid, item, + "health, IDs and timestamp availability agree", **context) + if health == "stale": + pending("C10_HEALTH", item, "review of producer-reported stale evidence; the label alone is not a QA verdict", **context) + mapped_sources = {} + if not absent: + for source_name, stamp in (("host_receipt", host), ("device", device)): + source_metadata = timestamps[source_name] + mapping = source_metadata.get("mapping_to_authoritative", {}) + if stamp["status"] == "unavailable": + # Device acquisition timestamps can explicitly be unavailable. + check("C10_MAPPING", source_name == "device" and bool(stamp.get("reason")), + stamp, "unavailable device timestamp has a reason", **context) + continue + if stamp["status"] == "pending": + valid_pending = (mapping.get("status") in ("pending", "unavailable") + and "raw" in stamp and bool(stamp.get("reason")) + and "mapped_authoritative_ns" not in stamp) + check("C10_MAPPING", valid_pending, stamp, "pending mapping preserves raw source time and reason", **context) + if valid_pending: + pending("C10_MAPPING", mapping, "valid calibrated mapping", **context) + continue + try: + calculated = _mapped(stamp["raw"], mapping) if mapping.get("status") == "valid" else None + valid = (calculated is not None and calculated == stamp.get("mapped_authoritative_ns") + and start <= calculated <= end) + except (KeyError, ValueError, InvalidOperation, TypeError, OverflowError): + calculated, valid = None, False + if source_name == "device" and source_metadata.get("availability") != "available": + valid = False + check("C10_MAPPING", valid, stamp, {"source": source_name, "recomputed_ns": calculated}, units="ns", **context) + if valid: + mapped_sources[source_name] = calculated + if selected is not None: + selection_valid = source == timestamps["capture_source"] and mapped_sources.get(source) == selected + check("C10_SELECTION", selection_valid, {"source": source, "selected_ns": selected}, + {"source": timestamps["capture_source"], "selected_ns": mapped_sources.get(source)}, units="ns", **context) + check("C10_TIME_ORDER", start <= selected <= end, selected, {"min": start, "max": end}, units="ns", **context) + skew = abs(selected - reference) + check("C10_SKEW", item["measured_skew_ns"] == skew, item["measured_skew_ns"], skew, units="ns", **context) + if selection_valid and health_valid: + usable[name] = selected + stats["available_count"] += 1 + stats["selected_timestamps_ns"].append(selected) + stats["freshness_ns"].append(max(0, reference - selected)) + if not absent: + seen_sequence = source_ids[name].get(item["source_frame_id"]) + if seen_sequence is not None: + check("C10_SEQUENCE", seen_sequence == item["sequence_id"], + {"source_frame_id": item["source_frame_id"], "sequence_id": item["sequence_id"]}, + {"original_sequence_id": seen_sequence}, **context) + source_ids[name][item["source_frame_id"]] = item["sequence_id"] + prior = previous.get(name) + if prior: + reuse = item["sequence_id"] == prior["sequence_id"] + preserved = all(item[key] == prior[key] for key in ( + "source_frame_id", "selected_timestamp_ns", "selection_timestamp_source", "host_receipt", "device_timestamp")) + sequence_valid = item["sequence_id"] >= prior["sequence_id"] + if reuse: + stats["reuse_count"] += 1 + runs[name]["reuse"] += 1 + stats["max_reuse_run"] = max(stats["max_reuse_run"], runs[name]["reuse"]) + sequence_valid &= preserved + check("C10_HEALTH", health != "fresh", health, "reused source samples must not claim fresh", **context) + else: + runs[name]["reuse"] = 0 + gap = max(0, item["sequence_id"] - prior["sequence_id"] - 1) + stats["sequence_gap_count"] += gap + stats["max_sequence_gap"] = max(stats["max_sequence_gap"], gap) + sequence_valid &= item["source_frame_id"] != prior["source_frame_id"] + for stamp_key in ("host_receipt", "device_timestamp"): + a, b = prior[stamp_key].get("raw"), item[stamp_key].get("raw") + if a is not None and b is not None: + sequence_valid &= _decimal(b) >= _decimal(a) + if selected is not None and prior["selected_timestamp_ns"] is not None: + sequence_valid &= selected >= prior["selected_timestamp_ns"] + check("C10_SEQUENCE", sequence_valid, item, + "nondecreasing sequence/time; new IDs on advance; exact original evidence on reuse", **context) + previous[name] = item + if all(name in usable for name in observations): + times = [usable[name] for name in observations] + skew = max(times) - min(times) + metrics["inter_observation_skew_ns"].append({"frame_index": frame, "value": skew}) + tolerance = policy.get("synchronization", {}).get("max_skew_ns") + declaration = robot["synchronization"] + if declaration["status"] == "pending_evidence": + pending("C10_SYNC", {"skew_ns": skew, "declaration": declaration}, + "declared tolerance backed by approved external synchronization policy", units="ns", frame=frame) + elif approved and _finite(tolerance) and tolerance >= 0: + check("C10_SYNC", skew <= tolerance and declaration["tolerance_ns"] == tolerance, + {"skew_ns": skew, "declared_tolerance_ns": declaration["tolerance_ns"]}, + "declared tolerance matches policy; inter-observation skew within approved tolerance", + units="ns", threshold=tolerance, policy=policy_ref, frame=frame) + else: + pending("C10_SYNC", {"skew_ns": skew, "declaration": robot["synchronization"]}, + "approved external synchronization policy", units="ns", frame=frame) + else: + pending("C10_SYNC", {"unavailable_streams": [name for name in observations if name not in usable]}, + "mapped selected time for every observation stream", frame=frame) + if "action" in usable: + latency_policy = policy.get("action_latency", {}) + offset = latency_policy.get("reference_frame_offset") if approved else None + if type(offset) is not int or frame + offset not in frame_times: + pending("C10_ACTION_LATENCY", {"reference_frame_offset": offset}, + "approved explicit reference-frame offset and an existing reference frame", frame=frame) + continue + reference_frame = frame + offset + latency = usable["action"] - frame_times[reference_frame] + metrics["action_latency_ns"].append({"frame_index": frame, "reference_frame_index": reference_frame, "value": latency}) + if (approved and all(_finite(latency_policy.get(key)) for key in ("min_ns", "max_ns")) + and latency_policy["min_ns"] <= latency_policy["max_ns"]): + semantics = {key: robot["action_semantics"][key] for key in + ("values", "timestamp", "observation_relationship")} + valid = (all(latency_policy.get(key) == value for key, value in semantics.items()) + and latency_policy["min_ns"] <= latency <= latency_policy["max_ns"]) + check("C10_ACTION_LATENCY", valid, {"latency_ns": latency, **semantics}, + latency_policy, units="ns", threshold=latency_policy, policy=policy_ref, frame=frame) + else: + pending("C10_ACTION_LATENCY", latency, "approved latency bounds and exact declared observation relationship", units="ns", frame=frame) + else: + pending("C10_ACTION_LATENCY", None, "mapped action timestamp", frame=frame) + + for name, stats in metrics["streams"].items(): + selected_times = stats["selected_timestamps_ns"] + stats["observed_intervals_ns"] = [b - a for a, b in zip(selected_times, selected_times[1:])] + times = sorted(set(stats["selected_timestamps_ns"])) + stats["measured_fps"] = (len(times) - 1) * 1_000_000_000 / (times[-1] - times[0]) if len(times) > 1 else None + stats["max_freshness_ns"] = max(stats["freshness_ns"], default=None) + quality = policy.get("stream_quality", {}).get(name) + if not approved or not isinstance(quality, dict) or not quality: + pending("C10_STREAM_QUALITY", stats, "approved source FPS/freshness/gap/reuse policy", stream=name) + continue + checks = {"min_fps": (stats["measured_fps"], lambda a, b: a >= b), + "max_freshness_ns": (stats["max_freshness_ns"], lambda a, b: a <= b), + "max_sequence_gap": (stats["max_sequence_gap"], lambda a, b: a <= b), + "max_reuse": (stats["reuse_count"], lambda a, b: a <= b)} + if set(quality) < set(checks): + pending("C10_STREAM_QUALITY", stats, "all FPS, freshness, sequence-gap and reuse policy fields", + policy=policy_ref, stream=name) + continue + valid = (set(quality) == set(checks) + and all(_finite(value) and value >= 0 for value in quality.values())) + unavailable = False + for key, limit in quality.items(): + if key not in checks or not _finite(limit): + continue + value, compare = checks[key] + if value is None: + unavailable = True + else: + valid &= compare(value, limit) + if valid and (unavailable or stats["available_count"] != len(samples)): + pending("C10_STREAM_QUALITY", stats, quality, policy=policy_ref, stream=name) + else: + check("C10_STREAM_QUALITY", valid, stats, quality, threshold=quality, policy=policy_ref, stream=name) + + for key in ("observation.state", "action"): + descriptor = features[key] + stream_name = "robot_state" if key == "observation.state" else "action" + actual_profile = {"profile": descriptor["profile"], "dimensions": descriptor["dimensions"], + "reference_frame": streams[stream_name]["reference_frame"]} + wanted_profile = policy.get("profiles", {}).get(key) + if approved and isinstance(wanted_profile, dict): + check("C10_PROFILE", actual_profile == wanted_profile, actual_profile, wanted_profile, policy=policy_ref, stream=key) + else: + pending("C10_PROFILE", actual_profile, "approved explicit ordered dimension, unit, meaning and reference-frame profile", stream=key) + + if rows is None: + pending("C10_VECTOR_VALUES", None, "decoded native state/action rows") + pending("C10_ACTION_BOUNDS", None, "decoded native action rows and approved named bounds") + else: + check("C10_COVERAGE", [row.get("frame_index") for row in rows] == frame_indices, + [row.get("frame_index") for row in rows], frame_indices) + for row in rows: + frame = row.get("frame_index") + check("C10_TASK", row.get("episode_index") == binding["episode_index"] and row.get("task_index") == task["task_index"], + {"episode_index": row.get("episode_index"), "task_index": row.get("task_index")}, + {"episode_index": binding["episode_index"], "task_index": task["task_index"]}, frame=frame) + if isinstance(frame, int): + grid_time = frame / binding["fps"] + float32_time = struct.unpack("f", struct.pack("f", grid_time))[0] + check("C10_TIME_ORDER", row.get("timestamp") in (grid_time, float32_time), row.get("timestamp"), + {"native_grid_seconds": grid_time, "float32_seconds": float32_time}, units="s", frame=frame) + for key in ("observation.state", "action"): + value = row.get(key) + valid_vector = isinstance(value, (list, tuple)) and len(value) == features[key]["shape"][0] and all(_finite(v) for v in value) + check("C10_VECTOR_VALUES", valid_vector, value, + {"length": features[key]["shape"][0], "values": "finite numeric values"}, frame=frame, stream=key) + if key != "action": + continue + bounds = policy.get("action_bounds") + if not approved or not isinstance(bounds, dict) or not bounds: + pending("C10_ACTION_BOUNDS", value, "approved bounds keyed by every declared action dimension and unit", frame=frame) + continue + dimensions = features[key]["dimensions"] + valid_bounds = set(bounds) == {dim["name"] for dim in dimensions} + for dim, number in zip(dimensions, value if valid_vector else []): + bound = bounds.get(dim["name"], {}) + valid_bounds &= (isinstance(bound, dict) and _finite(bound.get("min")) and _finite(bound.get("max")) + and bound.get("unit") == dim["unit"] and bound["min"] <= number <= bound["max"]) + check("C10_ACTION_BOUNDS", valid_vector and valid_bounds, value, bounds, threshold=bounds, policy=policy_ref, frame=frame) + + event_ids = [event["event_id"] for event in robot["events"]] + check("C10_EVENTS", len(event_ids) == len(set(event_ids)), event_ids, "unique event IDs") + for event in robot["events"]: + lower, upper = event["start_timestamp_ns"], event["end_timestamp_ns"] + valid = start <= lower <= upper <= end and all( + index in frame_times and lower <= frame_times[index] <= upper for index in event["linked_frame_indices"]) + check("C10_EVENTS", valid, event, "ordered event bounds and linked frame times within recording and event interval") + outcome, recording = robot["outcome"], robot["recording"] + reason_valid = isinstance(outcome.get("reason"), str) and bool(outcome["reason"].strip()) + complete = recording["status"] == "complete" + consistent = (outcome["status"] == "success" or reason_valid) + if outcome["status"] == "aborted": + consistent &= any(event["type"] == "abort" for event in robot["events"]) + if outcome["status"] == "success": + consistent &= not any(event["type"] == "abort" for event in robot["events"]) + check("C10_COMPLETENESS", complete and consistent, + {"recording": recording, "outcome": outcome}, + "complete recording; non-success reason; aborted outcomes linked to abort evidence") + return metrics diff --git a/tools/check_c10_dataset.py b/tools/check_c10_dataset.py new file mode 100644 index 0000000..ac1b004 --- /dev/null +++ b/tools/check_c10_dataset.py @@ -0,0 +1,434 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Read-only C9 conformance reference gate. See docs/c10-readme.md.""" + +import argparse +from collections import Counter +from decimal import Decimal +import hashlib +import json +import math +from pathlib import Path +import shutil +import subprocess +import sys + +from jsonschema import Draft202012Validator +import yaml + +from lint_schemas import load_data, unique_pairs +from c10_semantics import check_episode + +ROOT = Path(__file__).resolve().parents[1] + + +def report_value(value): + """Keep non-finite native evidence visible in standards-compliant JSON.""" + if isinstance(value, float) and not math.isfinite(value): + return str(value) + if isinstance(value, Decimal): + return str(value) + if isinstance(value, dict): + return {str(k): report_value(v) for k,v in value.items()} + if isinstance(value, (list,tuple)): + return [report_value(v) for v in value] + return value + + +def preserve_clock_precision(path, sidecar): + """After existing structural validation, retain decimal clock lexemes. + + Converting a raw fractional tick through binary float can move a value + across an exact nanosecond half-tie. Only clock arithmetic fields are + replaced; native descriptors retain the existing loader's representations. + """ + if path.suffix == ".json": + exact = json.loads(path.read_text(), object_pairs_hook=unique_pairs, parse_float=Decimal) + else: + class DecimalLoader(yaml.SafeLoader): + pass + DecimalLoader.add_constructor("tag:yaml.org,2002:float", + lambda loader,node: Decimal(loader.construct_scalar(node).replace("_",""))) + exact = yaml.load(path.read_text(), Loader=DecimalLoader) + for name,stream in sidecar["openamrobot"]["streams"].items(): + precise = exact["openamrobot"]["streams"][name]["timestamps"] + for source in ("host_receipt","device"): + if "mapping_to_authoritative" in precise[source]: + stream["timestamps"][source]["mapping_to_authoritative"] = precise[source]["mapping_to_authoritative"] + for sample, precise in zip(sidecar["openamrobot"]["samples"],exact["openamrobot"]["samples"]): + for name,item in sample["streams"].items(): + for source in ("host_receipt","device_timestamp"): + if "raw" in item[source]: + item[source]["raw"] = precise["streams"][name][source]["raw"] + + +def inside(root, relative): + path = (root / relative).resolve() + path.relative_to(root.resolve()) + return path + + +def jsonl(path): + return [json.loads(line, object_pairs_hook=unique_pairs, + parse_constant=lambda x: (_ for _ in ()).throw(ValueError(x))) + for line in path.read_text().splitlines() if line.strip()] + + +def parquet(path): + import pyarrow.parquet as pq + table = pq.read_table(path) + # Duplicate physical columns otherwise collapse silently in to_pylist(). + if len(set(table.column_names)) != len(table.column_names): + raise ValueError("duplicate Parquet columns") + return table + + +def native_metadata(root): + info = load_data(inside(root, "meta/info.json")) + if not isinstance(info, dict) or not isinstance(info.get("features"), dict): + raise ValueError("native info must contain a features object") + task_path = inside(root, "meta/tasks.parquet") + if task_path.exists(): + task_rows = parquet(task_path).to_pylist() + else: + task_rows = jsonl(inside(root, "meta/tasks.jsonl")) + tasks = {} + for row in task_rows: + index = row["task_index"] + if index in tasks: + raise ValueError("duplicate native task_index") + tasks[index] = row.get("task", row.get("instruction", row.get("__index_level_0__"))) + episode_paths = sorted((root / "meta/episodes").glob("chunk-*/*.parquet")) + episodes = [row for path in episode_paths for row in parquet(inside(root,path)).to_pylist()] if episode_paths else jsonl(inside(root, "meta/episodes.jsonl")) + indexes = [row["episode_index"] for row in episodes] + if len(set(indexes)) != len(indexes): + raise ValueError("duplicate native episode_index") + return info, tasks, {row["episode_index"]: row for row in episodes} + + +def read_rows(root, info, episode, emit, known_episode_indices=None): + try: + template = info.get("data_path", "data/chunk-{chunk_index:03d}/file-{file_index:03d}.parquet") + args = dict(episode_index=episode["episode_index"], + episode_chunk=episode["episode_index"] // info.get("chunks_size", 1000), + chunk_index=episode.get("data/chunk_index", 0), + file_index=episode.get("data/file_index", 0)) + path = inside(root, template.format(**args)) + if not path.is_file(): + emit("C10_NATIVE_DATA", "NOT_VERIFIED", observed=str(path.relative_to(root)), + fix="Provide the native episode Parquet artifact.") + return None + table = parquet(path) + all_rows = table.to_pylist() + if known_episode_indices is not None: + orphans = sorted({row.get("episode_index") for row in all_rows if row.get("episode_index") not in known_episode_indices},key=str) + emit("C10_NATIVE_DATA", "REJECT" if orphans else "PASS",observed=orphans, + expected="every stored episode_index belongs to the native episode registry", + fix="Reconcile orphan rows in the referenced native Parquet file with immutable episode metadata.") + rows = [row for row in all_rows if row.get("episode_index") == episode["episode_index"]] + emit("C10_NATIVE_DATA", "PASS", observed=len(rows), expected=episode.get("length"), units="frames") + import pyarrow as pa + for key in ("observation.state", "action"): + dtype = table.schema.field(key).type + flat = (pa.types.is_list(dtype) or pa.types.is_fixed_size_list(dtype)) and pa.types.is_float32(dtype.value_type) + emit("C10_PARQUET_DTYPE", "PASS" if flat else "REJECT", observed=str(dtype), + expected="list", stream=key, fix="Record native float32 vectors using the declared feature descriptor.") + if "dataset_from_index" in episode and "dataset_to_index" in episode: + actual = [row.get("index") for row in rows] + expected = list(range(episode["dataset_from_index"], episode["dataset_to_index"])) + emit("C10_NATIVE_DATA", "PASS" if actual == expected else "REJECT", observed=actual, expected=expected, + fix="Restore the immutable native episode row range.") + return rows + except ImportError: + emit("C10_NATIVE_DATA", "NOT_VERIFIED", fix="Install optional pyarrow to inspect native Parquet artifacts.") + except (OSError, ValueError, KeyError, TypeError, IndexError) as error: + emit("C10_NATIVE_DATA", "REJECT", observed=str(error), fix="Restore a readable native Parquet artifact and its metadata binding.") + return None + + +def videos(root, sidecar, info, episode, policy, emit): + for name, stream in sidecar["openamrobot"]["streams"].items(): + if "frame" not in stream: + continue + key = stream["native_feature_key"] + try: + template = info.get("video_path") + if not template: + raise FileNotFoundError("No native video_path descriptor") + chunk = episode.get(f"videos/{key}/chunk_index") + file_index = episode.get(f"videos/{key}/file_index") + # v3 shared containers must have explicit per-episode timestamp offsets. + start = episode.get(f"videos/{key}/from_timestamp") + end = episode.get(f"videos/{key}/to_timestamp") + shared = "{file_index" in template + if shared and (chunk is None or file_index is None or start is None or end is None): + raise FileNotFoundError("Missing v3 video chunk/file/time range binding") + path = inside(root, template.format(video_key=key, chunk_index=chunk, file_index=file_index, + episode_index=episode["episode_index"], + episode_chunk=episode["episode_index"] // info.get("chunks_size", 1000))) + if not path.is_file(): + raise FileNotFoundError(str(path.relative_to(root))) + if not shutil.which("ffprobe"): + raise FileNotFoundError("ffprobe is unavailable; video decode/count was not performed") + proc = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_frames", + "-show_entries", "frame=best_effort_timestamp_time,width,height", "-of", "json", str(path)], + capture_output=True, text=True, timeout=60) + if proc.returncode or proc.stderr.strip(): + raise ValueError(proc.stderr or "ffprobe decode failed") + all_frames = json.loads(proc.stdout)["frames"] + positions = list(range(len(all_frames))) + if shared: + positions = [i for i,f in enumerate(all_frames) if start <= float(f["best_effort_timestamp_time"]) < end] + frames = [all_frames[i] for i in positions] + shape = stream["frame"] + valid = len(frames) == episode["length"] and all(f["width"] == shape["width"] and f["height"] == shape["height"] for f in frames) + emit("C10_VIDEO", "PASS" if valid else "REJECT", stream=name, observed={"frames":len(frames)}, + expected={"frames":episode["length"],"width":shape["width"],"height":shape["height"]}, + units="frames,pixels", fix="Restore the recorded video segment and native frame binding.") + # Source IDs cannot prove pixel freshness. Identical images are not + # automatically bad (a static scene is legitimate); policy decides. + content_policy = policy.get("video_content", {}).get(name, {}) + limit = content_policy.get("max_identical_run") + reviewed = policy.get("approved") is True and bool(policy.get("id")) and bool(policy.get("evidence")) + if not reviewed or type(limit) is not int or limit < 1 or not shutil.which("ffmpeg"): + emit("C10_VIDEO", "NOT_VERIFIED", stream=name, + observed="pixel-content freshness was not measured", + expected="decoded image-content evidence with reviewed static-scene policy", + fix="Supply ffmpeg and a reviewed video_content max_identical_run policy; source IDs alone do not prove fresh pixels.") + elif valid: + decoded = subprocess.run(["ffmpeg","-v","error","-i",str(path),"-map","0:v:0","-f","framemd5","-"], + capture_output=True,text=True,timeout=60) + if decoded.returncode or decoded.stderr.strip(): + raise ValueError(decoded.stderr or "ffmpeg content decode failed") + hashes = [line.split(",")[-1].strip() for line in decoded.stdout.splitlines() if line and not line.startswith("#")] + if len(hashes) != len(all_frames): + raise ValueError("decoded image hashes and frame metadata count disagree") + run, maximum, previous = 0, 0, None + for i in positions: + digest = hashes[i] + run = run + 1 if digest == previous else 1 + previous = digest + maximum = max(maximum,run) + emit("C10_VIDEO", "PASS" if maximum <= limit else "REJECT", stream=name, + observed={"max_identical_run":maximum},expected=content_policy,units="frames", + threshold=limit,policy=policy.get("id"),fix="Review repeated decoded images against the recorded scene and approved freshness policy.") + except FileNotFoundError as error: + emit("C10_VIDEO", "NOT_VERIFIED", stream=name, observed=str(error), fix="Provide native video artifacts, metadata offsets and ffprobe.") + except (OSError, ValueError, KeyError, TypeError, subprocess.TimeoutExpired) as error: + emit("C10_VIDEO", "REJECT", stream=name, observed=str(error), fix="Restore a decodable video and consistent frame range.") + + +def evidence(root, sidecar, policy, emit): + am = sidecar["openamrobot"] + approved = policy.get("approved") is True and bool(policy.get("id")) and bool(policy.get("evidence")) + artifacts = policy.get("artifacts", []) if approved else [] + source = policy.get("source_profile", {}) if approved else {} + declared_profile = am["provenance"]["source_profile"] + if not source: + emit("C10_PROFILE", "NOT_VERIFIED", observed=declared_profile, + fix="Supply the reviewed source-profile identity and immutable serialized profile artifact.") + else: + emit("C10_PROFILE", "PASS" if source.get("id") == declared_profile else "REJECT", + observed=declared_profile, expected=source.get("id"), + fix="Use the recording's actual approved source-profile identity.") + verify_artifact(root, source.get("artifact", {}), source.get("artifact", {}).get("sha256"), emit, "source_profile") + references = [("robot_configuration", None, am["provenance"]["robot_configuration_hash"])] + references += [("stream_configuration", name, stream["configuration_hash"]) for name,stream in am["streams"].items()] + for kind, name, declared in references: + matches = [a for a in artifacts if a.get("kind") == kind and a.get("stream") == name] + if not matches: + emit("C10_ARTIFACT", "NOT_VERIFIED", stream=name, observed=declared, expected="immutable source bytes + reviewed hash", fix="Supply an exact-byte artifact reference in the reviewed policy.") + for artifact in matches: + verify_artifact(root, artifact, declared, emit, name) + for name,stream in am["streams"].items(): + if "calibration" not in stream: + continue + cal = stream["calibration"] + matches = [a for a in artifacts if a.get("kind") == "calibration" and a.get("stream") == name] + if not matches: + emit("C10_CALIBRATION", "NOT_VERIFIED", stream=name, observed=cal, fix="Supply immutable calibration evidence and a reviewed acceptance policy.") + for artifact in matches: + same = (artifact.get("id"), artifact.get("version")) == (cal["id"], cal["version"]) + emit("C10_CALIBRATION", "PASS" if same else "REJECT", stream=name, + observed=[cal["id"],cal["version"]], expected=[artifact.get("id"),artifact.get("version")], fix="Bind the recording to its actual versioned calibration artifact.") + verify_artifact(root, artifact, artifact.get("sha256"), emit, name) + # A measured residual is evidence, not automatically an approved result. + acceptance = policy.get("calibration", {}).get(name, {}) if approved else {} + residual = cal["residual"] + if residual["status"] != "measured" or not all(k in acceptance for k in ("max_value","unit","name","statistic","method")): + emit("C10_CALIBRATION", "NOT_VERIFIED", stream=name, observed=residual, fix="Provide measured residuals and owner-reviewed acceptance criteria.") + else: + valid = all(residual.get(k) == acceptance[k] for k in ("unit","name","statistic","method")) and residual["value"] <= acceptance["max_value"] + emit("C10_CALIBRATION", "PASS" if valid else "REJECT", stream=name, observed=residual, expected=acceptance, + units=acceptance["unit"], threshold=acceptance["max_value"], policy=policy.get("id"), fix="Resolve calibration evidence against the approved criterion.") + actual = am["provenance"]["lerobot_compatibility"]["validated_release_pin"] + release = policy.get("release") if approved else None + recording = [s for s in am["provenance"]["software"] if s["name"].lower() == "lerobot"] + if actual is None or not release or not recording: + emit("C10_RELEASE", "NOT_VERIFIED", observed=actual, fix="Obtain a release-owner pin and identify the recording LeRobot version/commit.") + else: + valid = (actual == release.get("pin") and release.get("version") == actual["release"] + and release.get("commit") == actual["commit"] and actual["release"] != "unknown" + and all(s["version"] == actual["release"] and s["commit"] == actual["commit"] for s in recording)) + emit("C10_RELEASE", "PASS" if valid else "REJECT", observed={"pin":actual,"software":recording}, expected=release, + fix="Reconcile recording software provenance with the approved release pin.") + real = policy.get("real_episode", {}) if approved else {} + real_verified = (am["provenance"]["data_origin"] == "actual" and bool(real.get("evidence")) + and all(real.get(key) == sidecar["native_lerobot"][key] + for key in ("dataset_id","dataset_revision","episode_id","episode_index"))) + emit("C10_REAL_EPISODE", "PASS" if real_verified else "NOT_VERIFIED", + observed=am["provenance"]["data_origin"], expected="actual recording with provenance reviewed externally", + fix="Provide a real recorded episode; synthetic fixtures establish QA behavior only.") + + +def verify_artifact(root, artifact, declared, emit, stream): + try: + path = inside(root, artifact["path"]) + if not path.is_file(): + raise FileNotFoundError(artifact["path"]) + digest = "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() + expected = artifact.get("sha256") + emit("C10_ARTIFACT", "PASS" if digest == expected == declared else "REJECT", stream=stream, + observed={"bytes_sha256":digest,"declared":declared}, expected=expected, + fix="Restore the exact immutable artifact bytes and matching declared hash.") + except FileNotFoundError as error: + emit("C10_ARTIFACT", "NOT_VERIFIED", stream=stream, observed=str(error), fix="Make the referenced immutable artifact available under the dataset root.") + except (OSError, ValueError, KeyError, TypeError) as error: + emit("C10_ARTIFACT", "REJECT", stream=stream, observed=str(error), fix="Use an in-dataset artifact path with an explicit SHA-256 identity.") + + +def check_dataset(dataset_path, policy_path=None, episode_indices=None): + root = Path(dataset_path).resolve() + findings, reports = [], [] + context = {"episode":None} + def emit(check_id,status,observed=None,expected=None,units=None,threshold=None,policy=None,fix="Resolve the reported mismatch.",frame=None,stream=None): + findings.append(dict(check_id=check_id,status=status,severity={"PASS":"INFO","REJECT":"ERROR","NOT_VERIFIED":"BLOCKER"}[status], + episode=context["episode"],frame=frame,stream=stream,observed=observed,expected=expected, + units=units,threshold=threshold,policy=policy,fix=fix)) + try: + if not root.is_dir(): + raise FileNotFoundError("dataset path is not an available directory") + policy = load_data(Path(policy_path)) if policy_path else {} + if not isinstance(policy,dict): + raise ValueError("policy must be an object") + if policy.get("approved") is True and any(not isinstance(policy.get(key),str) or not policy[key].strip() for key in ("id","evidence")): + raise ValueError("approved policy requires nonempty string id and evidence") + for key in ("profiles","action_bounds","synchronization","stream_quality","action_latency", + "calibration","release","real_episode","native_identity","source_profile","video_content"): + if key in policy and not isinstance(policy[key],dict): + raise ValueError(f"policy {key} must be an object") + for key in ("profiles","action_bounds","stream_quality","calibration","video_content"): + if any(not isinstance(value,dict) for value in policy.get(key,{}).values()): + raise ValueError(f"policy {key} entries must be objects") + if "artifact" in policy.get("source_profile",{}) and not isinstance(policy["source_profile"]["artifact"],dict): + raise ValueError("source_profile artifact must be an object") + if "episode_ids" in policy.get("native_identity",{}) and not isinstance(policy["native_identity"]["episode_ids"],dict): + raise ValueError("native_identity episode_ids must be an object") + if not isinstance(policy.get("artifacts",[]),list) or any(not isinstance(a,dict) for a in policy.get("artifacts",[])): + raise ValueError("policy artifacts must be an array of objects") + schema = load_data(ROOT / "schemas/json/c9-lerobot-dataset.schema.json") + validator = Draft202012Validator(schema) + paths = sorted(p for p in (root / "meta/openamrobot/episodes").glob("episode-*") if p.suffix in (".json",".yaml",".yml")) + if not paths: + raise FileNotFoundError("No C9 sidecars at meta/openamrobot/episodes/episode-*.(json|yaml|yml)") + try: + info,tasks,episodes = native_metadata(root) + emit("C10_NATIVE_METADATA","PASS",observed=len(episodes),units="episodes") + measured_counts = {"total_episodes":len(episodes),"total_tasks":len(tasks), + "total_frames":sum(episode["length"] for episode in episodes.values())} + for key,value in measured_counts.items(): + emit("C10_NATIVE_METADATA", "NOT_VERIFIED" if key not in info else "PASS" if info[key] == value else "REJECT", + observed=info.get(key),expected=value,units=key, + fix="Reconcile native info totals with immutable task and episode metadata.") + except ImportError: + info,tasks,episodes = {},{},{} + emit("C10_NATIVE_METADATA","NOT_VERIFIED",fix="Install optional pyarrow to read native v3 metadata.") + except (OSError,ValueError,KeyError,TypeError) as error: + info,tasks,episodes = {},{},{} + emit("C10_NATIVE_METADATA","REJECT",observed=str(error),fix="Provide valid native info/task/episode metadata without duplicate identities.") + seen = set() + for path in paths: + sidecar = load_data(inside(root,path)) + errors = sorted(validator.iter_errors(sidecar),key=lambda e:str(e.path)) + if errors: + emit("C10_SCHEMA","REJECT",observed=[{"path":list(e.path),"message":e.message} for e in errors],fix="Conform the sidecar to the unchanged accepted C9 v1 schema.") + continue + preserve_clock_precision(path, sidecar) + index = sidecar["native_lerobot"]["episode_index"] + if episode_indices is not None and index not in episode_indices: + continue + context["episode"] = index + emit("C10_SCHEMA","PASS",observed=str(path.relative_to(root))) + if index in seen: + emit("C10_NATIVE_METADATA","REJECT",observed=index,expected="one sidecar per episode",fix="Remove duplicate episode sidecars from the recording export.") + seen.add(index) + episode = episodes.get(index,{"episode_index":index}) + rows = read_rows(root,info,episode,emit,set(episodes)) if index in episodes else None + native = {"info":info,"tasks":tasks,"episode":episode,"rows":rows} + metrics = check_episode(sidecar,native,policy,emit) + if rows is not None: + covered = set(row.get("frame_index") for row in rows) & set(sample["frame_index"] for sample in sidecar["openamrobot"]["samples"]) + length = episode.get("length") + metrics["native_frame_coverage"] = { + "native_frames_loaded":len(rows),"covered_native_frames":len(covered), + "expected_native_length":length, + "fraction":len(covered)/length if isinstance(length,int) and length > 0 else None} + videos(root,sidecar,info,episode,policy,emit) + evidence(root,sidecar,policy,emit) + episode_findings = [f for f in findings if f["episode"] == index] + episode_status = "REJECT" if any(f["status"] == "REJECT" for f in episode_findings) else "NOT_VERIFIED" if any(f["status"] == "NOT_VERIFIED" for f in episode_findings) else "PASS" + reports.append({"episode_index":index,"status":episode_status,"metrics":metrics}) + context["episode"] = None + expected = set(episodes) if episode_indices is None else set(episode_indices) + if expected - seen or not seen: + emit("C10_NATIVE_METADATA","REJECT",observed=sorted(seen),expected=sorted(expected),fix="Supply exactly one C9 sidecar for every selected native episode.") + except (FileNotFoundError,PermissionError,ImportError) as error: + emit("C10_INPUT","NOT_VERIFIED",observed=str(error),fix="Provide the accessible dataset, policy and inspection dependencies.") + except (OSError,ValueError,KeyError,TypeError,ArithmeticError,AttributeError,yaml.YAMLError,RecursionError) as error: + emit("C10_INPUT","REJECT",observed=str(error),fix="Provide readable, duplicate-key-free C9/native metadata and a valid reviewed policy.") + counts = Counter(f["status"] for f in findings) + status = "REJECT" if counts["REJECT"] else "NOT_VERIFIED" if counts["NOT_VERIFIED"] else "PASS" + metrics = {"episodes_checked":len(reports),"findings_by_status":dict(sorted(counts.items()))} + skews = [value["value"] for episode in reports for value in episode["metrics"].get("inter_observation_skew_ns",[])] + if skews: + metrics["inter_observation_skew_ns"] = {"measured_frames":len(skews),"min":min(skews),"max":max(skews)} + coverage = [episode["metrics"]["native_frame_coverage"] for episode in reports if "native_frame_coverage" in episode["metrics"]] + if coverage: + metrics["native_frame_coverage"] = {"episodes_measured":len(coverage),"native_frames_loaded":sum(c["native_frames_loaded"] for c in coverage), + "covered_native_frames":sum(c["covered_native_frames"] for c in coverage)} + return report_value({"report_version":1,"status":status,"exit_code":{"PASS":0,"REJECT":1,"NOT_VERIFIED":2}[status], + "training_ready":status == "PASS","production_acceptance":"NOT_VERIFIED", + "findings":findings,"episodes":reports, + "metrics":metrics}) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dataset-path",type=Path,required=True) + parser.add_argument("--report",type=Path,required=True) + parser.add_argument("--policy",type=Path) + parser.add_argument("--episode",action="append",type=int) + args = parser.parse_args() + if args.policy and args.report.resolve() == args.policy.resolve(): + parser.error("--report must not overwrite the reviewed policy") + try: + args.report.resolve().relative_to(args.dataset_path.resolve()) + except ValueError: + pass + else: + parser.error("--report must be outside the read-only dataset") + report = check_dataset(args.dataset_path,args.policy,args.episode) + try: + args.report.write_text(json.dumps(report,indent=2,sort_keys=True,allow_nan=False)+"\n") + except (OSError,ValueError) as error: + print(f"C10 report could not be written: {error}",file=sys.stderr) + return 2 + print(f"C10 {report['status']}: {len(report['episodes'])} episodes; {report['metrics']['findings_by_status']}") + for finding in report["findings"]: + if finding["status"] != "PASS": + print(f" {finding['status']} {finding['check_id']} episode={finding['episode']} frame={finding['frame']} stream={finding['stream']}: {finding['fix']}") + return report["exit_code"] + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/generate_c10_evidence.py b/tools/generate_c10_evidence.py new file mode 100644 index 0000000..792885c --- /dev/null +++ b/tools/generate_c10_evidence.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Generate a fresh synthetic CLI evidence bundle without approving any data. + +Exit zero verifies the reporting path's expected synthetic NOT_VERIFIED result; +the checker still exits two and its original report is retained unchanged. +Without --expected-checker-sha, dirty source is permitted and explicitly +recorded. With that option, both the exact Git HEAD and a clean tree are +required. The ignored .verification/c10-evidence/ directory is recommended. +""" + +import argparse +from datetime import datetime, timezone +import hashlib +import importlib.metadata +import importlib.util +import json +from pathlib import Path +import platform +import re +import shlex +import subprocess +import sys + + +ROOT = Path(__file__).resolve().parents[1] +SOURCE_FILES = ( + "tools/check_c10_dataset.py", + "tools/c10_semantics.py", + "tools/lint_schemas.py", + "schemas/json/c9-lerobot-dataset.schema.json", + "schemas/yaml/c9-lerobot-dataset-valid.yaml", + "tests/c10_fixture.py", + "tools/generate_c10_evidence.py", +) +EXPECTED_SYNTHETIC_RESULT = { + "process_exit_code": 2, + "status": "NOT_VERIFIED", + "report_exit_code": 2, + "training_ready": False, + "production_acceptance": "NOT_VERIFIED", +} + + +def sha256_file(path): + """Hash file bytes without loading a dataset artifact into memory.""" + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def file_inventory(directory): + """Inventory only this helper's newly generated synthetic directory.""" + directory = Path(directory) + if not directory.exists(): + return {} + result = {} + for path in sorted(directory.rglob("*")): + if path.is_symlink(): + raise ValueError(f"Unexpected symbolic link in generated dataset: {path}") + if path.is_file(): + result[path.relative_to(directory).as_posix()] = sha256_file(path) + return result + + +def git_state(): + """Report actual checkout identity; never substitute an expected SHA.""" + head = subprocess.run(["git", "rev-parse", "HEAD"], cwd=ROOT, + check=True, capture_output=True, text=True).stdout.strip() + status = subprocess.run(["git", "status", "--porcelain", "--untracked-files=all"], + cwd=ROOT, check=True, capture_output=True, text=True).stdout + return head, not bool(status) + + +def run_checker(argv, cwd, stdout_path, stderr_path): + """Run the actual CLI and preserve its stdout and stderr bytes verbatim.""" + with Path(stdout_path).open("wb") as stdout, Path(stderr_path).open("wb") as stderr: + return subprocess.run(argv, cwd=cwd, stdout=stdout, stderr=stderr, check=False).returncode + + +def _fixture_module(): + # Import the existing fixture by path to avoid collisions with installed + # packages named tests. No alternative fixture or policy lives here. + spec = importlib.util.spec_from_file_location("c10_evidence_fixture", ROOT / "tests/c10_fixture.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _environment(): + packages = {} + for distribution in importlib.metadata.distributions(): + name = distribution.metadata.get("Name") + if name: + packages[name] = distribution.version + return {"python_executable": sys.executable, "python_version": sys.version, + "python_implementation": platform.python_implementation(), + "platform": platform.platform(), "packages": dict(sorted(packages.items()))} + + +def _read_json(path): + def unique_pairs(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f"Duplicate JSON key: {key}") + result[key] = value + return result + + def invalid_constant(value): + raise ValueError(f"Nonfinite JSON constant: {value}") + + return json.loads(Path(path).read_text(), object_pairs_hook=unique_pairs, + parse_constant=invalid_constant) + + +def _synthetic_identity(dataset): + sidecars = sorted((dataset / "meta/openamrobot/episodes").glob("episode-*.json")) + if len(sidecars) != 1: + raise ValueError("Expected exactly one generated synthetic sidecar") + sidecar = _read_json(sidecars[0]) + binding = sidecar["native_lerobot"] + identity = {key: binding[key] for key in + ("dataset_id", "dataset_revision", "episode_id", "episode_index")} + identity["data_origin"] = sidecar["openamrobot"]["provenance"]["data_origin"] + if identity["data_origin"] != "synthetic": + raise ValueError("Generated fixture does not declare synthetic data_origin") + return identity + + +def expected_gate_result(report, process_exit): + """Independently verify the expected synthetic result, without changing it.""" + if not isinstance(report, dict): + return False + findings = report.get("findings", []) + # Exact types matter: False equals 0, but it is not an integer exit code. + return (type(process_exit) is int and process_exit == 2 + and type(report.get("exit_code")) is int and report["exit_code"] == 2 + and report.get("status") == "NOT_VERIFIED" + and report.get("training_ready") is False + and report.get("production_acceptance") == "NOT_VERIFIED" + and isinstance(findings, list) and any( + isinstance(item, dict) and item.get("check_id") == "C10_REAL_EPISODE" + and item.get("status") == "NOT_VERIFIED" and item.get("observed") == "synthetic" + for item in findings)) + + +def generate_evidence(output_dir, expected_checker_sha=None): + """Create an evidence bundle, returning 0 only for the expected raw result. + + An existing output path is never modified, even if it is empty. Once a + fresh directory exists, failures retain the generated files, logs and a + partial evidence.json whenever the filesystem permits writing it. + """ + output_dir = Path(output_dir).expanduser().absolute() + try: + # mkdir(exist_ok=False) refuses existing files, directories and symlinks. + output_dir.mkdir(parents=True, exist_ok=False) + except OSError as error: + print(f"C10 evidence output must be a fresh directory: {error}", file=sys.stderr) + return 1 + + output_dir = output_dir.resolve() + dataset = output_dir / "dataset" + report_path = output_dir / "c10-report.json" + stdout_path, stderr_path = output_dir / "cli-stdout.log", output_dir / "cli-stderr.log" + manifest = { + "evidence_version": 1, + "created_at_utc": datetime.now(timezone.utc).isoformat(), + "purpose": "Synthetic CLI reporting-path evidence only; owner adoption and production acceptance remain pending.", + "checker": {"git_head": None, "expected_checker_sha": expected_checker_sha, + "source_tree_clean": None, "immutable_checkout_verified": False}, + "invocation": None, + "dataset": {"path": str(dataset)}, + "artifacts": {"report": "c10-report.json", "stdout": "cli-stdout.log", + "stderr": "cli-stderr.log", "manifest": "evidence.json"}, + "environment": None, + "source_files_sha256": {}, + "source_files_sha256_after": {}, + "dataset_files_sha256_before": {}, + "dataset_files_sha256_after": {}, + "dataset_unchanged": None, + "report_sha256": None, + "gate": {key: None for key in EXPECTED_SYNTHETIC_RESULT}, + "expected_synthetic_result": dict(EXPECTED_SYNTHETIC_RESULT), + "reporting_path_success": False, + "helper_exit_code": 1, + "errors": [], + } + try: + stdout_path.touch(exist_ok=False) + stderr_path.touch(exist_ok=False) + head, clean = git_state() + manifest["checker"].update(git_head=head, source_tree_clean=clean) + manifest["environment"] = _environment() + manifest["source_files_sha256"] = {name: sha256_file(ROOT / name) for name in SOURCE_FILES} + if expected_checker_sha is not None: + if not re.fullmatch(r"[0-9a-f]{40}", expected_checker_sha): + raise ValueError("--expected-checker-sha must be a full lowercase 40-character Git SHA") + if head != expected_checker_sha: + raise ValueError(f"Checker HEAD {head} differs from expected SHA {expected_checker_sha}") + if not clean: + raise ValueError("--expected-checker-sha requires a clean tracked and untracked source tree") + + fixture_module = _fixture_module() + sidecar, native, policy = fixture_module.fixture() + if sidecar["openamrobot"]["provenance"]["data_origin"] != "synthetic": + raise ValueError("Existing fixture generator did not return synthetic data") + fixture_module.write_dataset(dataset, sidecar, native, policy) + identity = _synthetic_identity(dataset) + manifest["dataset"].update(identity) + manifest["dataset_files_sha256_before"] = file_inventory(dataset) + argv = [sys.executable, str(ROOT / "tools/check_c10_dataset.py"), + "--dataset-path", str(dataset), "--policy", str(dataset / "synthetic-policy.json"), + "--report", str(report_path)] + manifest["invocation"] = {"argv": argv, "cwd": str(ROOT), "shell_command": shlex.join(argv)} + manifest["gate"]["process_exit_code"] = run_checker(argv, ROOT, stdout_path, stderr_path) + manifest["dataset_files_sha256_after"] = file_inventory(dataset) + manifest["dataset_unchanged"] = (manifest["dataset_files_sha256_before"] == manifest["dataset_files_sha256_after"]) + if not report_path.is_file() or report_path.is_symlink(): + raise ValueError("Checker did not create an ordinary report file") + manifest["report_sha256"] = sha256_file(report_path) + try: + report = _read_json(report_path) + except (OSError, ValueError) as error: + raise ValueError(f"Checker report could not be read ({type(error).__name__}): {error}") from error + if not isinstance(report, dict): + raise ValueError("Checker report must be a JSON object") + manifest["gate"].update(status=report.get("status"), report_exit_code=report.get("exit_code"), + training_ready=report.get("training_ready"), + production_acceptance=report.get("production_acceptance")) + if not manifest["dataset_unchanged"]: + raise ValueError("Generated dataset changed while the read-only checker ran") + if _synthetic_identity(dataset) != identity: + raise ValueError("Generated synthetic identity changed while the checker ran") + if not expected_gate_result(report, manifest["gate"]["process_exit_code"]): + raise ValueError("Checker outcome differs from the expected synthetic NOT_VERIFIED result or lacks synthetic-origin evidence") + manifest["reporting_path_success"] = True + manifest["helper_exit_code"] = 0 + except Exception as error: + manifest["errors"].append(f"{type(error).__name__}: {error}") + finally: + # Preserve useful partial evidence even if generation or invocation + # failed. The original report and CLI logs are never rewritten here. + for field, operation in ( + ("dataset_files_sha256_after", lambda: file_inventory(dataset)), + ("source_files_sha256_after", lambda: {name: sha256_file(ROOT / name) for name in SOURCE_FILES}), + ): + try: + manifest[field] = operation() + except Exception as error: + manifest["errors"].append(f"Could not record {field}: {error}") + manifest["reporting_path_success"] = False + manifest["helper_exit_code"] = 1 + if manifest["dataset_files_sha256_before"]: + manifest["dataset_unchanged"] = manifest["dataset_files_sha256_before"] == manifest["dataset_files_sha256_after"] + try: + after_head, after_clean = git_state() + manifest["checker"].update(git_head_after=after_head, source_tree_clean_after=after_clean) + if (manifest["checker"]["git_head"] != after_head + or manifest["source_files_sha256"] != manifest["source_files_sha256_after"]): + raise ValueError("Checker source changed during evidence generation") + if expected_checker_sha is not None and not after_clean: + raise ValueError("Source tree is dirty after the exact-SHA evidence run") + if manifest["dataset_unchanged"] is False: + raise ValueError("Generated dataset differs from its pre-checker inventory") + manifest["checker"]["immutable_checkout_verified"] = bool( + expected_checker_sha == after_head and manifest["checker"]["source_tree_clean"] and after_clean) + except Exception as error: + manifest["errors"].append(f"Final identity verification failed: {error}") + manifest["reporting_path_success"] = False + manifest["helper_exit_code"] = 1 + if report_path.is_file() and not report_path.is_symlink(): + try: + manifest["report_sha256"] = sha256_file(report_path) + except OSError as error: + manifest["errors"].append(f"Could not hash report: {error}") + manifest["reporting_path_success"] = False + manifest["helper_exit_code"] = 1 + try: + (output_dir / "evidence.json").write_text(json.dumps(manifest, indent=2, sort_keys=True, allow_nan=False) + "\n") + except (OSError, ValueError) as error: + print(f"C10 partial evidence could not be saved: {error}", file=sys.stderr) + return 1 + + print(f"Checker HEAD: {manifest['checker']['git_head']}; source_tree_clean={manifest['checker']['source_tree_clean']}") + if "dataset_revision" in manifest["dataset"]: + print(f"Synthetic dataset: {manifest['dataset']['dataset_id']} at revision {manifest['dataset']['dataset_revision']}") + if manifest["invocation"]: + print(f"Checker command (cwd={ROOT}): {manifest['invocation']['shell_command']}") + if manifest["reporting_path_success"]: + print(f"C10 synthetic reporting path verified; raw gate remains NOT_VERIFIED (exit 2): {output_dir}") + else: + print(f"C10 evidence generation failed; retained available artifacts at {output_dir}", file=sys.stderr) + for error in manifest["errors"]: + print(f" {error}", file=sys.stderr) + return manifest["helper_exit_code"] + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", type=Path, required=True, + help="New evidence directory; existing paths are never modified") + parser.add_argument("--expected-checker-sha", + help="Require this full Git HEAD SHA and a clean source tree") + args = parser.parse_args(argv) + return generate_evidence(args.output_dir, args.expected_checker_sha) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/prove_c10_negative.py b/tools/prove_c10_negative.py new file mode 100644 index 0000000..f058c65 --- /dev/null +++ b/tools/prove_c10_negative.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Prove a specific corruption test detects reverting affine arithmetic QA. + +Only a temporary source copy is mutated. The production checkout and fixture +expectations are unchanged. Exit zero means the expected failure was observed. +""" + +from pathlib import Path +import subprocess +import sys +import tempfile + + +ROOT = Path(__file__).resolve().parents[1] +TARGET = "test_c10_dataset.C10SemanticTests.test_reject_affine_arithmetic" +CHILD = """ +import importlib.util +from pathlib import Path +import sys +import unittest +root, source, expected = Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3] +sys.path.insert(0, str(root / 'tools')) +sys.path.insert(0, str(root / 'tests')) +spec = importlib.util.spec_from_file_location('c10_semantics', source) +module = importlib.util.module_from_spec(spec) +sys.modules['c10_semantics'] = module +spec.loader.exec_module(module) +suite = unittest.defaultTestLoader.loadTestsFromName(sys.argv[4]) +result = unittest.TextTestRunner(verbosity=2).run(suite) +valid = (result.testsRun == 1 and not result.errors and not result.skipped + and len(result.failures) == int(expected)) +raise SystemExit(0 if valid else 1) +""" + + +def main(): + source = ROOT / "tools/c10_semantics.py" + original = source.read_text() + rule = 'calculated == stamp.get("mapped_authoritative_ns")' + if original.count(rule) != 1: + raise SystemExit("FAIL: arithmetic rule changed; update the precise revert experiment") + subprocess.run([sys.executable,"-c",CHILD,str(ROOT),str(source),"0",TARGET],check=True) + with tempfile.TemporaryDirectory(prefix="c10-reverted-rule-") as directory: + reverted = Path(directory) / source.name + reverted.write_text(original.replace(rule,"True # deliberately reverted arithmetic comparison\n ")) + subprocess.run([sys.executable,"-c",CHILD,str(ROOT),str(reverted),"1",TARGET],check=True) + if source.read_text() != original: + raise SystemExit("FAIL: source checkout changed during the experiment") + print("PASS: unchanged negative test passed on current code and failed exactly once on the reverted arithmetic rule; checkout preserved.") + + +if __name__ == "__main__": + main() diff --git a/tools/requirements-c10-native.txt b/tools/requirements-c10-native.txt new file mode 100644 index 0000000..4c3254e --- /dev/null +++ b/tools/requirements-c10-native.txt @@ -0,0 +1,3 @@ +-r requirements-c10.txt +# Optional native Parquet integration; PyPI pyarrow (Apache-2.0). +pyarrow>=17,<27 diff --git a/tools/requirements-c10.txt b/tools/requirements-c10.txt new file mode 100644 index 0000000..7f48a63 --- /dev/null +++ b/tools/requirements-c10.txt @@ -0,0 +1,4 @@ +# Core validators already used by tools/lint_schemas.py. +# PyPI jsonschema (MIT); PyYAML (MIT). No LeRobot runtime required. +jsonschema>=4.18,<5 +PyYAML>=6,<7 diff --git a/tools/verify.sh b/tools/verify.sh index 70ddc5c..d6877d9 100644 --- a/tools/verify.sh +++ b/tools/verify.sh @@ -41,6 +41,7 @@ clean_bash -c ' source /opt/ros/jazzy/setup.bash python3 "$1/tools/lint_schemas.py" --root "$1" --report "$2/lint-schema.json" python3 "$1/tools/run_verification_tests.py" "$1/tests" test_schema_validation.py + python3 "$1/tools/run_verification_tests.py" "$1/tests" test_c10_dataset.py ' verify "$root" "$run" 2>&1 | tee "$run/lint-schema.log" stage=interface-build