diff --git a/.github/workflows/arc-axis-collapse-corpus.yml b/.github/workflows/arc-axis-collapse-corpus.yml new file mode 100644 index 00000000..2c8697a2 --- /dev/null +++ b/.github/workflows/arc-axis-collapse-corpus.yml @@ -0,0 +1,336 @@ +name: ARC Axis Collapse Mechanistic Corpus + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-axis-collapse-corpus.yml' + +permissions: + contents: read + actions: read + +jobs: + training-corpus: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Download immutable benchmark corpus + uses: actions/download-artifact@v4 + with: + name: public-benchmarks-pinned-2026-08-04 + path: /tmp/pinned-public-benchmarks + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30929169392' + - name: Download immutable V2.2 baseline report + uses: actions/download-artifact@v4 + with: + name: arc-v2-2-training-e50965009efb96457fd769d97b42027f8a4e98c0 + path: /tmp/arc-v2-2-baseline + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30941837872' + - name: Verify V2.2 and frozen V2 gates + env: + PYTHONPATH: research + run: | + python -m pip install --disable-pip-version-check --quiet pytest + python -m pytest -q research/universal_core/holonomy_v2_2/tests + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py + git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Run all ARC training tasks with complete traces + run: | + mkdir -p /tmp/arc-v2-2-axis/traces + python - <<'PY' | tee /tmp/arc-v2-2-axis/console-summary.json + import json + import os + from pathlib import Path + + from research.universal_core.holonomy_v2.engine import HolonomyEngine + from research.universal_core.holonomy_v2.types import SearchConfig + from research.universal_core.holonomy_v2_2.mechanistic_trace import validate_trace + from research.universal_core.holonomy_v2_2.training_eval import run_training_corpus, write_report + from research.universal_core.public_benchmarks.suite import locate_vendor + + output = Path('/tmp/arc-v2-2-axis') + trace_dir = output / 'traces' + vendor = locate_vendor(Path('/tmp/pinned-public-benchmarks')) + report = run_training_corpus( + vendor, + HolonomyEngine(SearchConfig()), + run_label='arc-v2-2-axis-' + os.environ['GITHUB_SHA'][:12], + candidate_commit=os.environ['GITHUB_SHA'], + mechanistic_dir=trace_dir, + verify_parity=True, + ) + write_report(report, output / 'report.json') + + trace_paths = sorted( + path for path in trace_dir.glob('*.json') + if path.name != 'diagnosis.json' + ) + if len(trace_paths) != 1000: + raise SystemExit(f'expected 1000 task traces, found {len(trace_paths)}') + traces = {} + for path in trace_paths: + trace = json.loads(path.read_text(encoding='utf-8')) + validate_trace(trace) + traces[path.stem] = trace + if report['mechanistic']['trace_count'] != 1000: + raise SystemExit('aggregate trace count mismatch') + if report['mechanistic']['parity_count'] != 1000: + raise SystemExit('compact/mechanistic parity count mismatch') + + baseline = json.loads( + Path('/tmp/arc-v2-2-baseline/report.json').read_text(encoding='utf-8') + ) + baseline_rows = { + item['task_id']: int(item['metrics']['correct']) + for item in baseline['reports'] + } + current_rows = { + item['task_id']: int(item['metrics']['correct']) + for item in report['reports'] + } + lost_rows = { + task_id: { + 'baseline_correct': correct, + 'current_correct': current_rows.get(task_id, 0), + } + for task_id, correct in baseline_rows.items() + if current_rows.get(task_id, 0) < correct + } + gained_rows = { + task_id: { + 'baseline_correct': baseline_rows.get(task_id, 0), + 'current_correct': correct, + } + for task_id, correct in current_rows.items() + if correct > baseline_rows.get(task_id, 0) + } + expected_gain_ids = [ + '0c786b71', '2dee498d', '3aa6fb7a', '3af2c5a8', '46442a0e', + '4c4377d9', '62c24649', '67e8384a', '6d0aefbc', '6d75e8bb', + '6fa7a44f', '746b3537', '7953d61e', '7b7f7511', '7fe24cdd', + '833dafe3', '8be77c9e', 'a59b95c0', 'aabf363d', 'ac0a08a4', + 'b230c067', 'b91ae062', 'c9e6f938', 'ce8d95cc', 'd4b1c2b1', + 'e1baa8a4', 'e7639916', 'eb5a1d5d', 'ed98d772', + ] + expected_gains = { + task_id: {'baseline_correct': 0, 'current_correct': 1} + for task_id in expected_gain_ids + } + if lost_rows: + raise SystemExit('lost baseline-correct rows: ' + json.dumps(lost_rows, sort_keys=True)) + if gained_rows != expected_gains: + raise SystemExit( + 'unexpected gained rows: expected ' + + json.dumps(expected_gains, sort_keys=True) + + ' found ' + + json.dumps(gained_rows, sort_keys=True) + ) + + expected_metrics = { + 'rows': 1076, + 'correct': 88, + 'accepted': 88, + 'incorrect_attempts': 0, + 'failed': 0, + } + for key, expected in expected_metrics.items(): + if int(report['metrics'][key]) != expected: + raise SystemExit( + f'metric {key} changed: expected {expected}, found {report["metrics"][key]}' + ) + expected_status = { + 'ACCEPTED': 88, + 'AMBIGUOUS_PROGRAM': 1, + 'GRAMMAR_EXHAUSTED': 987, + } + if report['status_counts'] != expected_status: + raise SystemExit( + 'status counts changed: expected ' + + json.dumps(expected_status, sort_keys=True) + + ' found ' + + json.dumps(report['status_counts'], sort_keys=True) + ) + + axis_ids = ('2dee498d', '746b3537', '7b7f7511', 'ce8d95cc', 'e1baa8a4', 'eb5a1d5d') + axis_details = {} + for task_id in axis_ids: + trace = traces[task_id] + final = [event for event in trace['events'] if event.get('event') == 'final_decision'][0] + successes = [ + event for event in trace['events'] + if event.get('event') == 'hidden_execution' + and event.get('candidate_kind') == 'axis_duplicate_collapse' + and event.get('outcome') == 'SUCCEEDED' + ] + if trace['terminal_status'] != 'ACCEPTED': + raise SystemExit(f'{task_id} did not transition to ACCEPTED') + if not successes: + raise SystemExit(f'{task_id} has no successful axis-collapse candidate') + if not str(final.get('selected_candidate_id', '')).startswith('axis_collapse:'): + raise SystemExit(f'{task_id} did not select axis collapse') + axis_details[task_id] = { + 'trace_digest': trace['trace_digest'], + 'successful_axis_candidates': len(successes), + 'selected_candidate_id': final['selected_candidate_id'], + } + + primitive_guards = { + 'aabf363d': 'marker_recolor:', + 'b230c067': 'component_outlier:', + '3aa6fb7a': 'bbox_complete:', + '6d75e8bb': 'bbox_complete:', + 'e7639916': 'bbox_complete:', + '4c4377d9': 'mirror_concat:', + '6d0aefbc': 'mirror_concat:', + '6fa7a44f': 'mirror_concat:', + '8be77c9e': 'mirror_concat:', + 'c9e6f938': 'mirror_concat:', + '0c786b71': 'quadrant_mosaic:', + '3af2c5a8': 'quadrant_mosaic:', + '46442a0e': 'quadrant_mosaic:', + '62c24649': 'quadrant_mosaic:', + '67e8384a': 'quadrant_mosaic:', + '7953d61e': 'quadrant_mosaic:', + '7fe24cdd': 'quadrant_mosaic:', + '833dafe3': 'quadrant_mosaic:', + 'ed98d772': 'quadrant_mosaic:', + 'a59b95c0': 'palette_repeat:', + 'ac0a08a4': 'palette_repeat:', + 'b91ae062': 'palette_repeat:', + 'd4b1c2b1': 'palette_repeat:', + } + for task_id, prefix in primitive_guards.items(): + trace = traces[task_id] + final = [event for event in trace['events'] if event.get('event') == 'final_decision'][0] + if trace['terminal_status'] != 'ACCEPTED': + raise SystemExit(f'{task_id} regressed') + if not str(final.get('selected_candidate_id', '')).startswith(prefix): + raise SystemExit(f'{task_id} selected primitive changed') + + ambiguous = traces['60b61512'] + classes = [event for event in ambiguous['events'] if event.get('event') == 'equivalence_class'] + bbox_classes = [ + event for event in classes + if any(str(candidate_id).startswith('bbox_complete:') for candidate_id in event['candidate_ids']) + ] + if ambiguous['terminal_status'] != 'AMBIGUOUS_PROGRAM': + raise SystemExit('60b61512 did not remain ambiguity-safe') + if len(bbox_classes) < 2: + raise SystemExit('60b61512 lost bounding-box disagreement') + + transitions = { + 'schema_version': 1, + 'candidate_commit': os.environ['GITHUB_SHA'], + 'baseline_commit': 'e50965009efb96457fd769d97b42027f8a4e98c0', + 'lost_rows': lost_rows, + 'gained_rows': gained_rows, + 'expected_gains_matched': True, + 'incorrect_attempts': int(report['metrics']['incorrect_attempts']), + } + mechanisms = { + 'schema_version': 1, + 'axis_tasks': axis_details, + 'ownership_boundary': 'NO_DUPLICATE_AXIS_REDUCTION', + '60b61512': { + 'terminal_status': ambiguous['terminal_status'], + 'trace_digest': ambiguous['trace_digest'], + 'bbox_equivalence_class_count': len(bbox_classes), + 'safety_action': 'abstain_on_connectivity_disagreement', + }, + } + parity = { + 'schema_version': 1, + 'candidate_commit': os.environ['GITHUB_SHA'], + 'checked_tasks': 1000, + 'mismatches': 0, + 'lost_baseline_rows': lost_rows, + } + for name, data in ( + ('transitions.json', transitions), + ('mechanisms.json', mechanisms), + ('parity.json', parity), + ): + (output / name).write_text( + json.dumps(data, sort_keys=True, indent=2) + '\n', + encoding='utf-8', + ) + + print(json.dumps({ + 'report_digest': report['report_digest'], + 'diagnosis_digest': report['mechanistic']['diagnosis_digest'], + 'evaluated_tasks': report['evaluated_tasks'], + 'trace_count': report['mechanistic']['trace_count'], + 'parity_count': report['mechanistic']['parity_count'], + 'metrics': report['metrics'], + 'status_counts': report['status_counts'], + 'transitions': transitions, + 'mechanisms': mechanisms, + }, sort_keys=True)) + PY + - name: Build checksum manifest after all writers close + run: | + python - <<'PY' + import hashlib + from pathlib import Path + + output = Path('/tmp/arc-v2-2-axis') + lines = [] + for path in sorted( + item for item in output.rglob('*') + if item.is_file() and item.name != 'SHA256SUMS' + ): + digest = hashlib.sha256(path.read_bytes()).hexdigest() + lines.append(f'{digest} {path.relative_to(output).as_posix()}') + (output / 'SHA256SUMS').write_text('\n'.join(lines) + '\n', encoding='utf-8') + PY + - name: Verify checksum manifest independently + working-directory: /tmp/arc-v2-2-axis + run: sha256sum -c SHA256SUMS + - name: Publish axis-collapse evidence + uses: actions/upload-artifact@v4 + with: + name: arc-v2-2-axis-collapse-${{ github.sha }} + path: /tmp/arc-v2-2-axis + if-no-files-found: error + retention-days: 90 + compression-level: 6 + - name: Write run summary + run: | + python - <<'PY' + import json + from pathlib import Path + + root = Path('/tmp/arc-v2-2-axis') + report = json.loads((root / 'report.json').read_text()) + mechanisms = json.loads((root / 'mechanisms.json').read_text()) + lines = [ + '# ARC V2.2 axis duplicate collapse corpus', + '', + f"- Candidate: `{report['identity']['candidate_commit']}`", + f"- Report digest: `{report['report_digest']}`", + f"- Diagnosis digest: `{report['mechanistic']['diagnosis_digest']}`", + f"- Metrics: `{report['metrics']}`", + f"- Status counts: `{report['status_counts']}`", + f"- Axis gains: `{sorted(mechanisms['axis_tasks'])}`", + f"- Ownership boundary: `{mechanisms['ownership_boundary']}`", + f"- 60b61512 action: `{mechanisms['60b61512']['safety_action']}`", + '- Public ARC evaluation rerun: `no`', + ] + Path('/tmp/axis-summary.md').write_text('\n'.join(lines) + '\n', encoding='utf-8') + PY + cat /tmp/axis-summary.md >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/arc-axis-collapse-module-green.yml b/.github/workflows/arc-axis-collapse-module-green.yml new file mode 100644 index 00000000..c52c0a0e --- /dev/null +++ b/.github/workflows/arc-axis-collapse-module-green.yml @@ -0,0 +1,64 @@ +name: ARC Axis Collapse Module GREEN + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-axis-collapse-module-green.yml' + +permissions: + contents: write + +jobs: + verify-module: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Run isolated axis tests + run: | + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_axis_collapse_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_axis_collapse.py::test_adjacent_run_collapse_rows_columns_and_both \ + research/universal_core/holonomy_v2_2/tests/test_axis_collapse.py::test_global_first_collapse_preserves_first_occurrence_order \ + research/universal_core/holonomy_v2_2/tests/test_axis_collapse.py::test_row_and_column_collapse_commute \ + research/universal_core/holonomy_v2_2/tests/test_axis_collapse.py::test_program_derivation_skips_when_no_axis_reduction_occurs \ + | tee /tmp/module-green.log + - name: Confirm frozen bytes + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Commit module evidence + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + evidence = { + 'schema_version': 1, + 'source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': True, + 'test_count': 5, + 'test_output': Path('/tmp/module-green.log').read_text(encoding='utf-8'), + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'frozen_byte_gate': True, + 'public_arc_evaluation_rerun': False, + } + path = Path('research/universal_core/holonomy_v2_2/evidence/AXIS_COLLAPSE_MODULE_GREEN.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/AXIS_COLLAPSE_MODULE_GREEN.json + git commit -m 'arc: record axis collapse module GREEN' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-axis-collapse-red.yml b/.github/workflows/arc-axis-collapse-red.yml new file mode 100644 index 00000000..ee9d759f --- /dev/null +++ b/.github/workflows/arc-axis-collapse-red.yml @@ -0,0 +1,72 @@ +name: ARC Axis Collapse RED + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-axis-collapse-red.yml' + +permissions: + contents: write + +jobs: + verify-red: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - run: python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_axis_collapse_fixtures.py + - name: Require missing axis module + run: | + set +e + python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_axis_collapse.py > /tmp/axis-red.log 2>&1 + status=$? + set -e + cat /tmp/axis-red.log + test "$status" -ne 0 + grep -F "No module named 'research.universal_core.holonomy_v2_2.axis_collapse'" /tmp/axis-red.log + - name: Commit RED evidence + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + evidence = { + 'schema_version': 1, + 'source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'failure_confirmed': True, + 'fixture_integrity': True, + 'expected_failure': "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.axis_collapse'", + 'expected_new_exact_tasks': [ + '2dee498d', '746b3537', '7b7f7511', 'ce8d95cc', 'e1baa8a4', 'eb5a1d5d' + ], + 'training_scan': { + 'compatible_tasks': 6, + 'wrong_completed_hidden_outputs': 0, + 'ambiguous_hidden_outputs': 0, + 'compatible_accepted_tasks': 0, + }, + 'captured_log': Path('/tmp/axis-red.log').read_text(encoding='utf-8'), + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'public_arc_evaluation_rerun': False, + } + path = Path('research/universal_core/holonomy_v2_2/evidence/AXIS_COLLAPSE_RED.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/AXIS_COLLAPSE_RED.json + git commit -m 'arc: record axis collapse RED evidence' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-axis-collapse-wire.yml b/.github/workflows/arc-axis-collapse-wire.yml new file mode 100644 index 00000000..ce5207d4 --- /dev/null +++ b/.github/workflows/arc-axis-collapse-wire.yml @@ -0,0 +1,195 @@ +name: Wire ARC Axis Collapse + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-axis-collapse-wire.yml' + +permissions: + contents: write + +jobs: + wire-and-verify: + runs-on: ubuntu-latest + timeout-minutes: 50 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Apply guarded axis integration + run: | + python - <<'PY' + from pathlib import Path + + grid_path = Path('research/universal_core/holonomy_v2_2/grid_induction.py') + text = grid_path.read_text(encoding='utf-8') + import_anchor = 'from .palette_repeat import apply_palette_repeat\n' + if text.count(import_anchor) != 1 or 'from .axis_collapse import' in text: + raise SystemExit('axis import anchor mismatch') + text = text.replace( + import_anchor, + import_anchor + 'from .axis_collapse import apply_axis_collapse\n', + 1, + ) + if text.count('_VERSION = "grid-v2.2-11"') != 1: + raise SystemExit('axis version anchor mismatch') + text = text.replace('_VERSION = "grid-v2.2-11"', '_VERSION = "grid-v2.2-12"', 1) + apply_anchor = ' if kind == "palette_cardinality_repeat":\n' + if text.count(apply_anchor) != 1: + raise SystemExit('axis apply anchor mismatch') + text = text.replace( + apply_anchor, + ' if kind == "axis_duplicate_collapse":\n' + ' return apply_axis_collapse(program, grid)\n' + + apply_anchor, + 1, + ) + grid_path.write_text(text, encoding='utf-8') + + runtime_path = Path('research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py') + text = runtime_path.read_text(encoding='utf-8') + import_anchor = 'from .palette_repeat import palette_repeat_programs\n' + if text.count(import_anchor) != 1 or 'axis_collapse_programs' in text: + raise SystemExit('axis runtime import anchor mismatch') + text = text.replace( + import_anchor, + import_anchor + 'from .axis_collapse import axis_collapse_programs\n', + 1, + ) + derive_anchor = ' palette_repeat, palette_reason = palette_repeat_programs(demos)\n' + if text.count(derive_anchor) != 1: + raise SystemExit('axis derivation anchor mismatch') + text = text.replace( + derive_anchor, + derive_anchor + ' axis_collapse, axis_reason = axis_collapse_programs(demos)\n', + 1, + ) + batch_anchor = ' ("palette_repeat", palette_repeat, palette_reason),\n' + if text.count(batch_anchor) != 1: + raise SystemExit('axis batch anchor mismatch') + text = text.replace( + batch_anchor, + batch_anchor + ' ("axis_collapse", axis_collapse, axis_reason),\n', + 1, + ) + runtime_path.write_text(text, encoding='utf-8') + PY + python -m py_compile \ + research/universal_core/holonomy_v2_2/axis_collapse.py \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py + git diff --check + - name: Run focused axis tests + id: focused + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_axis_collapse_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_axis_collapse.py \ + research/universal_core/holonomy_v2_2/tests/test_palette_repeat_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_palette_repeat.py \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py \ + research/universal_core/holonomy_v2_2/tests/test_mirror_concat.py \ + research/universal_core/holonomy_v2_2/tests/test_bbox_complete.py \ + research/universal_core/holonomy_v2_2/tests/test_component_outlier.py \ + research/universal_core/holonomy_v2_2/tests/test_marker_recolor.py \ + research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py \ + research/universal_core/holonomy_v2_2/tests/test_grid_induction.py::test_cell_scale_and_whole_grid_tile_are_distinct \ + | tee /tmp/focused.log + - name: Run all V2.2 tests + id: cluster + if: always() + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q research/universal_core/holonomy_v2_2/tests | tee /tmp/v2-2.log + - name: Run frozen V2 integration tests + id: frozen + if: always() + continue-on-error: true + env: + PYTHONPATH: research + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py | tee /tmp/frozen.log + - name: Confirm frozen V2 bytes are unchanged + id: byte_gate + if: always() + continue-on-error: true + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Record result and commit only valid state + if: always() + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + FOCUSED_OUTCOME: ${{ steps.focused.outcome }} + CLUSTER_OUTCOME: ${{ steps.cluster.outcome }} + FROZEN_OUTCOME: ${{ steps.frozen.outcome }} + BYTE_OUTCOME: ${{ steps.byte_gate.outcome }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + outcomes = { + 'focused': os.environ['FOCUSED_OUTCOME'], + 'v2_2_cluster': os.environ['CLUSTER_OUTCOME'], + 'frozen_tests': os.environ['FROZEN_OUTCOME'], + 'frozen_byte_gate': os.environ['BYTE_OUTCOME'], + } + passed = all(value == 'success' for value in outcomes.values()) + evidence = { + 'schema_version': 1, + 'workflow_source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': passed, + 'outcomes': outcomes, + 'focused_output': Path('/tmp/focused.log').read_text(encoding='utf-8') if Path('/tmp/focused.log').exists() else '', + 'v2_2_output': Path('/tmp/v2-2.log').read_text(encoding='utf-8') if Path('/tmp/v2-2.log').exists() else '', + 'frozen_output': Path('/tmp/frozen.log').read_text(encoding='utf-8') if Path('/tmp/frozen.log').exists() else '', + 'ownership_boundary': 'NO_DUPLICATE_AXIS_REDUCTION', + 'expected_new_exact_tasks': [ + '2dee498d', '746b3537', '7b7f7511', 'ce8d95cc', 'e1baa8a4', 'eb5a1d5d' + ], + 'expected_training_delta': { + 'correct_rows': 6, + 'accepted_rows': 6, + 'grammar_exhausted_rows': -6, + }, + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'public_arc_evaluation_rerun': False, + } + name = 'AXIS_COLLAPSE_GREEN.json' if passed else 'AXIS_COLLAPSE_DIAGNOSTIC.json' + path = Path('research/universal_core/holonomy_v2_2/evidence') / name + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + Path('/tmp/passed').write_text('1' if passed else '0', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + if [ "$(cat /tmp/passed)" = "1" ]; then + git add \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py \ + research/universal_core/holonomy_v2_2/evidence/AXIS_COLLAPSE_GREEN.json + git commit -m 'arc: wire axis duplicate collapse' + else + git restore \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py + git add research/universal_core/holonomy_v2_2/evidence/AXIS_COLLAPSE_DIAGNOSTIC.json + git commit -m 'arc: record axis collapse diagnostic failure' + fi + git push origin HEAD:universal-core-arc-training-v2-2 + test "$(cat /tmp/passed)" = "1" diff --git a/.github/workflows/arc-axis-fixture-restore.yml b/.github/workflows/arc-axis-fixture-restore.yml new file mode 100644 index 00000000..71fcaa1c --- /dev/null +++ b/.github/workflows/arc-axis-fixture-restore.yml @@ -0,0 +1,83 @@ +name: Restore ARC Axis Collapse Fixtures + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-axis-fixture-restore.yml' + +permissions: + contents: write + actions: read + +jobs: + restore-and-verify: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Download immutable benchmark corpus + uses: actions/download-artifact@v4 + with: + name: public-benchmarks-pinned-2026-08-04 + path: /tmp/pinned-public-benchmarks + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30929169392' + - name: Restore exact fixture bytes + run: | + set -euo pipefail + vendor=$(find /tmp/pinned-public-benchmarks -type d -path '*/arc_agi_2/data/training' | head -1) + test -n "$vendor" + destination=research/universal_core/holonomy_v2_2/tests/fixtures/axis_collapse + mkdir -p "$destination" + for task_id in 2dee498d 746b3537 7b7f7511 ce8d95cc e1baa8a4 eb5a1d5d; do + cp "$vendor/$task_id.json" "$destination/$task_id.json" + done + printf '%s %s\n' \ + 38e27a5c17bf593547182b8914cf044052b158ebe4c5e802d14fb6af11458388 "$destination/2dee498d.json" \ + 2fa42394d0d48c4c9534aac1ce000ab97ff804f9d69a34d16126f6a17a757686 "$destination/746b3537.json" \ + 28480a33bbe9ff487d845a62a7c21f3d5bc2d1de493a5f91e0c19b57b1aa1d60 "$destination/7b7f7511.json" \ + 728025ba05aacc3b678ecd7a75d605bf8d9477c2bcd2b667e031c32406233c41 "$destination/ce8d95cc.json" \ + 16411bc668a55e51401716d4fc48424413443a8b21f97a670ce14df9ba325f9d "$destination/e1baa8a4.json" \ + 96a1772a93e9cecf1050532b031ad3605390cc5c907cb4b919d4e095eee2f54d "$destination/eb5a1d5d.json" \ + | sha256sum -c - + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Verify fixture regression + run: python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_axis_collapse_fixtures.py | tee /tmp/fixture-green.log + - name: Commit exact fixtures and evidence + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + evidence = { + 'schema_version': 1, + 'workflow_source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'fixture_count': 6, + 'fixture_integrity': True, + 'source_artifact_run': 30929169392, + 'test_output': Path('/tmp/fixture-green.log').read_text(encoding='utf-8'), + 'public_arc_evaluation_rerun': False, + } + path = Path('research/universal_core/holonomy_v2_2/evidence/AXIS_FIXTURE_INTEGRITY_GREEN.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add \ + research/universal_core/holonomy_v2_2/tests/fixtures/axis_collapse \ + research/universal_core/holonomy_v2_2/evidence/AXIS_FIXTURE_INTEGRITY_GREEN.json + git commit -m 'arc: restore pinned axis collapse fixtures' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-bbox-corpus.yml b/.github/workflows/arc-bbox-corpus.yml new file mode 100644 index 00000000..0cdb4e7c --- /dev/null +++ b/.github/workflows/arc-bbox-corpus.yml @@ -0,0 +1,310 @@ +name: ARC Bounding Box Mechanistic Corpus + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-bbox-corpus.yml' + +permissions: + contents: read + actions: read + +jobs: + training-corpus: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Download immutable benchmark corpus + uses: actions/download-artifact@v4 + with: + name: public-benchmarks-pinned-2026-08-04 + path: /tmp/pinned-public-benchmarks + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30929169392' + - name: Download immutable V2.2 baseline report + uses: actions/download-artifact@v4 + with: + name: arc-v2-2-training-e50965009efb96457fd769d97b42027f8a4e98c0 + path: /tmp/arc-v2-2-baseline + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30941837872' + - name: Verify V2.2 and frozen V2 gates + env: + PYTHONPATH: research + run: | + python -m pip install --disable-pip-version-check --quiet pytest + python -m pytest -q research/universal_core/holonomy_v2_2/tests + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py + git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Run all ARC training tasks with complete traces + run: | + mkdir -p /tmp/arc-v2-2-bbox/traces + python - <<'PY' | tee /tmp/arc-v2-2-bbox/console-summary.json + import json + import os + from pathlib import Path + + from research.universal_core.holonomy_v2.engine import HolonomyEngine + from research.universal_core.holonomy_v2.types import SearchConfig + from research.universal_core.holonomy_v2_2.mechanistic_trace import validate_trace + from research.universal_core.holonomy_v2_2.training_eval import run_training_corpus, write_report + from research.universal_core.public_benchmarks.suite import locate_vendor + + output = Path('/tmp/arc-v2-2-bbox') + trace_dir = output / 'traces' + vendor = locate_vendor(Path('/tmp/pinned-public-benchmarks')) + report = run_training_corpus( + vendor, + HolonomyEngine(SearchConfig()), + run_label='arc-v2-2-bbox-' + os.environ['GITHUB_SHA'][:12], + candidate_commit=os.environ['GITHUB_SHA'], + mechanistic_dir=trace_dir, + verify_parity=True, + ) + write_report(report, output / 'report.json') + + trace_paths = sorted( + path for path in trace_dir.glob('*.json') + if path.name != 'diagnosis.json' + ) + if len(trace_paths) != 1000: + raise SystemExit(f'expected 1000 task traces, found {len(trace_paths)}') + traces = {} + for path in trace_paths: + trace = json.loads(path.read_text(encoding='utf-8')) + validate_trace(trace) + traces[path.stem] = trace + if report['mechanistic']['trace_count'] != 1000: + raise SystemExit('aggregate trace count mismatch') + if report['mechanistic']['parity_count'] != 1000: + raise SystemExit('compact/mechanistic parity count mismatch') + + baseline = json.loads( + Path('/tmp/arc-v2-2-baseline/report.json').read_text(encoding='utf-8') + ) + baseline_rows = { + item['task_id']: int(item['metrics']['correct']) + for item in baseline['reports'] + } + current_rows = { + item['task_id']: int(item['metrics']['correct']) + for item in report['reports'] + } + lost_rows = { + task_id: { + 'baseline_correct': correct, + 'current_correct': current_rows.get(task_id, 0), + } + for task_id, correct in baseline_rows.items() + if current_rows.get(task_id, 0) < correct + } + gained_rows = { + task_id: { + 'baseline_correct': baseline_rows.get(task_id, 0), + 'current_correct': correct, + } + for task_id, correct in current_rows.items() + if correct > baseline_rows.get(task_id, 0) + } + expected_gains = { + '3aa6fb7a': {'baseline_correct': 0, 'current_correct': 1}, + '6d75e8bb': {'baseline_correct': 0, 'current_correct': 1}, + 'aabf363d': {'baseline_correct': 0, 'current_correct': 1}, + 'b230c067': {'baseline_correct': 0, 'current_correct': 1}, + 'e7639916': {'baseline_correct': 0, 'current_correct': 1}, + } + if lost_rows: + raise SystemExit('lost baseline-correct rows: ' + json.dumps(lost_rows, sort_keys=True)) + if gained_rows != expected_gains: + raise SystemExit( + 'unexpected gained rows: expected ' + + json.dumps(expected_gains, sort_keys=True) + + ' found ' + + json.dumps(gained_rows, sort_keys=True) + ) + + expected_metrics = { + 'rows': 1076, + 'correct': 64, + 'accepted': 64, + 'incorrect_attempts': 0, + } + for key, expected in expected_metrics.items(): + if int(report['metrics'][key]) != expected: + raise SystemExit( + f'metric {key} changed: expected {expected}, found {report["metrics"][key]}' + ) + expected_status = { + 'ACCEPTED': 64, + 'AMBIGUOUS_PROGRAM': 1, + 'GRAMMAR_EXHAUSTED': 1011, + } + if report['status_counts'] != expected_status: + raise SystemExit( + 'status counts changed: expected ' + + json.dumps(expected_status, sort_keys=True) + + ' found ' + + json.dumps(report['status_counts'], sort_keys=True) + ) + + accepted_details = {} + for task_id in ('3aa6fb7a', '6d75e8bb', 'e7639916'): + trace = traces[task_id] + final = [event for event in trace['events'] if event.get('event') == 'final_decision'][0] + bbox_successes = [ + event for event in trace['events'] + if event.get('event') == 'hidden_execution' + and event.get('candidate_kind') == 'bounding_box_complete' + and event.get('outcome') == 'SUCCEEDED' + ] + if trace['terminal_status'] != 'ACCEPTED': + raise SystemExit(f'{task_id} did not transition to ACCEPTED') + if not bbox_successes: + raise SystemExit(f'{task_id} has no successful bounding-box candidate') + if not str(final.get('selected_candidate_id', '')).startswith('bbox_complete:'): + raise SystemExit(f'{task_id} did not select bounding-box completion') + accepted_details[task_id] = { + 'trace_digest': trace['trace_digest'], + 'successful_bbox_candidates': len(bbox_successes), + 'selected_candidate_id': final['selected_candidate_id'], + } + + ambiguous = traces['60b61512'] + classes = [event for event in ambiguous['events'] if event.get('event') == 'equivalence_class'] + bbox_class_indices = { + index + for index, event in enumerate(classes) + if any(str(candidate_id).startswith('bbox_complete:') for candidate_id in event['candidate_ids']) + } + if ambiguous['terminal_status'] != 'AMBIGUOUS_PROGRAM': + raise SystemExit('60b61512 did not remain ambiguity-safe') + if len(classes) < 2 or len(bbox_class_indices) < 2: + raise SystemExit('60b61512 does not expose multiple bounding-box output classes') + + for task_id, prefix in ( + ('aabf363d', 'marker_recolor:'), + ('b230c067', 'component_outlier:'), + ): + trace = traces[task_id] + final = [event for event in trace['events'] if event.get('event') == 'final_decision'][0] + if trace['terminal_status'] != 'ACCEPTED': + raise SystemExit(f'{task_id} regressed') + if not str(final.get('selected_candidate_id', '')).startswith(prefix): + raise SystemExit(f'{task_id} selected primitive changed') + + transitions = { + 'schema_version': 1, + 'candidate_commit': os.environ['GITHUB_SHA'], + 'baseline_commit': 'e50965009efb96457fd769d97b42027f8a4e98c0', + 'lost_rows': lost_rows, + 'gained_rows': gained_rows, + 'expected_gains_matched': True, + 'incorrect_attempts': int(report['metrics']['incorrect_attempts']), + } + (output / 'transitions.json').write_text( + json.dumps(transitions, sort_keys=True, indent=2) + '\n', + encoding='utf-8', + ) + mechanisms = { + 'schema_version': 1, + 'accepted_bbox_tasks': accepted_details, + '60b61512': { + 'terminal_status': ambiguous['terminal_status'], + 'trace_digest': ambiguous['trace_digest'], + 'equivalence_class_count': len(classes), + 'bbox_equivalence_class_count': len(bbox_class_indices), + 'safety_action': 'abstain_on_connectivity_disagreement', + }, + } + (output / 'mechanisms.json').write_text( + json.dumps(mechanisms, sort_keys=True, indent=2) + '\n', + encoding='utf-8', + ) + parity = { + 'schema_version': 1, + 'candidate_commit': os.environ['GITHUB_SHA'], + 'checked_tasks': 1000, + 'mismatches': 0, + 'lost_baseline_rows': lost_rows, + } + (output / 'parity.json').write_text( + json.dumps(parity, sort_keys=True, indent=2) + '\n', + encoding='utf-8', + ) + + print(json.dumps({ + 'report_digest': report['report_digest'], + 'diagnosis_digest': report['mechanistic']['diagnosis_digest'], + 'evaluated_tasks': report['evaluated_tasks'], + 'trace_count': report['mechanistic']['trace_count'], + 'parity_count': report['mechanistic']['parity_count'], + 'metrics': report['metrics'], + 'status_counts': report['status_counts'], + 'transitions': transitions, + 'mechanisms': mechanisms, + }, sort_keys=True)) + PY + - name: Build checksum manifest after all writers close + run: | + python - <<'PY' + import hashlib + from pathlib import Path + + output = Path('/tmp/arc-v2-2-bbox') + lines = [] + for path in sorted( + item for item in output.rglob('*') + if item.is_file() and item.name != 'SHA256SUMS' + ): + digest = hashlib.sha256(path.read_bytes()).hexdigest() + lines.append(f'{digest} {path.relative_to(output).as_posix()}') + (output / 'SHA256SUMS').write_text('\n'.join(lines) + '\n', encoding='utf-8') + PY + - name: Verify checksum manifest independently + working-directory: /tmp/arc-v2-2-bbox + run: sha256sum -c SHA256SUMS + - name: Publish bounding-box evidence + uses: actions/upload-artifact@v4 + with: + name: arc-v2-2-bbox-${{ github.sha }} + path: /tmp/arc-v2-2-bbox + if-no-files-found: error + retention-days: 90 + compression-level: 6 + - name: Write run summary + run: | + python - <<'PY' + import json + from pathlib import Path + + root = Path('/tmp/arc-v2-2-bbox') + report = json.loads((root / 'report.json').read_text()) + mechanisms = json.loads((root / 'mechanisms.json').read_text()) + lines = [ + '# ARC V2.2 bounding-box corpus', + '', + f"- Candidate: `{report['identity']['candidate_commit']}`", + f"- Report digest: `{report['report_digest']}`", + f"- Diagnosis digest: `{report['mechanistic']['diagnosis_digest']}`", + f"- Metrics: `{report['metrics']}`", + f"- Status counts: `{report['status_counts']}`", + f"- Safe new tasks: `{sorted(mechanisms['accepted_bbox_tasks'])}`", + f"- 60b61512 action: `{mechanisms['60b61512']['safety_action']}`", + '- Public ARC evaluation rerun: `no`', + ] + Path('/tmp/bbox-summary.md').write_text('\n'.join(lines) + '\n', encoding='utf-8') + PY + cat /tmp/bbox-summary.md >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/arc-bbox-evidence-capture.yml b/.github/workflows/arc-bbox-evidence-capture.yml new file mode 100644 index 00000000..c18512e2 --- /dev/null +++ b/.github/workflows/arc-bbox-evidence-capture.yml @@ -0,0 +1,147 @@ +name: Capture ARC Bounding Box Evidence + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-bbox-evidence-capture.yml' + +permissions: + actions: read + contents: write + +jobs: + capture: + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + GH_TOKEN: ${{ github.token }} + TARGET_SHA: f92ca46da7698522dfd3cd73be0e0a28e2a158f5 + REPOSITORY: Chrisbryan17/MLE-agent + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - name: Resolve exact successful run and artifact + run: | + set -euo pipefail + runs=$(gh api --method GET "repos/$REPOSITORY/actions/runs" -f head_sha="$TARGET_SHA" -f per_page=100) + run_id=$(printf '%s' "$runs" | jq -r ' + [.workflow_runs[] + | select(.name == "ARC Bounding Box Mechanistic Corpus") + | select(.head_sha == env.TARGET_SHA) + | select(.status == "completed")] + | sort_by(.created_at) + | last + | .id // empty') + test -n "$run_id" + run=$(gh api "repos/$REPOSITORY/actions/runs/$run_id") + conclusion=$(printf '%s' "$run" | jq -r '.conclusion') + test "$conclusion" = success + artifacts=$(gh api "repos/$REPOSITORY/actions/runs/$run_id/artifacts?per_page=100") + artifact_id=$(printf '%s' "$artifacts" | jq -r --arg name "arc-v2-2-bbox-$TARGET_SHA" ' + [.artifacts[] | select(.name == $name)] | first | .id // empty') + test -n "$artifact_id" + printf '%s\n' "$run_id" > /tmp/run-id + printf '%s\n' "$artifact_id" > /tmp/artifact-id + printf '%s' "$run" > /tmp/run.json + printf '%s' "$artifacts" > /tmp/artifacts.json + - name: Download exact artifact + run: | + mkdir -p /tmp/bbox-artifact + gh run download "$(cat /tmp/run-id)" \ + --repo "$REPOSITORY" \ + --name "arc-v2-2-bbox-$TARGET_SHA" \ + --dir /tmp/bbox-artifact + - name: Independently validate artifact + run: | + cd /tmp/bbox-artifact + sha256sum -c SHA256SUMS + python - <<'PY' + import json + from pathlib import Path + from research.universal_core.holonomy_v2_2.mechanistic_trace import validate_trace + + root = Path('/tmp/bbox-artifact') + report = json.loads((root / 'report.json').read_text(encoding='utf-8')) + diagnosis = json.loads((root / 'traces' / 'diagnosis.json').read_text(encoding='utf-8')) + transitions = json.loads((root / 'transitions.json').read_text(encoding='utf-8')) + mechanisms = json.loads((root / 'mechanisms.json').read_text(encoding='utf-8')) + trace_paths = sorted( + path for path in (root / 'traces').glob('*.json') + if path.name != 'diagnosis.json' + ) + assert len(trace_paths) == 1000 + for path in trace_paths: + validate_trace(json.loads(path.read_text(encoding='utf-8'))) + assert report['metrics']['correct'] == 64 + assert report['metrics']['accepted'] == 64 + assert report['metrics']['incorrect_attempts'] == 0 + assert report['status_counts'] == { + 'ACCEPTED': 64, + 'AMBIGUOUS_PROGRAM': 1, + 'GRAMMAR_EXHAUSTED': 1011, + } + assert not transitions['lost_rows'] + assert sorted(mechanisms['accepted_bbox_tasks']) == [ + '3aa6fb7a', '6d75e8bb', 'e7639916' + ] + assert mechanisms['60b61512']['terminal_status'] == 'AMBIGUOUS_PROGRAM' + assert mechanisms['60b61512']['bbox_equivalence_class_count'] >= 2 + assert diagnosis['trace_count'] == 1000 + PY + - name: Commit sealed evidence + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + root = Path('/tmp/bbox-artifact') + report = json.loads((root / 'report.json').read_text(encoding='utf-8')) + diagnosis = json.loads((root / 'traces' / 'diagnosis.json').read_text(encoding='utf-8')) + transitions = json.loads((root / 'transitions.json').read_text(encoding='utf-8')) + mechanisms = json.loads((root / 'mechanisms.json').read_text(encoding='utf-8')) + artifacts = json.loads(Path('/tmp/artifacts.json').read_text(encoding='utf-8')) + artifact_id = int(Path('/tmp/artifact-id').read_text(encoding='utf-8').strip()) + artifact = next(item for item in artifacts['artifacts'] if item['id'] == artifact_id) + manifest_entries = len((root / 'SHA256SUMS').read_text(encoding='utf-8').splitlines()) + evidence = { + 'schema_version': 1, + 'candidate_commit': os.environ['TARGET_SHA'], + 'run_id': int(Path('/tmp/run-id').read_text(encoding='utf-8').strip()), + 'artifact': { + 'id': artifact['id'], + 'name': artifact['name'], + 'size_in_bytes': artifact['size_in_bytes'], + 'digest': artifact.get('digest'), + 'expired': artifact['expired'], + }, + 'checksum_manifest': { + 'entry_count': manifest_entries, + 'verified': True, + }, + 'report_digest': report['report_digest'], + 'diagnosis_digest': diagnosis['diagnosis_digest'], + 'metrics': report['metrics'], + 'status_counts': report['status_counts'], + 'mechanistic': report['mechanistic'], + 'total_event_count': diagnosis['total_event_count'], + 'accepted_selected_kind_counts': diagnosis['accepted_selected_kind_counts'], + 'hidden_failure_counts': diagnosis['hidden_failure_counts'], + 'transitions': transitions, + 'mechanisms': mechanisms, + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'public_arc_evaluation_rerun': False, + } + path = Path('docs/evidence/arc_v2_2/TRAINING_BBOX_0009.json') + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add docs/evidence/arc_v2_2/TRAINING_BBOX_0009.json + git commit -m 'arc: seal bounding box training evidence' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-bbox-fixtures.yml b/.github/workflows/arc-bbox-fixtures.yml new file mode 100644 index 00000000..edc30f89 --- /dev/null +++ b/.github/workflows/arc-bbox-fixtures.yml @@ -0,0 +1,49 @@ +name: Import ARC Bounding Box Fixtures + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-bbox-fixtures.yml' + +permissions: + contents: write + actions: read + +jobs: + import-fixtures: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - name: Download immutable benchmark corpus + uses: actions/download-artifact@v4 + with: + name: public-benchmarks-pinned-2026-08-04 + path: /tmp/pinned-public-benchmarks + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30929169392' + - name: Copy exact training fixtures + run: | + set -euo pipefail + destination='research/universal_core/holonomy_v2_2/tests/fixtures/bbox' + mkdir -p "$destination" + for task_id in 3aa6fb7a 6d75e8bb e7639916 60b61512; do + mapfile -t matches < <(find /tmp/pinned-public-benchmarks -type f -path "*/arc_agi_2/data/training/${task_id}.json" | sort) + test "${#matches[@]}" -eq 1 + cp "${matches[0]}" "$destination/${task_id}.json" + python -m json.tool "$destination/${task_id}.json" > /dev/null + done + test "$(find "$destination" -type f -name '*.json' | wc -l)" -eq 4 + - name: Commit immutable fixtures + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/tests/fixtures/bbox + git commit -m 'arc: import bounding box training fixtures' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-bbox-red.yml b/.github/workflows/arc-bbox-red.yml new file mode 100644 index 00000000..9f7b6328 --- /dev/null +++ b/.github/workflows/arc-bbox-red.yml @@ -0,0 +1,69 @@ +name: ARC Bounding Box RED + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-bbox-red.yml' + +permissions: + contents: write + +jobs: + verify-red: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Require missing bounding-box module + run: | + set +e + python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_bbox_complete.py > /tmp/bbox-red.log 2>&1 + status=$? + set -e + cat /tmp/bbox-red.log + test "$status" -ne 0 + grep -F "No module named 'research.universal_core.holonomy_v2_2.bbox_complete'" /tmp/bbox-red.log + - name: Commit RED evidence + env: + RED_SOURCE_SHA: ${{ github.sha }} + RED_RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + evidence = { + 'schema_version': 1, + 'red_commit': os.environ['RED_SOURCE_SHA'], + 'red_run_id': os.environ['RED_RUN_ID'], + 'test_command': 'python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_bbox_complete.py', + 'expected_failure': "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.bbox_complete'", + 'failure_confirmed': True, + 'expected_new_exact_tasks': ['3aa6fb7a', '6d75e8bb', 'e7639916'], + 'required_ambiguity': '60b61512', + 'mechanistic_family': { + 'scope': ['global', 'components'], + 'mode': ['fill', 'outline'], + 'preserve_existing_cells': True, + 'component_connectivity': [4, 8], + }, + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + } + path = Path('research/universal_core/holonomy_v2_2/evidence/BBOX_COMPLETE_RED.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/BBOX_COMPLETE_RED.json + git commit -m 'arc: record bounding box RED evidence' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-bbox-run-verifier.yml b/.github/workflows/arc-bbox-run-verifier.yml new file mode 100644 index 00000000..722475c2 --- /dev/null +++ b/.github/workflows/arc-bbox-run-verifier.yml @@ -0,0 +1,154 @@ +name: Verify ARC Bounding Box Corpus + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-bbox-run-verifier.yml' + +permissions: + actions: read + contents: write + +jobs: + verify-target-run: + runs-on: ubuntu-latest + timeout-minutes: 110 + env: + GH_TOKEN: ${{ github.token }} + TARGET_SHA: f92ca46da7698522dfd3cd73be0e0a28e2a158f5 + REPOSITORY: Chrisbryan17/MLE-agent + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - name: Wait for exact bounding-box corpus run + run: | + set -euo pipefail + run_id='' + status='' + conclusion='' + for attempt in $(seq 1 180); do + runs=$(gh api --method GET "repos/$REPOSITORY/actions/runs" \ + -f head_sha="$TARGET_SHA" \ + -f per_page=100) + run_id=$(printf '%s' "$runs" | jq -r ' + [.workflow_runs[] + | select(.name == "ARC Bounding Box Mechanistic Corpus") + | select(.head_sha == env.TARGET_SHA)] + | sort_by(.created_at) + | last + | .id // empty') + if [ -n "$run_id" ]; then + run=$(gh api "repos/$REPOSITORY/actions/runs/$run_id") + status=$(printf '%s' "$run" | jq -r '.status') + conclusion=$(printf '%s' "$run" | jq -r '.conclusion // ""') + echo "run=$run_id status=$status conclusion=$conclusion attempt=$attempt" + if [ "$status" = 'completed' ]; then + break + fi + else + echo "target run not visible yet; attempt=$attempt" + fi + sleep 30 + done + test -n "$run_id" + test "$status" = 'completed' + printf '%s\n' "$run_id" > /tmp/run-id + printf '%s\n' "$conclusion" > /tmp/conclusion + - name: Capture run evidence + if: always() + run: | + set -euo pipefail + run_id=$(cat /tmp/run-id 2>/dev/null || true) + conclusion=$(cat /tmp/conclusion 2>/dev/null || true) + mkdir -p /tmp/run-evidence + if [ -n "$run_id" ]; then + gh api "repos/$REPOSITORY/actions/runs/$run_id" > /tmp/run-evidence/run.json + gh api "repos/$REPOSITORY/actions/runs/$run_id/jobs?per_page=100" > /tmp/run-evidence/jobs.json + gh api "repos/$REPOSITORY/actions/runs/$run_id/artifacts?per_page=100" > /tmp/run-evidence/artifacts.json + if [ "$conclusion" != 'success' ]; then + gh run view "$run_id" --repo "$REPOSITORY" --log-failed > /tmp/run-evidence/failed.log 2>&1 || true + fi + else + printf '%s\n' 'target run was not discovered before verifier timeout' > /tmp/run-evidence/failed.log + fi + - name: Commit immutable run evidence + if: always() + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + root = Path('/tmp/run-evidence') + run = json.loads((root / 'run.json').read_text()) if (root / 'run.json').exists() else {} + jobs = json.loads((root / 'jobs.json').read_text()) if (root / 'jobs.json').exists() else {'jobs': []} + artifacts = json.loads((root / 'artifacts.json').read_text()) if (root / 'artifacts.json').exists() else {'artifacts': []} + target_name = 'arc-v2-2-bbox-' + os.environ['TARGET_SHA'] + artifact = next( + (item for item in artifacts.get('artifacts', []) if item.get('name') == target_name), + None, + ) + failed_log = (root / 'failed.log').read_text(encoding='utf-8', errors='replace') if (root / 'failed.log').exists() else '' + evidence = { + 'schema_version': 1, + 'target_commit': os.environ['TARGET_SHA'], + 'workflow_name': run.get('name'), + 'run_id': run.get('id'), + 'run_number': run.get('run_number'), + 'status': run.get('status'), + 'conclusion': run.get('conclusion'), + 'event': run.get('event'), + 'created_at': run.get('created_at'), + 'updated_at': run.get('updated_at'), + 'artifact': None if artifact is None else { + 'id': artifact.get('id'), + 'name': artifact.get('name'), + 'size_in_bytes': artifact.get('size_in_bytes'), + 'digest': artifact.get('digest'), + 'expired': artifact.get('expired'), + }, + 'jobs': [ + { + 'id': item.get('id'), + 'name': item.get('name'), + 'status': item.get('status'), + 'conclusion': item.get('conclusion'), + 'started_at': item.get('started_at'), + 'completed_at': item.get('completed_at'), + 'steps': [ + { + 'number': step.get('number'), + 'name': step.get('name'), + 'status': step.get('status'), + 'conclusion': step.get('conclusion'), + } + for step in item.get('steps', []) + ], + } + for item in jobs.get('jobs', []) + ], + 'failed_log_tail': failed_log[-20000:], + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'public_arc_evaluation_rerun': False, + } + path = Path('docs/evidence/arc_v2_2/FULL_BBOX_RUN_0009.json') + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add docs/evidence/arc_v2_2/FULL_BBOX_RUN_0009.json + git commit -m 'arc: record bounding box corpus run' + git push origin HEAD:universal-core-arc-training-v2-2 + conclusion=$(python -c "import json; print(json.load(open('docs/evidence/arc_v2_2/FULL_BBOX_RUN_0009.json'))['conclusion'])") + test "$conclusion" = 'success' + python - <<'PY' + import json + evidence = json.load(open('docs/evidence/arc_v2_2/FULL_BBOX_RUN_0009.json')) + if not evidence['artifact']: + raise SystemExit('successful run did not publish the expected artifact') + PY diff --git a/.github/workflows/arc-bbox-wire.yml b/.github/workflows/arc-bbox-wire.yml new file mode 100644 index 00000000..edc51068 --- /dev/null +++ b/.github/workflows/arc-bbox-wire.yml @@ -0,0 +1,175 @@ +name: Wire ARC Bounding Box Completion + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-bbox-wire.yml' + +permissions: + contents: write + +jobs: + wire-and-verify: + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Apply guarded bounding-box integration + run: | + python - <<'PY' + from pathlib import Path + + grid_path = Path('research/universal_core/holonomy_v2_2/grid_induction.py') + text = grid_path.read_text(encoding='utf-8') + import_anchor = 'from .component_outlier import apply_component_outlier\n' + import_value = import_anchor + 'from .bbox_complete import apply_bbox_complete\n' + if text.count(import_anchor) != 1 or 'from .bbox_complete import' in text: + raise SystemExit('bbox import anchor mismatch') + text = text.replace(import_anchor, import_value, 1) + + version_anchor = '_VERSION = "grid-v2.2-7"' + if text.count(version_anchor) != 1: + raise SystemExit('bbox version anchor mismatch') + text = text.replace(version_anchor, '_VERSION = "grid-v2.2-8"', 1) + + apply_anchor = ' if kind == "component_area_outlier_color":\n' + apply_value = ( + ' if kind == "bounding_box_complete":\n' + ' return apply_bbox_complete(program, grid)\n' + ) + if text.count(apply_anchor) != 1: + raise SystemExit('bbox apply anchor mismatch') + text = text.replace(apply_anchor, apply_value + apply_anchor, 1) + grid_path.write_text(text, encoding='utf-8') + + runtime_path = Path('research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py') + text = runtime_path.read_text(encoding='utf-8') + runtime_import = 'from .component_outlier import component_outlier_programs\n' + runtime_import_value = runtime_import + 'from .bbox_complete import bbox_complete_programs\n' + if text.count(runtime_import) != 1 or 'bbox_complete_programs' in text: + raise SystemExit('bbox runtime import anchor mismatch') + text = text.replace(runtime_import, runtime_import_value, 1) + + derived_anchor = ' component_outlier, outlier_reason = component_outlier_programs(demos)\n' + derived_value = derived_anchor + ' bbox_complete, bbox_reason = bbox_complete_programs(demos)\n' + if text.count(derived_anchor) != 1: + raise SystemExit('bbox derivation anchor mismatch') + text = text.replace(derived_anchor, derived_value, 1) + + batch_anchor = ' ("component_outlier", component_outlier, outlier_reason),\n' + batch_value = batch_anchor + ' ("bbox_complete", bbox_complete, bbox_reason),\n' + if text.count(batch_anchor) != 1: + raise SystemExit('bbox batch anchor mismatch') + text = text.replace(batch_anchor, batch_value, 1) + runtime_path.write_text(text, encoding='utf-8') + PY + python -m py_compile \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py \ + research/universal_core/holonomy_v2_2/bbox_complete.py + git diff --check + - name: Run focused bounding-box tests + id: focused + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_bbox_complete.py \ + research/universal_core/holonomy_v2_2/tests/test_component_outlier.py \ + research/universal_core/holonomy_v2_2/tests/test_marker_recolor.py \ + research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py \ + | tee /tmp/focused.log + - name: Run all V2.2 tests + id: cluster + if: always() + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q research/universal_core/holonomy_v2_2/tests | tee /tmp/v2-2.log + - name: Run frozen V2 integration tests + id: frozen + if: always() + continue-on-error: true + env: + PYTHONPATH: research + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py | tee /tmp/frozen.log + - name: Confirm frozen V2 bytes are unchanged + id: byte_gate + if: always() + continue-on-error: true + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Record result and commit only valid state + if: always() + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + FOCUSED_OUTCOME: ${{ steps.focused.outcome }} + CLUSTER_OUTCOME: ${{ steps.cluster.outcome }} + FROZEN_OUTCOME: ${{ steps.frozen.outcome }} + BYTE_OUTCOME: ${{ steps.byte_gate.outcome }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + outcomes = { + 'focused': os.environ['FOCUSED_OUTCOME'], + 'v2_2_cluster': os.environ['CLUSTER_OUTCOME'], + 'frozen_tests': os.environ['FROZEN_OUTCOME'], + 'frozen_byte_gate': os.environ['BYTE_OUTCOME'], + } + passed = all(value == 'success' for value in outcomes.values()) + evidence = { + 'schema_version': 1, + 'workflow_source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': passed, + 'outcomes': outcomes, + 'focused_output': Path('/tmp/focused.log').read_text(encoding='utf-8') if Path('/tmp/focused.log').exists() else '', + 'v2_2_output': Path('/tmp/v2-2.log').read_text(encoding='utf-8') if Path('/tmp/v2-2.log').exists() else '', + 'frozen_output': Path('/tmp/frozen.log').read_text(encoding='utf-8') if Path('/tmp/frozen.log').exists() else '', + 'expected_new_exact_tasks': ['3aa6fb7a', '6d75e8bb', 'e7639916'], + 'required_ambiguity': '60b61512', + 'expected_training_delta': { + 'correct_rows': 3, + 'accepted_rows': 3, + 'grammar_exhausted_rows': -3, + }, + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + } + name = 'BBOX_COMPLETE_GREEN.json' if passed else 'BBOX_COMPLETE_DIAGNOSTIC.json' + path = Path('research/universal_core/holonomy_v2_2/evidence') / name + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + Path('/tmp/passed').write_text('1' if passed else '0', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + if [ "$(cat /tmp/passed)" = "1" ]; then + git add \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py \ + research/universal_core/holonomy_v2_2/evidence/BBOX_COMPLETE_GREEN.json + git commit -m 'arc: wire bounding box completion' + else + git restore \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py + git add research/universal_core/holonomy_v2_2/evidence/BBOX_COMPLETE_DIAGNOSTIC.json + git commit -m 'arc: record bounding box diagnostic failure' + fi + git push origin HEAD:universal-core-arc-training-v2-2 + test "$(cat /tmp/passed)" = "1" diff --git a/.github/workflows/arc-block-fixture-restore.yml b/.github/workflows/arc-block-fixture-restore.yml new file mode 100644 index 00000000..39ba1292 --- /dev/null +++ b/.github/workflows/arc-block-fixture-restore.yml @@ -0,0 +1,78 @@ +name: Restore ARC Block Reduction Fixtures + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-block-fixture-restore.yml' + +permissions: + contents: write + actions: read + +jobs: + restore-and-verify: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Download immutable benchmark corpus + uses: actions/download-artifact@v4 + with: + name: public-benchmarks-pinned-2026-08-04 + path: /tmp/pinned-public-benchmarks + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30929169392' + - name: Restore exact fixture bytes + run: | + set -euo pipefail + vendor=$(find /tmp/pinned-public-benchmarks -type d -path '*/arc_agi_2/data/training' | head -1) + test -n "$vendor" + destination=research/universal_core/holonomy_v2_2/tests/fixtures/block_reduce + mkdir -p "$destination" + cp "$vendor/5614dbcf.json" "$destination/5614dbcf.json" + cp "$vendor/68b67ca3.json" "$destination/68b67ca3.json" + printf '%s %s\n' \ + 0751d867504f022964f0d4ef909ae9e68c928f499841f6a3e8b7143791fca53e "$destination/5614dbcf.json" \ + 767e0baf6a71a7f288ab5a08f13a4ecc65ac5a43144586dec0c3ccd2eff5edb4 "$destination/68b67ca3.json" \ + | sha256sum -c - + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Verify fixture regression + run: python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_block_reduce_fixtures.py | tee /tmp/fixture-green.log + - name: Commit exact fixtures and evidence + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + evidence = { + 'schema_version': 1, + 'workflow_source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'fixture_count': 2, + 'fixture_integrity': True, + 'source_artifact_run': 30929169392, + 'test_output': Path('/tmp/fixture-green.log').read_text(encoding='utf-8'), + 'public_arc_evaluation_rerun': False, + } + path = Path('research/universal_core/holonomy_v2_2/evidence/BLOCK_REDUCE_FIXTURE_GREEN.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add \ + research/universal_core/holonomy_v2_2/tests/fixtures/block_reduce \ + research/universal_core/holonomy_v2_2/evidence/BLOCK_REDUCE_FIXTURE_GREEN.json + git commit -m 'arc: restore pinned block reduction fixtures' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-block-reduce-corpus.yml b/.github/workflows/arc-block-reduce-corpus.yml new file mode 100644 index 00000000..a3c54b1f --- /dev/null +++ b/.github/workflows/arc-block-reduce-corpus.yml @@ -0,0 +1,348 @@ +name: ARC Block Reduce Mechanistic Corpus + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-block-reduce-corpus.yml' + +permissions: + contents: read + actions: read + +jobs: + training-corpus: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Download immutable benchmark corpus + uses: actions/download-artifact@v4 + with: + name: public-benchmarks-pinned-2026-08-04 + path: /tmp/pinned-public-benchmarks + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30929169392' + - name: Download immutable V2.2 baseline report + uses: actions/download-artifact@v4 + with: + name: arc-v2-2-training-e50965009efb96457fd769d97b42027f8a4e98c0 + path: /tmp/arc-v2-2-baseline + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30941837872' + - name: Verify V2.2 and frozen V2 gates + env: + PYTHONPATH: research + run: | + python -m pip install --disable-pip-version-check --quiet pytest + python -m pytest -q research/universal_core/holonomy_v2_2/tests + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py + git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Run all ARC training tasks with complete traces + run: | + mkdir -p /tmp/arc-v2-2-block/traces + python - <<'PY' | tee /tmp/arc-v2-2-block/console-summary.json + import json + import os + from pathlib import Path + + from research.universal_core.holonomy_v2.engine import HolonomyEngine + from research.universal_core.holonomy_v2.types import SearchConfig + from research.universal_core.holonomy_v2_2.mechanistic_trace import validate_trace + from research.universal_core.holonomy_v2_2.training_eval import run_training_corpus, write_report + from research.universal_core.public_benchmarks.suite import locate_vendor + + output = Path('/tmp/arc-v2-2-block') + trace_dir = output / 'traces' + vendor = locate_vendor(Path('/tmp/pinned-public-benchmarks')) + report = run_training_corpus( + vendor, + HolonomyEngine(SearchConfig()), + run_label='arc-v2-2-block-' + os.environ['GITHUB_SHA'][:12], + candidate_commit=os.environ['GITHUB_SHA'], + mechanistic_dir=trace_dir, + verify_parity=True, + ) + write_report(report, output / 'report.json') + + trace_paths = sorted( + path for path in trace_dir.glob('*.json') + if path.name != 'diagnosis.json' + ) + if len(trace_paths) != 1000: + raise SystemExit(f'expected 1000 task traces, found {len(trace_paths)}') + traces = {} + for path in trace_paths: + trace = json.loads(path.read_text(encoding='utf-8')) + validate_trace(trace) + traces[path.stem] = trace + if report['mechanistic']['trace_count'] != 1000: + raise SystemExit('aggregate trace count mismatch') + if report['mechanistic']['parity_count'] != 1000: + raise SystemExit('compact/mechanistic parity count mismatch') + + baseline = json.loads( + Path('/tmp/arc-v2-2-baseline/report.json').read_text(encoding='utf-8') + ) + baseline_rows = { + item['task_id']: int(item['metrics']['correct']) + for item in baseline['reports'] + } + current_rows = { + item['task_id']: int(item['metrics']['correct']) + for item in report['reports'] + } + lost_rows = { + task_id: { + 'baseline_correct': correct, + 'current_correct': current_rows.get(task_id, 0), + } + for task_id, correct in baseline_rows.items() + if current_rows.get(task_id, 0) < correct + } + gained_rows = { + task_id: { + 'baseline_correct': baseline_rows.get(task_id, 0), + 'current_correct': correct, + } + for task_id, correct in current_rows.items() + if correct > baseline_rows.get(task_id, 0) + } + expected_gain_ids = [ + '0c786b71', '2dee498d', '3aa6fb7a', '3af2c5a8', '46442a0e', + '4c4377d9', '5614dbcf', '62c24649', '67e8384a', '68b67ca3', + '6d0aefbc', '6d75e8bb', '6fa7a44f', '746b3537', '7953d61e', + '7b7f7511', '7fe24cdd', '833dafe3', '8be77c9e', 'a59b95c0', + 'aabf363d', 'ac0a08a4', 'b230c067', 'b91ae062', 'c9e6f938', + 'ce8d95cc', 'd4b1c2b1', 'e1baa8a4', 'e7639916', 'eb5a1d5d', + 'ed98d772', + ] + expected_gains = { + task_id: {'baseline_correct': 0, 'current_correct': 1} + for task_id in expected_gain_ids + } + if lost_rows: + raise SystemExit('lost baseline-correct rows: ' + json.dumps(lost_rows, sort_keys=True)) + if gained_rows != expected_gains: + raise SystemExit( + 'unexpected gained rows: expected ' + + json.dumps(expected_gains, sort_keys=True) + + ' found ' + + json.dumps(gained_rows, sort_keys=True) + ) + + expected_metrics = { + 'rows': 1076, + 'correct': 90, + 'accepted': 90, + 'incorrect_attempts': 0, + 'failed': 0, + } + for key, expected in expected_metrics.items(): + if int(report['metrics'][key]) != expected: + raise SystemExit( + f'metric {key} changed: expected {expected}, found {report["metrics"][key]}' + ) + expected_status = { + 'ACCEPTED': 90, + 'AMBIGUOUS_PROGRAM': 1, + 'GRAMMAR_EXHAUSTED': 985, + } + if report['status_counts'] != expected_status: + raise SystemExit( + 'status counts changed: expected ' + + json.dumps(expected_status, sort_keys=True) + + ' found ' + + json.dumps(report['status_counts'], sort_keys=True) + ) + + block_ids = ('5614dbcf', '68b67ca3') + block_details = {} + for task_id in block_ids: + trace = traces[task_id] + final = [event for event in trace['events'] if event.get('event') == 'final_decision'][0] + successes = [ + event for event in trace['events'] + if event.get('event') == 'hidden_execution' + and event.get('candidate_kind') == 'fixed_block_reduce' + and event.get('outcome') == 'SUCCEEDED' + ] + if trace['terminal_status'] != 'ACCEPTED': + raise SystemExit(f'{task_id} did not transition to ACCEPTED') + if not successes: + raise SystemExit(f'{task_id} has no successful fixed-block candidate') + if not str(final.get('selected_candidate_id', '')).startswith('block_reduce:'): + raise SystemExit(f'{task_id} did not select block reduction') + block_details[task_id] = { + 'trace_digest': trace['trace_digest'], + 'successful_block_candidates': len(successes), + 'selected_candidate_id': final['selected_candidate_id'], + } + + primitive_guards = { + 'aabf363d': 'marker_recolor:', + 'b230c067': 'component_outlier:', + '3aa6fb7a': 'bbox_complete:', + '6d75e8bb': 'bbox_complete:', + 'e7639916': 'bbox_complete:', + '4c4377d9': 'mirror_concat:', + '6d0aefbc': 'mirror_concat:', + '6fa7a44f': 'mirror_concat:', + '8be77c9e': 'mirror_concat:', + 'c9e6f938': 'mirror_concat:', + '0c786b71': 'quadrant_mosaic:', + '3af2c5a8': 'quadrant_mosaic:', + '46442a0e': 'quadrant_mosaic:', + '62c24649': 'quadrant_mosaic:', + '67e8384a': 'quadrant_mosaic:', + '7953d61e': 'quadrant_mosaic:', + '7fe24cdd': 'quadrant_mosaic:', + '833dafe3': 'quadrant_mosaic:', + 'ed98d772': 'quadrant_mosaic:', + 'a59b95c0': 'palette_repeat:', + 'ac0a08a4': 'palette_repeat:', + 'b91ae062': 'palette_repeat:', + 'd4b1c2b1': 'palette_repeat:', + '2dee498d': 'axis_collapse:', + '746b3537': 'axis_collapse:', + '7b7f7511': 'axis_collapse:', + 'ce8d95cc': 'axis_collapse:', + 'e1baa8a4': 'axis_collapse:', + 'eb5a1d5d': 'axis_collapse:', + } + prior_mechanisms = {} + for task_id, prefix in primitive_guards.items(): + trace = traces[task_id] + final = [event for event in trace['events'] if event.get('event') == 'final_decision'][0] + if trace['terminal_status'] != 'ACCEPTED': + raise SystemExit(f'{task_id} regressed') + selected = str(final.get('selected_candidate_id', '')) + if not selected.startswith(prefix): + raise SystemExit(f'{task_id} selected primitive changed: {selected}') + prior_mechanisms[task_id] = { + 'selected_candidate_id': selected, + 'trace_digest': trace['trace_digest'], + } + + ambiguous = traces['60b61512'] + classes = [event for event in ambiguous['events'] if event.get('event') == 'equivalence_class'] + bbox_classes = [ + event for event in classes + if any(str(candidate_id).startswith('bbox_complete:') for candidate_id in event['candidate_ids']) + ] + if ambiguous['terminal_status'] != 'AMBIGUOUS_PROGRAM': + raise SystemExit('60b61512 did not remain ambiguity-safe') + if len(bbox_classes) < 2: + raise SystemExit('60b61512 lost bounding-box disagreement') + + transitions = { + 'schema_version': 1, + 'candidate_commit': os.environ['GITHUB_SHA'], + 'baseline_commit': 'e50965009efb96457fd769d97b42027f8a4e98c0', + 'lost_rows': lost_rows, + 'gained_rows': gained_rows, + 'expected_gains_matched': True, + 'incorrect_attempts': int(report['metrics']['incorrect_attempts']), + } + mechanisms = { + 'schema_version': 1, + 'block_tasks': block_details, + 'prior_mechanisms': prior_mechanisms, + '60b61512': { + 'terminal_status': ambiguous['terminal_status'], + 'trace_digest': ambiguous['trace_digest'], + 'bbox_equivalence_class_count': len(bbox_classes), + 'safety_action': 'abstain_on_connectivity_disagreement', + }, + } + parity = { + 'schema_version': 1, + 'candidate_commit': os.environ['GITHUB_SHA'], + 'checked_tasks': 1000, + 'mismatches': 0, + 'lost_baseline_rows': lost_rows, + } + for name, data in ( + ('transitions.json', transitions), + ('mechanisms.json', mechanisms), + ('parity.json', parity), + ): + (output / name).write_text( + json.dumps(data, sort_keys=True, indent=2) + '\n', + encoding='utf-8', + ) + + print(json.dumps({ + 'report_digest': report['report_digest'], + 'diagnosis_digest': report['mechanistic']['diagnosis_digest'], + 'evaluated_tasks': report['evaluated_tasks'], + 'trace_count': report['mechanistic']['trace_count'], + 'parity_count': report['mechanistic']['parity_count'], + 'metrics': report['metrics'], + 'status_counts': report['status_counts'], + 'transitions': transitions, + 'block_tasks': block_details, + }, sort_keys=True)) + PY + - name: Build checksum manifest after all writers close + run: | + python - <<'PY' + import hashlib + from pathlib import Path + + output = Path('/tmp/arc-v2-2-block') + lines = [] + for path in sorted( + item for item in output.rglob('*') + if item.is_file() and item.name != 'SHA256SUMS' + ): + digest = hashlib.sha256(path.read_bytes()).hexdigest() + lines.append(f'{digest} {path.relative_to(output).as_posix()}') + (output / 'SHA256SUMS').write_text('\n'.join(lines) + '\n', encoding='utf-8') + PY + - name: Verify checksum manifest independently + working-directory: /tmp/arc-v2-2-block + run: sha256sum -c SHA256SUMS + - name: Publish fixed-block evidence + uses: actions/upload-artifact@v4 + with: + name: arc-v2-2-block-reduce-${{ github.sha }} + path: /tmp/arc-v2-2-block + if-no-files-found: error + retention-days: 90 + compression-level: 6 + - name: Write run summary + run: | + python - <<'PY' + import json + from pathlib import Path + + root = Path('/tmp/arc-v2-2-block') + report = json.loads((root / 'report.json').read_text()) + mechanisms = json.loads((root / 'mechanisms.json').read_text()) + lines = [ + '# ARC V2.2 fixed-block reduction corpus', + '', + f"- Candidate: `{report['identity']['candidate_commit']}`", + f"- Report digest: `{report['report_digest']}`", + f"- Diagnosis digest: `{report['mechanistic']['diagnosis_digest']}`", + f"- Metrics: `{report['metrics']}`", + f"- Status counts: `{report['status_counts']}`", + f"- Block gains: `{sorted(mechanisms['block_tasks'])}`", + f"- 60b61512 action: `{mechanisms['60b61512']['safety_action']}`", + '- Public ARC evaluation rerun: `no`', + ] + Path('/tmp/block-summary.md').write_text('\n'.join(lines) + '\n', encoding='utf-8') + PY + cat /tmp/block-summary.md >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/arc-block-reduce-module-green.yml b/.github/workflows/arc-block-reduce-module-green.yml new file mode 100644 index 00000000..d77414aa --- /dev/null +++ b/.github/workflows/arc-block-reduce-module-green.yml @@ -0,0 +1,67 @@ +name: ARC Block Reduce Module GREEN + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-block-reduce-module-green.yml' + +permissions: + contents: write + +jobs: + verify-module: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Run isolated block reduction tests + run: | + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_block_reduce_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_block_reduce.py::test_strict_mode_reduces_each_fixed_block \ + research/universal_core/holonomy_v2_2/tests/test_block_reduce.py::test_strict_mode_rejects_tied_block \ + research/universal_core/holonomy_v2_2/tests/test_block_reduce.py::test_unique_non_background_reduces_sparse_blocks \ + research/universal_core/holonomy_v2_2/tests/test_block_reduce.py::test_unique_non_background_rejects_conflicting_block \ + research/universal_core/holonomy_v2_2/tests/test_block_reduce.py::test_program_derivation_exposes_stable_block_factors \ + research/universal_core/holonomy_v2_2/tests/test_block_reduce.py::test_program_derivation_rejects_unstable_block_factors \ + | tee /tmp/block-module-green.log + - name: Confirm frozen bytes + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Commit module evidence + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + evidence = { + 'schema_version': 1, + 'source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': True, + 'test_count': 7, + 'test_output': Path('/tmp/block-module-green.log').read_text(encoding='utf-8'), + 'expected_new_exact_tasks': ['5614dbcf', '68b67ca3'], + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'frozen_byte_gate': True, + 'public_arc_evaluation_rerun': False, + } + path = Path('research/universal_core/holonomy_v2_2/evidence/BLOCK_REDUCE_MODULE_GREEN.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/BLOCK_REDUCE_MODULE_GREEN.json + git commit -m 'arc: record block reduction module GREEN' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-block-reduce-red.yml b/.github/workflows/arc-block-reduce-red.yml new file mode 100644 index 00000000..669c31a4 --- /dev/null +++ b/.github/workflows/arc-block-reduce-red.yml @@ -0,0 +1,80 @@ +name: ARC Block Reduce RED + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-block-reduce-red.yml' + +permissions: + contents: write + +jobs: + verify-red: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Verify pinned block fixtures + run: python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_block_reduce_fixtures.py + - name: Require missing block reduction module + run: | + set +e + python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_block_reduce.py > /tmp/block-red.log 2>&1 + status=$? + set -e + cat /tmp/block-red.log + test "$status" -ne 0 + grep -F "No module named 'research.universal_core.holonomy_v2_2.block_reduce'" /tmp/block-red.log + - name: Verify frozen V2.1 byte identity + run: | + git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Commit RED evidence + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + evidence = { + 'schema_version': 1, + 'source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'failure_confirmed': True, + 'fixture_integrity': True, + 'expected_failure': "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.block_reduce'", + 'expected_new_exact_tasks': ['5614dbcf', '68b67ca3'], + 'expected_cumulative_metrics': { + 'rows': 1076, + 'accepted': 90, + 'correct': 90, + 'incorrect_attempts': 0, + 'failed': 0, + }, + 'expected_status_counts': { + 'ACCEPTED': 90, + 'AMBIGUOUS_PROGRAM': 1, + 'GRAMMAR_EXHAUSTED': 985, + }, + 'captured_log': Path('/tmp/block-red.log').read_text(encoding='utf-8'), + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'public_arc_evaluation_rerun': False, + } + path = Path('research/universal_core/holonomy_v2_2/evidence/BLOCK_REDUCE_RED.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/BLOCK_REDUCE_RED.json + git commit -m 'arc: record block reduction RED evidence' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-block-reduce-wire.yml b/.github/workflows/arc-block-reduce-wire.yml new file mode 100644 index 00000000..ae263b00 --- /dev/null +++ b/.github/workflows/arc-block-reduce-wire.yml @@ -0,0 +1,201 @@ +name: Wire ARC Block Reduce + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-block-reduce-wire.yml' + +permissions: + contents: write + +jobs: + wire-and-verify: + runs-on: ubuntu-latest + timeout-minutes: 50 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Apply guarded block reduction integration + run: | + python - <<'PY' + from pathlib import Path + + grid_path = Path('research/universal_core/holonomy_v2_2/grid_induction.py') + text = grid_path.read_text(encoding='utf-8') + import_anchor = 'from .axis_collapse import apply_axis_collapse\n' + if text.count(import_anchor) != 1 or 'from .block_reduce import' in text: + raise SystemExit('block import anchor mismatch') + text = text.replace( + import_anchor, + import_anchor + 'from .block_reduce import apply_block_reduce\n', + 1, + ) + if text.count('_VERSION = "grid-v2.2-12"') != 1: + raise SystemExit('block version anchor mismatch') + text = text.replace('_VERSION = "grid-v2.2-12"', '_VERSION = "grid-v2.2-13"', 1) + apply_anchor = ' if kind == "axis_duplicate_collapse":\n' + if text.count(apply_anchor) != 1: + raise SystemExit('block apply anchor mismatch') + text = text.replace( + apply_anchor, + ' if kind == "fixed_block_reduce":\n' + ' return apply_block_reduce(program, grid)\n' + + apply_anchor, + 1, + ) + grid_path.write_text(text, encoding='utf-8') + + runtime_path = Path('research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py') + text = runtime_path.read_text(encoding='utf-8') + import_anchor = 'from .axis_collapse import axis_collapse_programs\n' + if text.count(import_anchor) != 1 or 'block_reduce_programs' in text: + raise SystemExit('block runtime import anchor mismatch') + text = text.replace( + import_anchor, + import_anchor + 'from .block_reduce import block_reduce_programs\n', + 1, + ) + derive_anchor = ' axis_collapse, axis_reason = axis_collapse_programs(demos)\n' + if text.count(derive_anchor) != 1: + raise SystemExit('block derivation anchor mismatch') + text = text.replace( + derive_anchor, + derive_anchor + ' block_reduce, block_reason = block_reduce_programs(demos)\n', + 1, + ) + batch_anchor = ' ("axis_collapse", axis_collapse, axis_reason),\n' + if text.count(batch_anchor) != 1: + raise SystemExit('block batch anchor mismatch') + text = text.replace( + batch_anchor, + batch_anchor + ' ("block_reduce", block_reduce, block_reason),\n', + 1, + ) + runtime_path.write_text(text, encoding='utf-8') + PY + python -m py_compile \ + research/universal_core/holonomy_v2_2/block_reduce.py \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py + git diff --check + - name: Run focused block reduction tests + id: focused + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_block_reduce_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_block_reduce.py \ + research/universal_core/holonomy_v2_2/tests/test_axis_collapse_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_axis_collapse.py \ + research/universal_core/holonomy_v2_2/tests/test_palette_repeat_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_palette_repeat.py \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py \ + research/universal_core/holonomy_v2_2/tests/test_mirror_concat.py \ + research/universal_core/holonomy_v2_2/tests/test_bbox_complete.py \ + research/universal_core/holonomy_v2_2/tests/test_component_outlier.py \ + research/universal_core/holonomy_v2_2/tests/test_marker_recolor.py \ + research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py \ + research/universal_core/holonomy_v2_2/tests/test_grid_induction.py::test_cell_scale_and_whole_grid_tile_are_distinct \ + | tee /tmp/block-focused.log + - name: Run all V2.2 tests + id: cluster + if: always() + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q research/universal_core/holonomy_v2_2/tests | tee /tmp/block-v2-2.log + - name: Run frozen V2 integration tests + id: frozen + if: always() + continue-on-error: true + env: + PYTHONPATH: research + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py | tee /tmp/block-frozen.log + - name: Confirm frozen V2 bytes are unchanged + id: byte_gate + if: always() + continue-on-error: true + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Record result and commit only valid state + if: always() + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + FOCUSED_OUTCOME: ${{ steps.focused.outcome }} + CLUSTER_OUTCOME: ${{ steps.cluster.outcome }} + FROZEN_OUTCOME: ${{ steps.frozen.outcome }} + BYTE_OUTCOME: ${{ steps.byte_gate.outcome }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + outcomes = { + 'focused': os.environ['FOCUSED_OUTCOME'], + 'v2_2_cluster': os.environ['CLUSTER_OUTCOME'], + 'frozen_tests': os.environ['FROZEN_OUTCOME'], + 'frozen_byte_gate': os.environ['BYTE_OUTCOME'], + } + passed = all(value == 'success' for value in outcomes.values()) + evidence = { + 'schema_version': 1, + 'workflow_source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': passed, + 'outcomes': outcomes, + 'focused_output': Path('/tmp/block-focused.log').read_text(encoding='utf-8') if Path('/tmp/block-focused.log').exists() else '', + 'v2_2_output': Path('/tmp/block-v2-2.log').read_text(encoding='utf-8') if Path('/tmp/block-v2-2.log').exists() else '', + 'frozen_output': Path('/tmp/block-frozen.log').read_text(encoding='utf-8') if Path('/tmp/block-frozen.log').exists() else '', + 'expected_new_exact_tasks': ['5614dbcf', '68b67ca3'], + 'expected_training_delta': { + 'correct_rows': 2, + 'accepted_rows': 2, + 'grammar_exhausted_rows': -2, + }, + 'expected_cumulative_metrics': { + 'rows': 1076, + 'correct': 90, + 'accepted': 90, + 'incorrect_attempts': 0, + 'failed': 0, + }, + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'public_arc_evaluation_rerun': False, + } + name = 'BLOCK_REDUCE_GREEN.json' if passed else 'BLOCK_REDUCE_DIAGNOSTIC.json' + path = Path('research/universal_core/holonomy_v2_2/evidence') / name + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + Path('/tmp/block-passed').write_text('1' if passed else '0', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + if [ "$(cat /tmp/block-passed)" = "1" ]; then + git add \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py \ + research/universal_core/holonomy_v2_2/evidence/BLOCK_REDUCE_GREEN.json + git commit -m 'arc: wire fixed-block reduction' + else + git restore \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py + git add research/universal_core/holonomy_v2_2/evidence/BLOCK_REDUCE_DIAGNOSTIC.json + git commit -m 'arc: record block reduction diagnostic failure' + fi + git push origin HEAD:universal-core-arc-training-v2-2 + test "$(cat /tmp/block-passed)" = "1" diff --git a/.github/workflows/arc-component-outlier-corpus.yml b/.github/workflows/arc-component-outlier-corpus.yml new file mode 100644 index 00000000..c1283723 --- /dev/null +++ b/.github/workflows/arc-component-outlier-corpus.yml @@ -0,0 +1,328 @@ +name: ARC Component Outlier Mechanistic Corpus + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-component-outlier-corpus.yml' + +permissions: + contents: read + actions: read + +jobs: + training-corpus: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Download immutable benchmark corpus + uses: actions/download-artifact@v4 + with: + name: public-benchmarks-pinned-2026-08-04 + path: /tmp/pinned-public-benchmarks + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30929169392' + - name: Download immutable V2.2 baseline report + uses: actions/download-artifact@v4 + with: + name: arc-v2-2-training-e50965009efb96457fd769d97b42027f8a4e98c0 + path: /tmp/arc-v2-2-baseline + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30941837872' + - name: Verify V2.2 and frozen V2 gates + env: + PYTHONPATH: research + run: | + python -m pip install --disable-pip-version-check --quiet pytest + python -m pytest -q research/universal_core/holonomy_v2_2/tests + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py + git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Run all ARC training tasks with complete traces + run: | + mkdir -p /tmp/arc-v2-2-outlier/traces + python - <<'PY' | tee /tmp/arc-v2-2-outlier/console-summary.json + import json + import os + from pathlib import Path + + from research.universal_core.holonomy_v2.engine import HolonomyEngine + from research.universal_core.holonomy_v2.types import SearchConfig + from research.universal_core.holonomy_v2_2.mechanistic_trace import validate_trace + from research.universal_core.holonomy_v2_2.training_eval import run_training_corpus, write_report + from research.universal_core.public_benchmarks.suite import locate_vendor + + output = Path('/tmp/arc-v2-2-outlier') + trace_dir = output / 'traces' + vendor = locate_vendor(Path('/tmp/pinned-public-benchmarks')) + report = run_training_corpus( + vendor, + HolonomyEngine(SearchConfig()), + run_label='arc-v2-2-outlier-' + os.environ['GITHUB_SHA'][:12], + candidate_commit=os.environ['GITHUB_SHA'], + mechanistic_dir=trace_dir, + verify_parity=True, + ) + write_report(report, output / 'report.json') + + trace_paths = sorted( + path for path in trace_dir.glob('*.json') + if path.name != 'diagnosis.json' + ) + if len(trace_paths) != 1000: + raise SystemExit(f'expected 1000 task traces, found {len(trace_paths)}') + traces = {} + for path in trace_paths: + trace = json.loads(path.read_text(encoding='utf-8')) + validate_trace(trace) + traces[path.stem] = trace + if report['mechanistic']['trace_count'] != 1000: + raise SystemExit('aggregate trace count mismatch') + if report['mechanistic']['parity_count'] != 1000: + raise SystemExit('compact/mechanistic parity count mismatch') + + baseline = json.loads( + Path('/tmp/arc-v2-2-baseline/report.json').read_text(encoding='utf-8') + ) + baseline_rows = { + item['task_id']: int(item['metrics']['correct']) + for item in baseline['reports'] + } + current_rows = { + item['task_id']: int(item['metrics']['correct']) + for item in report['reports'] + } + lost_rows = { + task_id: { + 'baseline_correct': correct, + 'current_correct': current_rows.get(task_id, 0), + } + for task_id, correct in baseline_rows.items() + if current_rows.get(task_id, 0) < correct + } + gained_rows = { + task_id: { + 'baseline_correct': baseline_rows.get(task_id, 0), + 'current_correct': correct, + } + for task_id, correct in current_rows.items() + if correct > baseline_rows.get(task_id, 0) + } + expected_gains = { + 'aabf363d': {'baseline_correct': 0, 'current_correct': 1}, + 'b230c067': {'baseline_correct': 0, 'current_correct': 1}, + } + if lost_rows: + raise SystemExit('lost baseline-correct rows: ' + json.dumps(lost_rows, sort_keys=True)) + if gained_rows != expected_gains: + raise SystemExit( + 'unexpected gained rows: expected ' + + json.dumps(expected_gains, sort_keys=True) + + ' found ' + + json.dumps(gained_rows, sort_keys=True) + ) + + expected_metrics = { + 'rows': 1076, + 'correct': 61, + 'accepted': 61, + 'incorrect_attempts': 0, + } + for key, expected in expected_metrics.items(): + if int(report['metrics'][key]) != expected: + raise SystemExit( + f'metric {key} changed: expected {expected}, found {report["metrics"][key]}' + ) + expected_status = { + 'ACCEPTED': 61, + 'GRAMMAR_EXHAUSTED': 1015, + } + if report['status_counts'] != expected_status: + raise SystemExit( + 'status counts changed: expected ' + + json.dumps(expected_status, sort_keys=True) + + ' found ' + + json.dumps(report['status_counts'], sort_keys=True) + ) + + aab = traces['aabf363d'] + aab_events = aab['events'] + aab_color_failures = [ + event for event in aab_events + if event.get('event') == 'hidden_execution' + and event.get('candidate_kind') == 'color_map' + and event.get('outcome') == 'FAILED' + and event.get('exception_type') == 'KeyError' + and 'unmapped grid cell' in event.get('exception_message', '') + ] + aab_marker_successes = [ + event for event in aab_events + if event.get('event') == 'hidden_execution' + and event.get('candidate_kind') == 'marker_recolor' + and event.get('outcome') == 'SUCCEEDED' + ] + aab_final = [event for event in aab_events if event.get('event') == 'final_decision'][0] + if aab['terminal_status'] != 'ACCEPTED': + raise SystemExit('aabf363d regressed from ACCEPTED') + if len(aab_color_failures) != 1 or len(aab_marker_successes) != 2: + raise SystemExit('aabf363d causal ledger changed') + if not str(aab_final.get('selected_candidate_id', '')).startswith('marker_recolor:'): + raise SystemExit('aabf363d selected primitive changed') + + b230 = traces['b230c067'] + b230_events = b230['events'] + rank_failures = [ + event for event in b230_events + if event.get('event') == 'hidden_execution' + and event.get('candidate_kind') == 'component_rank_color' + and event.get('outcome') == 'FAILED' + and event.get('exception_type') == 'KeyError' + and 'unmapped component rank' in event.get('exception_message', '') + ] + outlier_failures = [ + event for event in b230_events + if event.get('event') == 'hidden_execution' + and event.get('candidate_kind') == 'component_area_outlier_color' + and event.get('outcome') == 'FAILED' + and event.get('exception_type') == 'ValueError' + and 'component peer areas must be uniform' in event.get('exception_message', '') + ] + outlier_successes = [ + event for event in b230_events + if event.get('event') == 'hidden_execution' + and event.get('candidate_kind') == 'component_area_outlier_color' + and event.get('outcome') == 'SUCCEEDED' + ] + b230_final = [event for event in b230_events if event.get('event') == 'final_decision'][0] + if b230['terminal_status'] != 'ACCEPTED': + raise SystemExit('b230c067 did not transition to ACCEPTED') + if len(rank_failures) != 2: + raise SystemExit(f'b230c067 expected two rank failures, found {len(rank_failures)}') + if len(outlier_failures) != 1: + raise SystemExit(f'b230c067 expected one outlier rejection, found {len(outlier_failures)}') + if len(outlier_successes) != 1: + raise SystemExit(f'b230c067 expected one outlier success, found {len(outlier_successes)}') + if not str(b230_final.get('selected_candidate_id', '')).startswith('component_outlier:'): + raise SystemExit('b230c067 did not select component outlier') + + transitions = { + 'schema_version': 1, + 'candidate_commit': os.environ['GITHUB_SHA'], + 'baseline_commit': 'e50965009efb96457fd769d97b42027f8a4e98c0', + 'lost_rows': lost_rows, + 'gained_rows': gained_rows, + 'expected_gains_matched': True, + 'incorrect_attempts': int(report['metrics']['incorrect_attempts']), + } + (output / 'transitions.json').write_text( + json.dumps(transitions, sort_keys=True, indent=2) + '\n', + encoding='utf-8', + ) + parity = { + 'schema_version': 1, + 'candidate_commit': os.environ['GITHUB_SHA'], + 'checked_tasks': 1000, + 'mismatches': 0, + 'lost_baseline_rows': lost_rows, + } + (output / 'parity.json').write_text( + json.dumps(parity, sort_keys=True, indent=2) + '\n', + encoding='utf-8', + ) + mechanisms = { + 'schema_version': 1, + 'aabf363d': { + 'terminal_status': aab['terminal_status'], + 'trace_digest': aab['trace_digest'], + 'preserved_color_map_failure_count': len(aab_color_failures), + 'marker_success_count': len(aab_marker_successes), + 'selected_candidate_id': aab_final['selected_candidate_id'], + }, + 'b230c067': { + 'terminal_status': b230['terminal_status'], + 'trace_digest': b230['trace_digest'], + 'preserved_rank_failure_count': len(rank_failures), + 'four_connected_rejection_count': len(outlier_failures), + 'eight_connected_success_count': len(outlier_successes), + 'invariant': 'one unique area extreme and uniform-area peers', + 'selected_candidate_id': b230_final['selected_candidate_id'], + }, + } + (output / 'mechanisms.json').write_text( + json.dumps(mechanisms, sort_keys=True, indent=2) + '\n', + encoding='utf-8', + ) + + print(json.dumps({ + 'report_digest': report['report_digest'], + 'diagnosis_digest': report['mechanistic']['diagnosis_digest'], + 'evaluated_tasks': report['evaluated_tasks'], + 'trace_count': report['mechanistic']['trace_count'], + 'parity_count': report['mechanistic']['parity_count'], + 'metrics': report['metrics'], + 'status_counts': report['status_counts'], + 'transitions': transitions, + 'mechanisms': mechanisms, + }, sort_keys=True)) + PY + - name: Build checksum manifest after all writers close + run: | + python - <<'PY' + import hashlib + from pathlib import Path + + output = Path('/tmp/arc-v2-2-outlier') + lines = [] + for path in sorted( + item for item in output.rglob('*') + if item.is_file() and item.name != 'SHA256SUMS' + ): + digest = hashlib.sha256(path.read_bytes()).hexdigest() + lines.append(f'{digest} {path.relative_to(output).as_posix()}') + (output / 'SHA256SUMS').write_text('\n'.join(lines) + '\n', encoding='utf-8') + PY + - name: Verify checksum manifest independently + working-directory: /tmp/arc-v2-2-outlier + run: sha256sum -c SHA256SUMS + - name: Publish component-outlier evidence + uses: actions/upload-artifact@v4 + with: + name: arc-v2-2-outlier-${{ github.sha }} + path: /tmp/arc-v2-2-outlier + if-no-files-found: error + retention-days: 90 + compression-level: 6 + - name: Write run summary + run: | + python - <<'PY' + import json + from pathlib import Path + + root = Path('/tmp/arc-v2-2-outlier') + report = json.loads((root / 'report.json').read_text()) + mechanisms = json.loads((root / 'mechanisms.json').read_text()) + lines = [ + '# ARC V2.2 component-outlier corpus', + '', + f"- Candidate: `{report['identity']['candidate_commit']}`", + f"- Report digest: `{report['report_digest']}`", + f"- Diagnosis digest: `{report['mechanistic']['diagnosis_digest']}`", + f"- Metrics: `{report['metrics']}`", + f"- Status counts: `{report['status_counts']}`", + f"- b230c067 invariant: `{mechanisms['b230c067']['invariant']}`", + '- Public ARC evaluation rerun: `no`', + ] + Path('/tmp/outlier-summary.md').write_text('\n'.join(lines) + '\n', encoding='utf-8') + PY + cat /tmp/outlier-summary.md >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/arc-component-outlier-red.yml b/.github/workflows/arc-component-outlier-red.yml new file mode 100644 index 00000000..9c23df3d --- /dev/null +++ b/.github/workflows/arc-component-outlier-red.yml @@ -0,0 +1,72 @@ +name: ARC Component Outlier RED + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-component-outlier-red.yml' + +permissions: + contents: write + +jobs: + verify-red: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Require missing component-outlier module + run: | + set +e + python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_component_outlier.py > /tmp/component-outlier-red.log 2>&1 + status=$? + set -e + cat /tmp/component-outlier-red.log + test "$status" -ne 0 + grep -F "No module named 'research.universal_core.holonomy_v2_2.component_outlier'" /tmp/component-outlier-red.log + - name: Commit RED evidence + env: + RED_SOURCE_SHA: ${{ github.sha }} + RED_RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + evidence = { + 'schema_version': 1, + 'red_commit': os.environ['RED_SOURCE_SHA'], + 'red_run_id': os.environ['RED_RUN_ID'], + 'test_command': 'python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_component_outlier.py', + 'expected_failure': "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.component_outlier'", + 'failure_confirmed': True, + 'captured_log': Path('/tmp/component-outlier-red.log').read_text(encoding='utf-8'), + 'source_trace': { + 'task_id': 'b230c067', + 'trace_digest': 'e881dc495aada3c875dd8016cc6768e37a2560e1588cb0bee0b46986b412f4ac', + 'failure': "two component_rank_color KeyErrors: 'unmapped component rank'", + }, + 'mechanistic_hypothesis': { + 'primitive': 'component_area_outlier_color', + 'invariant': 'one unique area extreme and uniform-area peers', + 'hidden_disambiguation': 'four-connectivity violates peer-area uniformity; eight-connectivity completes', + }, + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + } + path = Path('research/universal_core/holonomy_v2_2/evidence/COMPONENT_OUTLIER_RED.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/COMPONENT_OUTLIER_RED.json + git commit -m 'arc: record component outlier RED evidence' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-component-outlier-run-verifier.yml b/.github/workflows/arc-component-outlier-run-verifier.yml new file mode 100644 index 00000000..d6ba0dfa --- /dev/null +++ b/.github/workflows/arc-component-outlier-run-verifier.yml @@ -0,0 +1,154 @@ +name: Verify ARC Component Outlier Corpus + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-component-outlier-run-verifier.yml' + +permissions: + actions: read + contents: write + +jobs: + verify-target-run: + runs-on: ubuntu-latest + timeout-minutes: 110 + env: + GH_TOKEN: ${{ github.token }} + TARGET_SHA: 0439bd2e8611bc2ad0549249b417c01034ce546e + REPOSITORY: Chrisbryan17/MLE-agent + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - name: Wait for exact component-outlier corpus run + run: | + set -euo pipefail + run_id='' + status='' + conclusion='' + for attempt in $(seq 1 180); do + runs=$(gh api --method GET "repos/$REPOSITORY/actions/runs" \ + -f head_sha="$TARGET_SHA" \ + -f per_page=100) + run_id=$(printf '%s' "$runs" | jq -r ' + [.workflow_runs[] + | select(.name == "ARC Component Outlier Mechanistic Corpus") + | select(.head_sha == env.TARGET_SHA)] + | sort_by(.created_at) + | last + | .id // empty') + if [ -n "$run_id" ]; then + run=$(gh api "repos/$REPOSITORY/actions/runs/$run_id") + status=$(printf '%s' "$run" | jq -r '.status') + conclusion=$(printf '%s' "$run" | jq -r '.conclusion // ""') + echo "run=$run_id status=$status conclusion=$conclusion attempt=$attempt" + if [ "$status" = 'completed' ]; then + break + fi + else + echo "target run not visible yet; attempt=$attempt" + fi + sleep 30 + done + test -n "$run_id" + test "$status" = 'completed' + printf '%s\n' "$run_id" > /tmp/run-id + printf '%s\n' "$conclusion" > /tmp/conclusion + - name: Capture run evidence + if: always() + run: | + set -euo pipefail + run_id=$(cat /tmp/run-id 2>/dev/null || true) + conclusion=$(cat /tmp/conclusion 2>/dev/null || true) + mkdir -p /tmp/run-evidence + if [ -n "$run_id" ]; then + gh api "repos/$REPOSITORY/actions/runs/$run_id" > /tmp/run-evidence/run.json + gh api "repos/$REPOSITORY/actions/runs/$run_id/jobs?per_page=100" > /tmp/run-evidence/jobs.json + gh api "repos/$REPOSITORY/actions/runs/$run_id/artifacts?per_page=100" > /tmp/run-evidence/artifacts.json + if [ "$conclusion" != 'success' ]; then + gh run view "$run_id" --repo "$REPOSITORY" --log-failed > /tmp/run-evidence/failed.log 2>&1 || true + fi + else + printf '%s\n' 'target run was not discovered before verifier timeout' > /tmp/run-evidence/failed.log + fi + - name: Commit immutable run evidence + if: always() + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + root = Path('/tmp/run-evidence') + run = json.loads((root / 'run.json').read_text()) if (root / 'run.json').exists() else {} + jobs = json.loads((root / 'jobs.json').read_text()) if (root / 'jobs.json').exists() else {'jobs': []} + artifacts = json.loads((root / 'artifacts.json').read_text()) if (root / 'artifacts.json').exists() else {'artifacts': []} + target_name = 'arc-v2-2-outlier-' + os.environ['TARGET_SHA'] + artifact = next( + (item for item in artifacts.get('artifacts', []) if item.get('name') == target_name), + None, + ) + failed_log = (root / 'failed.log').read_text(encoding='utf-8', errors='replace') if (root / 'failed.log').exists() else '' + evidence = { + 'schema_version': 1, + 'target_commit': os.environ['TARGET_SHA'], + 'workflow_name': run.get('name'), + 'run_id': run.get('id'), + 'run_number': run.get('run_number'), + 'status': run.get('status'), + 'conclusion': run.get('conclusion'), + 'event': run.get('event'), + 'created_at': run.get('created_at'), + 'updated_at': run.get('updated_at'), + 'artifact': None if artifact is None else { + 'id': artifact.get('id'), + 'name': artifact.get('name'), + 'size_in_bytes': artifact.get('size_in_bytes'), + 'digest': artifact.get('digest'), + 'expired': artifact.get('expired'), + }, + 'jobs': [ + { + 'id': item.get('id'), + 'name': item.get('name'), + 'status': item.get('status'), + 'conclusion': item.get('conclusion'), + 'started_at': item.get('started_at'), + 'completed_at': item.get('completed_at'), + 'steps': [ + { + 'number': step.get('number'), + 'name': step.get('name'), + 'status': step.get('status'), + 'conclusion': step.get('conclusion'), + } + for step in item.get('steps', []) + ], + } + for item in jobs.get('jobs', []) + ], + 'failed_log_tail': failed_log[-20000:], + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'public_arc_evaluation_rerun': False, + } + path = Path('docs/evidence/arc_v2_2/FULL_COMPONENT_OUTLIER_RUN_0008.json') + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add docs/evidence/arc_v2_2/FULL_COMPONENT_OUTLIER_RUN_0008.json + git commit -m 'arc: record component outlier corpus run' + git push origin HEAD:universal-core-arc-training-v2-2 + conclusion=$(python -c "import json; print(json.load(open('docs/evidence/arc_v2_2/FULL_COMPONENT_OUTLIER_RUN_0008.json'))['conclusion'])") + test "$conclusion" = 'success' + python - <<'PY' + import json + evidence = json.load(open('docs/evidence/arc_v2_2/FULL_COMPONENT_OUTLIER_RUN_0008.json')) + if not evidence['artifact']: + raise SystemExit('successful run did not publish the expected artifact') + PY diff --git a/.github/workflows/arc-component-outlier-wire-v2.yml b/.github/workflows/arc-component-outlier-wire-v2.yml new file mode 100644 index 00000000..e93f6344 --- /dev/null +++ b/.github/workflows/arc-component-outlier-wire-v2.yml @@ -0,0 +1,175 @@ +name: Wire ARC Component Outlier V2 + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-component-outlier-wire-v2.yml' + +permissions: + contents: write + +jobs: + wire-and-verify: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Apply guarded component-outlier integration + run: | + python - <<'PY' + from pathlib import Path + + grid_path = Path('research/universal_core/holonomy_v2_2/grid_induction.py') + text = grid_path.read_text(encoding='utf-8') + import_anchor = 'from .marker_recolor import apply_marker_recolor\n' + import_value = import_anchor + 'from .component_outlier import apply_component_outlier\n' + if text.count(import_anchor) != 1 or 'from .component_outlier import' in text: + raise SystemExit('outlier import anchor mismatch') + text = text.replace(import_anchor, import_value, 1) + + version_anchor = '_VERSION = "grid-v2.2-6"' + if text.count(version_anchor) != 1: + raise SystemExit('outlier version anchor mismatch') + text = text.replace(version_anchor, '_VERSION = "grid-v2.2-7"', 1) + + apply_anchor = ' if kind == "marker_recolor":\n' + apply_value = ( + ' if kind == "component_area_outlier_color":\n' + ' return apply_component_outlier(program, grid)\n' + ) + if text.count(apply_anchor) != 1: + raise SystemExit('outlier apply anchor mismatch') + text = text.replace(apply_anchor, apply_value + apply_anchor, 1) + grid_path.write_text(text, encoding='utf-8') + + runtime_path = Path('research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py') + text = runtime_path.read_text(encoding='utf-8') + runtime_import = 'from .marker_recolor import marker_recolor_programs\n' + runtime_import_value = runtime_import + 'from .component_outlier import component_outlier_programs\n' + if text.count(runtime_import) != 1 or 'component_outlier_programs' in text: + raise SystemExit('outlier runtime import anchor mismatch') + text = text.replace(runtime_import, runtime_import_value, 1) + + derived_anchor = ' marker_recolor, marker_reason = marker_recolor_programs(demos)\n' + derived_value = derived_anchor + ' component_outlier, outlier_reason = component_outlier_programs(demos)\n' + if text.count(derived_anchor) != 1: + raise SystemExit('outlier derivation anchor mismatch') + text = text.replace(derived_anchor, derived_value, 1) + + batch_anchor = ' ("marker_recolor", marker_recolor, marker_reason),\n' + batch_value = batch_anchor + ' ("component_outlier", component_outlier, outlier_reason),\n' + if text.count(batch_anchor) != 1: + raise SystemExit('outlier batch anchor mismatch') + text = text.replace(batch_anchor, batch_value, 1) + runtime_path.write_text(text, encoding='utf-8') + PY + python -m py_compile \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py \ + research/universal_core/holonomy_v2_2/component_outlier.py + git diff --check + - name: Run focused component-outlier tests + id: focused + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_component_outlier.py \ + research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py \ + research/universal_core/holonomy_v2_2/tests/test_marker_recolor.py \ + | tee /tmp/focused.log + - name: Run all V2.2 tests + id: cluster + if: always() + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q research/universal_core/holonomy_v2_2/tests | tee /tmp/v2-2.log + - name: Run frozen V2 integration tests + id: frozen + if: always() + continue-on-error: true + env: + PYTHONPATH: research + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py | tee /tmp/frozen.log + - name: Confirm frozen V2 bytes are unchanged + id: byte_gate + if: always() + continue-on-error: true + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Record result and commit only valid state + if: always() + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + FOCUSED_OUTCOME: ${{ steps.focused.outcome }} + CLUSTER_OUTCOME: ${{ steps.cluster.outcome }} + FROZEN_OUTCOME: ${{ steps.frozen.outcome }} + BYTE_OUTCOME: ${{ steps.byte_gate.outcome }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + outcomes = { + 'focused': os.environ['FOCUSED_OUTCOME'], + 'v2_2_cluster': os.environ['CLUSTER_OUTCOME'], + 'frozen_tests': os.environ['FROZEN_OUTCOME'], + 'frozen_byte_gate': os.environ['BYTE_OUTCOME'], + } + passed = all(value == 'success' for value in outcomes.values()) + evidence = { + 'schema_version': 2, + 'workflow_source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': passed, + 'outcomes': outcomes, + 'focused_output': Path('/tmp/focused.log').read_text(encoding='utf-8') if Path('/tmp/focused.log').exists() else '', + 'v2_2_output': Path('/tmp/v2-2.log').read_text(encoding='utf-8') if Path('/tmp/v2-2.log').exists() else '', + 'frozen_output': Path('/tmp/frozen.log').read_text(encoding='utf-8') if Path('/tmp/frozen.log').exists() else '', + 'source_trace_digest': 'e881dc495aada3c875dd8016cc6768e37a2560e1588cb0bee0b46986b412f4ac', + 'expected_training_delta': { + 'task_id': 'b230c067', + 'correct_rows': 1, + 'accepted_rows': 1, + 'execution_failed_rows': -1, + }, + 'mechanistic_invariant': 'one unique area extreme and uniform-area peers', + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + } + name = 'COMPONENT_OUTLIER_GREEN.json' if passed else 'COMPONENT_OUTLIER_DIAGNOSTIC_V2.json' + path = Path('research/universal_core/holonomy_v2_2/evidence') / name + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + Path('/tmp/passed').write_text('1' if passed else '0', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + if [ "$(cat /tmp/passed)" = "1" ]; then + git add \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py \ + research/universal_core/holonomy_v2_2/evidence/COMPONENT_OUTLIER_GREEN.json + git commit -m 'arc: wire component area outlier' + else + git restore \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py + git add research/universal_core/holonomy_v2_2/evidence/COMPONENT_OUTLIER_DIAGNOSTIC_V2.json + git commit -m 'arc: record component outlier diagnostic failure v2' + fi + git push origin HEAD:universal-core-arc-training-v2-2 + test "$(cat /tmp/passed)" = "1" diff --git a/.github/workflows/arc-component-outlier-wire.yml b/.github/workflows/arc-component-outlier-wire.yml new file mode 100644 index 00000000..1eb65a2f --- /dev/null +++ b/.github/workflows/arc-component-outlier-wire.yml @@ -0,0 +1,175 @@ +name: Wire ARC Component Outlier + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-component-outlier-wire.yml' + +permissions: + contents: write + +jobs: + wire-and-verify: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Apply guarded component-outlier integration + run: | + python - <<'PY' + from pathlib import Path + + grid_path = Path('research/universal_core/holonomy_v2_2/grid_induction.py') + text = grid_path.read_text(encoding='utf-8') + import_anchor = 'from .marker_recolor import apply_marker_recolor\n' + import_value = import_anchor + 'from .component_outlier import apply_component_outlier\n' + if text.count(import_anchor) != 1 or 'from .component_outlier import' in text: + raise SystemExit('outlier import anchor mismatch') + text = text.replace(import_anchor, import_value, 1) + + version_anchor = '_VERSION = "grid-v2.2-6"' + if text.count(version_anchor) != 1: + raise SystemExit('outlier version anchor mismatch') + text = text.replace(version_anchor, '_VERSION = "grid-v2.2-7"', 1) + + apply_anchor = ' if kind == "marker_recolor":\n' + apply_value = ( + ' if kind == "component_area_outlier_color":\n' + ' return apply_component_outlier(program, grid)\n' + ) + if text.count(apply_anchor) != 1: + raise SystemExit('outlier apply anchor mismatch') + text = text.replace(apply_anchor, apply_value + apply_anchor, 1) + grid_path.write_text(text, encoding='utf-8') + + runtime_path = Path('research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py') + text = runtime_path.read_text(encoding='utf-8') + runtime_import = 'from .marker_recolor import marker_recolor_programs\n' + runtime_import_value = runtime_import + 'from .component_outlier import component_outlier_programs\n' + if text.count(runtime_import) != 1 or 'component_outlier_programs' in text: + raise SystemExit('outlier runtime import anchor mismatch') + text = text.replace(runtime_import, runtime_import_value, 1) + + derived_anchor = ' marker_recolor, marker_reason = marker_recolor_programs(demos)\n' + derived_value = derived_anchor + ' component_outlier, outlier_reason = component_outlier_programs(demos)\n' + if text.count(derived_anchor) != 1: + raise SystemExit('outlier derivation anchor mismatch') + text = text.replace(derived_anchor, derived_value, 1) + + batch_anchor = ' ("marker_recolor", marker_recolor, marker_reason),\n' + batch_value = batch_anchor + ' ("component_outlier", component_outlier, outlier_reason),\n' + if text.count(batch_anchor) != 1: + raise SystemExit('outlier batch anchor mismatch') + text = text.replace(batch_anchor, batch_value, 1) + runtime_path.write_text(text, encoding='utf-8') + PY + python -m py_compile \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py \ + research/universal_core/holonomy_v2_2/component_outlier.py + git diff --check + - name: Run focused component-outlier tests + id: focused + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_component_outlier.py \ + research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py \ + research/universal_core/holonomy_v2_2/tests/test_marker_recolor.py \ + | tee /tmp/focused.log + - name: Run all V2.2 tests + id: cluster + if: always() + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q research/universal_core/holonomy_v2_2/tests | tee /tmp/v2-2.log + - name: Run frozen V2 integration tests + id: frozen + if: always() + continue-on-error: true + env: + PYTHONPATH: research + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py | tee /tmp/frozen.log + - name: Confirm frozen V2 bytes are unchanged + id: byte_gate + if: always() + continue-on-error: true + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Record result and commit only valid state + if: always() + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + FOCUSED_OUTCOME: ${{ steps.focused.outcome }} + CLUSTER_OUTCOME: ${{ steps.cluster.outcome }} + FROZEN_OUTCOME: ${{ steps.frozen.outcome }} + BYTE_OUTCOME: ${{ steps.byte_gate.outcome }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + outcomes = { + 'focused': os.environ['FOCUSED_OUTCOME'], + 'v2_2_cluster': os.environ['CLUSTER_OUTCOME'], + 'frozen_tests': os.environ['FROZEN_OUTCOME'], + 'frozen_byte_gate': os.environ['BYTE_OUTCOME'], + } + passed = all(value == 'success' for value in outcomes.values()) + evidence = { + 'schema_version': 1, + 'workflow_source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': passed, + 'outcomes': outcomes, + 'focused_output': Path('/tmp/focused.log').read_text(encoding='utf-8') if Path('/tmp/focused.log').exists() else '', + 'v2_2_output': Path('/tmp/v2-2.log').read_text(encoding='utf-8') if Path('/tmp/v2-2.log').exists() else '', + 'frozen_output': Path('/tmp/frozen.log').read_text(encoding='utf-8') if Path('/tmp/frozen.log').exists() else '', + 'source_trace_digest': 'e881dc495aada3c875dd8016cc6768e37a2560e1588cb0bee0b46986b412f4ac', + 'expected_training_delta': { + 'task_id': 'b230c067', + 'correct_rows': 1, + 'accepted_rows': 1, + 'execution_failed_rows': -1, + }, + 'mechanistic_invariant': 'one unique area extreme and uniform-area peers', + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + } + name = 'COMPONENT_OUTLIER_GREEN.json' if passed else 'COMPONENT_OUTLIER_DIAGNOSTIC.json' + path = Path('research/universal_core/holonomy_v2_2/evidence') / name + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + Path('/tmp/passed').write_text('1' if passed else '0', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + if [ "$(cat /tmp/passed)" = "1" ]; then + git add \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py \ + research/universal_core/holonomy_v2_2/evidence/COMPONENT_OUTLIER_GREEN.json + git commit -m 'arc: wire component area outlier' + else + git restore \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py + git add research/universal_core/holonomy_v2_2/evidence/COMPONENT_OUTLIER_DIAGNOSTIC.json + git commit -m 'arc: record component outlier diagnostic failure' + fi + git push origin HEAD:universal-core-arc-training-v2-2 + test "$(cat /tmp/passed)" = "1" diff --git a/.github/workflows/arc-component-select-wire.yml b/.github/workflows/arc-component-select-wire.yml new file mode 100644 index 00000000..1436f6dd --- /dev/null +++ b/.github/workflows/arc-component-select-wire.yml @@ -0,0 +1,65 @@ +name: Wire ARC Component Selection + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-component-select-wire.yml' + +permissions: + contents: write + +jobs: + wire: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - name: Apply guarded integration patch + run: | + python - <<'PY' + from pathlib import Path + + path = Path('research/universal_core/holonomy_v2_2/grid_induction.py') + text = path.read_text(encoding='utf-8') + + import_anchor = 'from .region_fill import apply_region, region_candidates\n' + import_value = import_anchor + 'from .component_select import apply_component_select, component_select_candidates\n' + if text.count(import_anchor) != 1 or 'from .component_select import' in text: + raise SystemExit('component selection import anchor mismatch') + text = text.replace(import_anchor, import_value, 1) + + version_anchor = '_VERSION = "grid-v2.2-4"' + if text.count(version_anchor) != 1: + raise SystemExit('version anchor mismatch') + text = text.replace(version_anchor, '_VERSION = "grid-v2.2-5"', 1) + + apply_anchor = ' if kind == "color_map":\n' + apply_value = ( + ' if kind == "component_area_select":\n' + ' return apply_component_select(program, grid)\n' + ) + if text.count(apply_anchor) != 1: + raise SystemExit('component selection apply anchor mismatch') + text = text.replace(apply_anchor, apply_value + apply_anchor, 1) + + candidate_anchor = ' candidates.extend(region_candidates(demos))\n' + candidate_value = candidate_anchor + ' candidates.extend(component_select_candidates(demos))\n' + if text.count(candidate_anchor) != 1: + raise SystemExit('component selection candidate anchor mismatch') + text = text.replace(candidate_anchor, candidate_value, 1) + + path.write_text(text, encoding='utf-8') + PY + git diff --check + git diff -- research/universal_core/holonomy_v2_2/grid_induction.py + - name: Commit integration + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/grid_induction.py + git commit -m 'arc: wire component area candidates' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-gravity-compact-corpus.yml b/.github/workflows/arc-gravity-compact-corpus.yml new file mode 100644 index 00000000..46bb6522 --- /dev/null +++ b/.github/workflows/arc-gravity-compact-corpus.yml @@ -0,0 +1,350 @@ +name: ARC Gravity Compact Mechanistic Corpus + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-gravity-compact-corpus.yml' + +permissions: + contents: read + actions: read + +jobs: + training-corpus: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Download immutable benchmark corpus + uses: actions/download-artifact@v4 + with: + name: public-benchmarks-pinned-2026-08-04 + path: /tmp/pinned-public-benchmarks + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30929169392' + - name: Download immutable V2.2 baseline report + uses: actions/download-artifact@v4 + with: + name: arc-v2-2-training-e50965009efb96457fd769d97b42027f8a4e98c0 + path: /tmp/arc-v2-2-baseline + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30941837872' + - name: Verify V2.2 and frozen V2 gates + env: + PYTHONPATH: research + run: | + python -m pip install --disable-pip-version-check --quiet pytest + python -m pytest -q research/universal_core/holonomy_v2_2/tests + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py + git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Run all ARC training tasks with complete traces + run: | + mkdir -p /tmp/arc-v2-2-gravity/traces + python - <<'PY' | tee /tmp/arc-v2-2-gravity/console-summary.json + import json + import os + from pathlib import Path + + from research.universal_core.holonomy_v2.engine import HolonomyEngine + from research.universal_core.holonomy_v2.types import SearchConfig + from research.universal_core.holonomy_v2_2.mechanistic_trace import validate_trace + from research.universal_core.holonomy_v2_2.training_eval import run_training_corpus, write_report + from research.universal_core.public_benchmarks.suite import locate_vendor + + output = Path('/tmp/arc-v2-2-gravity') + trace_dir = output / 'traces' + vendor = locate_vendor(Path('/tmp/pinned-public-benchmarks')) + report = run_training_corpus( + vendor, + HolonomyEngine(SearchConfig()), + run_label='arc-v2-2-gravity-' + os.environ['GITHUB_SHA'][:12], + candidate_commit=os.environ['GITHUB_SHA'], + mechanistic_dir=trace_dir, + verify_parity=True, + ) + write_report(report, output / 'report.json') + + trace_paths = sorted( + path for path in trace_dir.glob('*.json') + if path.name != 'diagnosis.json' + ) + if len(trace_paths) != 1000: + raise SystemExit(f'expected 1000 task traces, found {len(trace_paths)}') + traces = {} + for path in trace_paths: + trace = json.loads(path.read_text(encoding='utf-8')) + validate_trace(trace) + traces[path.stem] = trace + if report['mechanistic']['trace_count'] != 1000: + raise SystemExit('aggregate trace count mismatch') + if report['mechanistic']['parity_count'] != 1000: + raise SystemExit('compact/mechanistic parity count mismatch') + + baseline = json.loads( + Path('/tmp/arc-v2-2-baseline/report.json').read_text(encoding='utf-8') + ) + baseline_rows = { + item['task_id']: int(item['metrics']['correct']) + for item in baseline['reports'] + } + current_rows = { + item['task_id']: int(item['metrics']['correct']) + for item in report['reports'] + } + lost_rows = { + task_id: { + 'baseline_correct': correct, + 'current_correct': current_rows.get(task_id, 0), + } + for task_id, correct in baseline_rows.items() + if current_rows.get(task_id, 0) < correct + } + gained_rows = { + task_id: { + 'baseline_correct': baseline_rows.get(task_id, 0), + 'current_correct': correct, + } + for task_id, correct in current_rows.items() + if correct > baseline_rows.get(task_id, 0) + } + expected_gain_ids = [ + '0c786b71', '1e0a9b12', '2dee498d', '3906de3d', '3aa6fb7a', + '3af2c5a8', '46442a0e', '4c4377d9', '5614dbcf', '62c24649', + '67e8384a', '68b67ca3', '6d0aefbc', '6d75e8bb', '6fa7a44f', + '746b3537', '7953d61e', '7b7f7511', '7fe24cdd', '833dafe3', + '8be77c9e', 'a59b95c0', 'aabf363d', 'ac0a08a4', 'b230c067', + 'b91ae062', 'c9e6f938', 'ce8d95cc', 'd4b1c2b1', 'e1baa8a4', + 'e7639916', 'eb5a1d5d', 'ed98d772', + ] + expected_gains = { + task_id: {'baseline_correct': 0, 'current_correct': 1} + for task_id in expected_gain_ids + } + if lost_rows: + raise SystemExit('lost baseline-correct rows: ' + json.dumps(lost_rows, sort_keys=True)) + if gained_rows != expected_gains: + raise SystemExit( + 'unexpected gained rows: expected ' + + json.dumps(expected_gains, sort_keys=True) + + ' found ' + + json.dumps(gained_rows, sort_keys=True) + ) + + expected_metrics = { + 'rows': 1076, + 'correct': 92, + 'accepted': 92, + 'incorrect_attempts': 0, + 'failed': 0, + } + for key, expected in expected_metrics.items(): + if int(report['metrics'][key]) != expected: + raise SystemExit( + f'metric {key} changed: expected {expected}, found {report["metrics"][key]}' + ) + expected_status = { + 'ACCEPTED': 92, + 'AMBIGUOUS_PROGRAM': 1, + 'GRAMMAR_EXHAUSTED': 983, + } + if report['status_counts'] != expected_status: + raise SystemExit( + 'status counts changed: expected ' + + json.dumps(expected_status, sort_keys=True) + + ' found ' + + json.dumps(report['status_counts'], sort_keys=True) + ) + + gravity_ids = ('1e0a9b12', '3906de3d') + gravity_details = {} + for task_id in gravity_ids: + trace = traces[task_id] + final = [event for event in trace['events'] if event.get('event') == 'final_decision'][0] + successes = [ + event for event in trace['events'] + if event.get('event') == 'hidden_execution' + and event.get('candidate_kind') == 'axis_gravity_compact' + and event.get('outcome') == 'SUCCEEDED' + ] + if trace['terminal_status'] != 'ACCEPTED': + raise SystemExit(f'{task_id} did not transition to ACCEPTED') + if not successes: + raise SystemExit(f'{task_id} has no successful gravity candidate') + if not str(final.get('selected_candidate_id', '')).startswith('gravity_compact:'): + raise SystemExit(f'{task_id} did not select gravity compaction') + gravity_details[task_id] = { + 'trace_digest': trace['trace_digest'], + 'successful_gravity_candidates': len(successes), + 'selected_candidate_id': final['selected_candidate_id'], + } + + primitive_guards = { + 'aabf363d': 'marker_recolor:', + 'b230c067': 'component_outlier:', + '3aa6fb7a': 'bbox_complete:', + '6d75e8bb': 'bbox_complete:', + 'e7639916': 'bbox_complete:', + '4c4377d9': 'mirror_concat:', + '6d0aefbc': 'mirror_concat:', + '6fa7a44f': 'mirror_concat:', + '8be77c9e': 'mirror_concat:', + 'c9e6f938': 'mirror_concat:', + '0c786b71': 'quadrant_mosaic:', + '3af2c5a8': 'quadrant_mosaic:', + '46442a0e': 'quadrant_mosaic:', + '62c24649': 'quadrant_mosaic:', + '67e8384a': 'quadrant_mosaic:', + '7953d61e': 'quadrant_mosaic:', + '7fe24cdd': 'quadrant_mosaic:', + '833dafe3': 'quadrant_mosaic:', + 'ed98d772': 'quadrant_mosaic:', + 'a59b95c0': 'palette_repeat:', + 'ac0a08a4': 'palette_repeat:', + 'b91ae062': 'palette_repeat:', + 'd4b1c2b1': 'palette_repeat:', + '2dee498d': 'axis_collapse:', + '746b3537': 'axis_collapse:', + '7b7f7511': 'axis_collapse:', + 'ce8d95cc': 'axis_collapse:', + 'e1baa8a4': 'axis_collapse:', + 'eb5a1d5d': 'axis_collapse:', + '5614dbcf': 'block_reduce:', + '68b67ca3': 'block_reduce:', + } + prior_mechanisms = {} + for task_id, prefix in primitive_guards.items(): + trace = traces[task_id] + final = [event for event in trace['events'] if event.get('event') == 'final_decision'][0] + if trace['terminal_status'] != 'ACCEPTED': + raise SystemExit(f'{task_id} regressed') + selected = str(final.get('selected_candidate_id', '')) + if not selected.startswith(prefix): + raise SystemExit(f'{task_id} selected primitive changed: {selected}') + prior_mechanisms[task_id] = { + 'selected_candidate_id': selected, + 'trace_digest': trace['trace_digest'], + } + + ambiguous = traces['60b61512'] + classes = [event for event in ambiguous['events'] if event.get('event') == 'equivalence_class'] + bbox_classes = [ + event for event in classes + if any(str(candidate_id).startswith('bbox_complete:') for candidate_id in event['candidate_ids']) + ] + if ambiguous['terminal_status'] != 'AMBIGUOUS_PROGRAM': + raise SystemExit('60b61512 did not remain ambiguity-safe') + if len(bbox_classes) < 2: + raise SystemExit('60b61512 lost bounding-box disagreement') + + transitions = { + 'schema_version': 1, + 'candidate_commit': os.environ['GITHUB_SHA'], + 'baseline_commit': 'e50965009efb96457fd769d97b42027f8a4e98c0', + 'lost_rows': lost_rows, + 'gained_rows': gained_rows, + 'expected_gains_matched': True, + 'incorrect_attempts': int(report['metrics']['incorrect_attempts']), + } + mechanisms = { + 'schema_version': 1, + 'gravity_tasks': gravity_details, + 'prior_mechanisms': prior_mechanisms, + '60b61512': { + 'terminal_status': ambiguous['terminal_status'], + 'trace_digest': ambiguous['trace_digest'], + 'bbox_equivalence_class_count': len(bbox_classes), + 'safety_action': 'abstain_on_connectivity_disagreement', + }, + } + parity = { + 'schema_version': 1, + 'candidate_commit': os.environ['GITHUB_SHA'], + 'checked_tasks': 1000, + 'mismatches': 0, + 'lost_baseline_rows': lost_rows, + } + for name, data in ( + ('transitions.json', transitions), + ('mechanisms.json', mechanisms), + ('parity.json', parity), + ): + (output / name).write_text( + json.dumps(data, sort_keys=True, indent=2) + '\n', + encoding='utf-8', + ) + + print(json.dumps({ + 'report_digest': report['report_digest'], + 'diagnosis_digest': report['mechanistic']['diagnosis_digest'], + 'evaluated_tasks': report['evaluated_tasks'], + 'trace_count': report['mechanistic']['trace_count'], + 'parity_count': report['mechanistic']['parity_count'], + 'metrics': report['metrics'], + 'status_counts': report['status_counts'], + 'transitions': transitions, + 'gravity_tasks': gravity_details, + }, sort_keys=True)) + PY + - name: Build checksum manifest after all writers close + run: | + python - <<'PY' + import hashlib + from pathlib import Path + + output = Path('/tmp/arc-v2-2-gravity') + lines = [] + for path in sorted( + item for item in output.rglob('*') + if item.is_file() and item.name != 'SHA256SUMS' + ): + digest = hashlib.sha256(path.read_bytes()).hexdigest() + lines.append(f'{digest} {path.relative_to(output).as_posix()}') + (output / 'SHA256SUMS').write_text('\n'.join(lines) + '\n', encoding='utf-8') + PY + - name: Verify checksum manifest independently + working-directory: /tmp/arc-v2-2-gravity + run: sha256sum -c SHA256SUMS + - name: Publish gravity evidence + uses: actions/upload-artifact@v4 + with: + name: arc-v2-2-gravity-compact-${{ github.sha }} + path: /tmp/arc-v2-2-gravity + if-no-files-found: error + retention-days: 90 + compression-level: 6 + - name: Write run summary + run: | + python - <<'PY' + import json + from pathlib import Path + + root = Path('/tmp/arc-v2-2-gravity') + report = json.loads((root / 'report.json').read_text()) + mechanisms = json.loads((root / 'mechanisms.json').read_text()) + lines = [ + '# ARC V2.2 axis gravity compaction corpus', + '', + f"- Candidate: `{report['identity']['candidate_commit']}`", + f"- Report digest: `{report['report_digest']}`", + f"- Diagnosis digest: `{report['mechanistic']['diagnosis_digest']}`", + f"- Metrics: `{report['metrics']}`", + f"- Status counts: `{report['status_counts']}`", + f"- Gravity gains: `{sorted(mechanisms['gravity_tasks'])}`", + f"- 60b61512 action: `{mechanisms['60b61512']['safety_action']}`", + '- Public ARC evaluation rerun: `no`', + ] + Path('/tmp/gravity-summary.md').write_text('\n'.join(lines) + '\n', encoding='utf-8') + PY + cat /tmp/gravity-summary.md >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/arc-gravity-compact-module-green.yml b/.github/workflows/arc-gravity-compact-module-green.yml new file mode 100644 index 00000000..a0bf3185 --- /dev/null +++ b/.github/workflows/arc-gravity-compact-module-green.yml @@ -0,0 +1,68 @@ +name: ARC Gravity Compact Module GREEN + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-gravity-compact-module-green.yml' + +permissions: + contents: write + +jobs: + verify-module: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Run isolated gravity tests + run: | + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_gravity_compact_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_gravity_compact.py::test_vertical_gravity_compacts_up_and_down \ + research/universal_core/holonomy_v2_2/tests/test_gravity_compact.py::test_horizontal_gravity_compacts_left_and_right \ + research/universal_core/holonomy_v2_2/tests/test_gravity_compact.py::test_gravity_preserves_shape_counts_and_lane_order \ + research/universal_core/holonomy_v2_2/tests/test_gravity_compact.py::test_gravity_rejects_unknown_direction \ + research/universal_core/holonomy_v2_2/tests/test_gravity_compact.py::test_program_derivation_returns_unique_demonstrated_candidate \ + research/universal_core/holonomy_v2_2/tests/test_gravity_compact.py::test_program_derivation_rejects_shape_change \ + research/universal_core/holonomy_v2_2/tests/test_gravity_compact.py::test_program_derivation_skips_noop_demonstration \ + | tee /tmp/gravity-module-green.log + - name: Confirm frozen bytes + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Commit module evidence + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + evidence = { + 'schema_version': 1, + 'source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': True, + 'test_count': 8, + 'test_output': Path('/tmp/gravity-module-green.log').read_text(encoding='utf-8'), + 'expected_new_exact_tasks': ['1e0a9b12', '3906de3d'], + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'frozen_byte_gate': True, + 'public_arc_evaluation_rerun': False, + } + path = Path('research/universal_core/holonomy_v2_2/evidence/GRAVITY_COMPACT_MODULE_GREEN.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/GRAVITY_COMPACT_MODULE_GREEN.json + git commit -m 'arc: record gravity compaction module GREEN' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-gravity-compact-red.yml b/.github/workflows/arc-gravity-compact-red.yml new file mode 100644 index 00000000..7710dbd8 --- /dev/null +++ b/.github/workflows/arc-gravity-compact-red.yml @@ -0,0 +1,86 @@ +name: ARC Gravity Compact RED + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-gravity-compact-red.yml' + +permissions: + contents: write + +jobs: + verify-red: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Verify pinned gravity fixtures + run: python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_gravity_compact_fixtures.py + - name: Require missing gravity module + run: | + set +e + python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_gravity_compact.py > /tmp/gravity-red.log 2>&1 + status=$? + set -e + cat /tmp/gravity-red.log + test "$status" -ne 0 + grep -F "No module named 'research.universal_core.holonomy_v2_2.gravity_compact'" /tmp/gravity-red.log + - name: Verify frozen V2.1 byte identity + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Commit RED evidence + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + evidence = { + 'schema_version': 1, + 'source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'failure_confirmed': True, + 'fixture_integrity': True, + 'expected_failure': "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.gravity_compact'", + 'expected_new_exact_tasks': ['1e0a9b12', '3906de3d'], + 'training_scan': { + 'compatible_tasks': 2, + 'fitting_programs': 2, + 'wrong_completed_hidden_outputs': 0, + 'ambiguous_hidden_outputs': 0, + 'compatible_accepted_tasks': 0, + }, + 'expected_cumulative_metrics': { + 'rows': 1076, + 'accepted': 92, + 'correct': 92, + 'incorrect_attempts': 0, + 'failed': 0, + }, + 'expected_status_counts': { + 'ACCEPTED': 92, + 'AMBIGUOUS_PROGRAM': 1, + 'GRAMMAR_EXHAUSTED': 983, + }, + 'captured_log': Path('/tmp/gravity-red.log').read_text(encoding='utf-8'), + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'public_arc_evaluation_rerun': False, + } + path = Path('research/universal_core/holonomy_v2_2/evidence/GRAVITY_COMPACT_RED.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/GRAVITY_COMPACT_RED.json + git commit -m 'arc: record gravity compaction RED evidence' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-gravity-compact-wire.yml b/.github/workflows/arc-gravity-compact-wire.yml new file mode 100644 index 00000000..1ecf60fe --- /dev/null +++ b/.github/workflows/arc-gravity-compact-wire.yml @@ -0,0 +1,203 @@ +name: Wire ARC Gravity Compact + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-gravity-compact-wire.yml' + +permissions: + contents: write + +jobs: + wire-and-verify: + runs-on: ubuntu-latest + timeout-minutes: 50 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Apply guarded gravity integration + run: | + python - <<'PY' + from pathlib import Path + + grid_path = Path('research/universal_core/holonomy_v2_2/grid_induction.py') + text = grid_path.read_text(encoding='utf-8') + import_anchor = 'from .block_reduce import apply_block_reduce\n' + if text.count(import_anchor) != 1 or 'from .gravity_compact import' in text: + raise SystemExit('gravity import anchor mismatch') + text = text.replace( + import_anchor, + import_anchor + 'from .gravity_compact import apply_gravity_compact\n', + 1, + ) + if text.count('_VERSION = "grid-v2.2-13"') != 1: + raise SystemExit('gravity version anchor mismatch') + text = text.replace('_VERSION = "grid-v2.2-13"', '_VERSION = "grid-v2.2-14"', 1) + apply_anchor = ' if kind == "fixed_block_reduce":\n' + if text.count(apply_anchor) != 1: + raise SystemExit('gravity apply anchor mismatch') + text = text.replace( + apply_anchor, + ' if kind == "axis_gravity_compact":\n' + ' return apply_gravity_compact(program, grid)\n' + + apply_anchor, + 1, + ) + grid_path.write_text(text, encoding='utf-8') + + runtime_path = Path('research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py') + text = runtime_path.read_text(encoding='utf-8') + import_anchor = 'from .block_reduce import block_reduce_programs\n' + if text.count(import_anchor) != 1 or 'gravity_compact_programs' in text: + raise SystemExit('gravity runtime import anchor mismatch') + text = text.replace( + import_anchor, + import_anchor + 'from .gravity_compact import gravity_compact_programs\n', + 1, + ) + derive_anchor = ' block_reduce, block_reason = block_reduce_programs(demos)\n' + if text.count(derive_anchor) != 1: + raise SystemExit('gravity derivation anchor mismatch') + text = text.replace( + derive_anchor, + derive_anchor + ' gravity_compact, gravity_reason = gravity_compact_programs(demos)\n', + 1, + ) + batch_anchor = ' ("block_reduce", block_reduce, block_reason),\n' + if text.count(batch_anchor) != 1: + raise SystemExit('gravity batch anchor mismatch') + text = text.replace( + batch_anchor, + batch_anchor + ' ("gravity_compact", gravity_compact, gravity_reason),\n', + 1, + ) + runtime_path.write_text(text, encoding='utf-8') + PY + python -m py_compile \ + research/universal_core/holonomy_v2_2/gravity_compact.py \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py + git diff --check + - name: Run focused gravity tests + id: focused + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_gravity_compact_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_gravity_compact.py \ + research/universal_core/holonomy_v2_2/tests/test_block_reduce_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_block_reduce.py \ + research/universal_core/holonomy_v2_2/tests/test_axis_collapse_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_axis_collapse.py \ + research/universal_core/holonomy_v2_2/tests/test_palette_repeat_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_palette_repeat.py \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py \ + research/universal_core/holonomy_v2_2/tests/test_mirror_concat.py \ + research/universal_core/holonomy_v2_2/tests/test_bbox_complete.py \ + research/universal_core/holonomy_v2_2/tests/test_component_outlier.py \ + research/universal_core/holonomy_v2_2/tests/test_marker_recolor.py \ + research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py \ + research/universal_core/holonomy_v2_2/tests/test_grid_induction.py::test_cell_scale_and_whole_grid_tile_are_distinct \ + | tee /tmp/gravity-focused.log + - name: Run all V2.2 tests + id: cluster + if: always() + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q research/universal_core/holonomy_v2_2/tests | tee /tmp/gravity-v2-2.log + - name: Run frozen V2 integration tests + id: frozen + if: always() + continue-on-error: true + env: + PYTHONPATH: research + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py | tee /tmp/gravity-frozen.log + - name: Confirm frozen V2 bytes are unchanged + id: byte_gate + if: always() + continue-on-error: true + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Record result and commit only valid state + if: always() + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + FOCUSED_OUTCOME: ${{ steps.focused.outcome }} + CLUSTER_OUTCOME: ${{ steps.cluster.outcome }} + FROZEN_OUTCOME: ${{ steps.frozen.outcome }} + BYTE_OUTCOME: ${{ steps.byte_gate.outcome }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + outcomes = { + 'focused': os.environ['FOCUSED_OUTCOME'], + 'v2_2_cluster': os.environ['CLUSTER_OUTCOME'], + 'frozen_tests': os.environ['FROZEN_OUTCOME'], + 'frozen_byte_gate': os.environ['BYTE_OUTCOME'], + } + passed = all(value == 'success' for value in outcomes.values()) + evidence = { + 'schema_version': 1, + 'workflow_source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': passed, + 'outcomes': outcomes, + 'focused_output': Path('/tmp/gravity-focused.log').read_text(encoding='utf-8') if Path('/tmp/gravity-focused.log').exists() else '', + 'v2_2_output': Path('/tmp/gravity-v2-2.log').read_text(encoding='utf-8') if Path('/tmp/gravity-v2-2.log').exists() else '', + 'frozen_output': Path('/tmp/gravity-frozen.log').read_text(encoding='utf-8') if Path('/tmp/gravity-frozen.log').exists() else '', + 'expected_new_exact_tasks': ['1e0a9b12', '3906de3d'], + 'expected_training_delta': { + 'correct_rows': 2, + 'accepted_rows': 2, + 'grammar_exhausted_rows': -2, + }, + 'expected_cumulative_metrics': { + 'rows': 1076, + 'correct': 92, + 'accepted': 92, + 'incorrect_attempts': 0, + 'failed': 0, + }, + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'public_arc_evaluation_rerun': False, + } + name = 'GRAVITY_COMPACT_GREEN.json' if passed else 'GRAVITY_COMPACT_DIAGNOSTIC.json' + path = Path('research/universal_core/holonomy_v2_2/evidence') / name + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + Path('/tmp/gravity-passed').write_text('1' if passed else '0', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + if [ "$(cat /tmp/gravity-passed)" = "1" ]; then + git add \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py \ + research/universal_core/holonomy_v2_2/evidence/GRAVITY_COMPACT_GREEN.json + git commit -m 'arc: wire axis gravity compaction' + else + git restore \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py + git add research/universal_core/holonomy_v2_2/evidence/GRAVITY_COMPACT_DIAGNOSTIC.json + git commit -m 'arc: record gravity compaction diagnostic failure' + fi + git push origin HEAD:universal-core-arc-training-v2-2 + test "$(cat /tmp/gravity-passed)" = "1" diff --git a/.github/workflows/arc-grid-mechanistic-red.yml b/.github/workflows/arc-grid-mechanistic-red.yml new file mode 100644 index 00000000..6c075955 --- /dev/null +++ b/.github/workflows/arc-grid-mechanistic-red.yml @@ -0,0 +1,62 @@ +name: ARC Grid Mechanistic RED + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-grid-mechanistic-red.yml' + +permissions: + contents: write + +jobs: + verify-red: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Require missing mechanistic entry-point failure + run: | + set +e + python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py > /tmp/grid-mechanistic-red.log 2>&1 + status=$? + set -e + cat /tmp/grid-mechanistic-red.log + test "$status" -ne 0 + grep -F "unexpected keyword argument 'mechanistic'" /tmp/grid-mechanistic-red.log + - name: Commit RED evidence + env: + RED_SOURCE_SHA: ${{ github.sha }} + RED_RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + evidence = { + 'schema_version': 1, + 'red_commit': os.environ['RED_SOURCE_SHA'], + 'red_run_id': os.environ['RED_RUN_ID'], + 'test_command': 'python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py', + 'expected_failure': "TypeError: run_public_task_v2_2() got an unexpected keyword argument 'mechanistic'", + 'failure_confirmed': True, + 'captured_log': Path('/tmp/grid-mechanistic-red.log').read_text(encoding='utf-8'), + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + } + path = Path('research/universal_core/holonomy_v2_2/evidence/GRID_MECHANISTIC_RED.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/GRID_MECHANISTIC_RED.json + git commit -m 'arc: record complete grid trace RED evidence' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-grid-mechanistic-wire-v2.yml b/.github/workflows/arc-grid-mechanistic-wire-v2.yml new file mode 100644 index 00000000..2822a6d1 --- /dev/null +++ b/.github/workflows/arc-grid-mechanistic-wire-v2.yml @@ -0,0 +1,213 @@ +name: Wire ARC Grid Mechanistic Runtime V2 + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-grid-mechanistic-wire-v2.yml' + +permissions: + contents: write + +jobs: + wire-and-verify: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Apply literal runtime dispatch and completeness patch + run: | + python - <<'PY' + from pathlib import Path + + grid_path = Path('research/universal_core/holonomy_v2_2/grid_induction.py') + grid_text = grid_path.read_text(encoding='utf-8') + marker = '\ndef run_public_task_v2_2(task: Any, engine: Any) -> dict[str, Any]:\n' + if grid_text.count(marker) != 1: + raise SystemExit('grid public entry marker mismatch') + prefix, _ = grid_text.split(marker, 1) + replacement = ( + '\n\n' + 'def run_public_task_v2_2(\n' + ' task: Any,\n' + ' engine: Any,\n' + ' *,\n' + ' mechanistic: bool = False,\n' + ') -> dict[str, Any]:\n' + ' if not _is_grid_task(task):\n' + ' result = run_public_task_v2(task, engine)\n' + ' if not mechanistic:\n' + ' return result\n' + ' from .grid_mechanistic_runtime import attach_delegate_trace\n' + '\n' + ' return attach_delegate_trace(task, result)\n' + '\n' + ' from .grid_mechanistic_runtime import run_grid_task\n' + '\n' + ' return run_grid_task(task, mechanistic=mechanistic)\n' + ) + grid_path.write_text(prefix.rstrip() + replacement, encoding='utf-8') + + trace_path = Path('research/universal_core/holonomy_v2_2/mechanistic_trace.py') + trace_text = trace_path.read_text(encoding='utf-8') + old = ( + ' if run_starts:\n' + ' hidden_count = run_starts[0].get("hidden_count")\n' + ' if type(hidden_count) is not int or hidden_count < 0:\n' + ' raise ValueError("run-start hidden count must be non-negative")\n' + ' hidden_events = _events_named(events, "hidden_execution")\n' + ' by_candidate: dict[str, list[Mapping[str, Any]]] = {}\n' + ' for item in hidden_events:\n' + ' candidate_id = item.get("candidate_id")\n' + ' if candidate_id not in representatives:\n' + ' raise ValueError("hidden execution references non-representative candidate")\n' + ' by_candidate.setdefault(str(candidate_id), []).append(item)\n' + ' expected_indices = list(range(hidden_count))\n' + ' for candidate_id in sorted(representatives):\n' + ' indices = sorted(int(item.get("hidden_index", -1)) for item in by_candidate.get(candidate_id, []))\n' + ' if indices != expected_indices:\n' + ' raise ValueError("hidden executions do not cover every hidden input")\n' + ) + new = ( + ' if run_starts:\n' + ' demo_count = run_starts[0].get("demo_count")\n' + ' hidden_count = run_starts[0].get("hidden_count")\n' + ' if type(demo_count) is not int or demo_count < 0:\n' + ' raise ValueError("run-start demonstration count must be non-negative")\n' + ' if type(hidden_count) is not int or hidden_count < 0:\n' + ' raise ValueError("run-start hidden count must be non-negative")\n' + '\n' + ' demo_events = _events_named(events, "demo_execution")\n' + ' demo_by_candidate: dict[str, list[Mapping[str, Any]]] = {}\n' + ' for item in demo_events:\n' + ' candidate_id = item.get("candidate_id")\n' + ' if candidate_id not in proposed:\n' + ' raise ValueError("demonstration execution references unknown candidate")\n' + ' demo_by_candidate.setdefault(str(candidate_id), []).append(item)\n' + ' expected_demo_indices = list(range(demo_count))\n' + ' for candidate_id in proposed_ids:\n' + ' indices = sorted(\n' + ' int(item.get("demo_index", -1))\n' + ' for item in demo_by_candidate.get(candidate_id, [])\n' + ' )\n' + ' if indices != expected_demo_indices:\n' + ' raise ValueError("demonstration executions do not cover every candidate and example")\n' + '\n' + ' hidden_events = _events_named(events, "hidden_execution")\n' + ' by_candidate: dict[str, list[Mapping[str, Any]]] = {}\n' + ' for item in hidden_events:\n' + ' candidate_id = item.get("candidate_id")\n' + ' if candidate_id not in representatives:\n' + ' raise ValueError("hidden execution references non-representative candidate")\n' + ' by_candidate.setdefault(str(candidate_id), []).append(item)\n' + ' expected_indices = list(range(hidden_count))\n' + ' for candidate_id in sorted(representatives):\n' + ' indices = sorted(\n' + ' int(item.get("hidden_index", -1))\n' + ' for item in by_candidate.get(candidate_id, [])\n' + ' )\n' + ' if indices != expected_indices:\n' + ' raise ValueError("hidden executions do not cover every hidden input")\n' + ) + if trace_text.count(old) != 1: + raise SystemExit('trace completeness anchor mismatch') + trace_path.write_text(trace_text.replace(old, new, 1), encoding='utf-8') + PY + python -m py_compile \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/mechanistic_trace.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py + git diff --check + - name: Run focused mechanistic tests + id: focused + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py \ + research/universal_core/holonomy_v2_2/tests/test_mechanistic_trace.py \ + | tee /tmp/focused.log + - name: Run all V2.2 tests + id: cluster + if: always() + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q research/universal_core/holonomy_v2_2/tests | tee /tmp/v2-2.log + - name: Run frozen V2 integration tests + id: frozen + if: always() + continue-on-error: true + env: + PYTHONPATH: research + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py | tee /tmp/frozen.log + - name: Confirm frozen V2 bytes are unchanged + id: byte_gate + if: always() + continue-on-error: true + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Record result and commit only valid state + if: always() + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + FOCUSED_OUTCOME: ${{ steps.focused.outcome }} + CLUSTER_OUTCOME: ${{ steps.cluster.outcome }} + FROZEN_OUTCOME: ${{ steps.frozen.outcome }} + BYTE_OUTCOME: ${{ steps.byte_gate.outcome }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + outcomes = { + 'focused': os.environ['FOCUSED_OUTCOME'], + 'v2_2_cluster': os.environ['CLUSTER_OUTCOME'], + 'frozen_tests': os.environ['FROZEN_OUTCOME'], + 'frozen_byte_gate': os.environ['BYTE_OUTCOME'], + } + passed = all(value == 'success' for value in outcomes.values()) + evidence = { + 'schema_version': 2, + 'workflow_source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': passed, + 'outcomes': outcomes, + 'focused_output': Path('/tmp/focused.log').read_text(encoding='utf-8') if Path('/tmp/focused.log').exists() else '', + 'v2_2_output': Path('/tmp/v2-2.log').read_text(encoding='utf-8') if Path('/tmp/v2-2.log').exists() else '', + 'frozen_output': Path('/tmp/frozen.log').read_text(encoding='utf-8') if Path('/tmp/frozen.log').exists() else '', + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + } + name = 'GRID_MECHANISTIC_GREEN.json' if passed else 'GRID_MECHANISTIC_DIAGNOSTIC_V2.json' + path = Path('research/universal_core/holonomy_v2_2/evidence') / name + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + Path('/tmp/passed').write_text('1' if passed else '0', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + if [ "$(cat /tmp/passed)" = "1" ]; then + git add \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/mechanistic_trace.py \ + research/universal_core/holonomy_v2_2/evidence/GRID_MECHANISTIC_GREEN.json + git commit -m 'arc: trace complete grid inference path' + else + git restore research/universal_core/holonomy_v2_2/grid_induction.py research/universal_core/holonomy_v2_2/mechanistic_trace.py + git add research/universal_core/holonomy_v2_2/evidence/GRID_MECHANISTIC_DIAGNOSTIC_V2.json + git commit -m 'arc: record grid trace diagnostic failure v2' + fi + git push origin HEAD:universal-core-arc-training-v2-2 + test "$(cat /tmp/passed)" = "1" diff --git a/.github/workflows/arc-grid-mechanistic-wire.yml b/.github/workflows/arc-grid-mechanistic-wire.yml new file mode 100644 index 00000000..0e16c29a --- /dev/null +++ b/.github/workflows/arc-grid-mechanistic-wire.yml @@ -0,0 +1,218 @@ +name: Wire ARC Grid Mechanistic Runtime + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-grid-mechanistic-wire.yml' + +permissions: + contents: write + +jobs: + wire-and-verify: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Apply guarded runtime dispatch and completeness patch + run: | + python - <<'PY' + from pathlib import Path + + grid_path = Path('research/universal_core/holonomy_v2_2/grid_induction.py') + grid_text = grid_path.read_text(encoding='utf-8') + marker = '\ndef run_public_task_v2_2(task: Any, engine: Any) -> dict[str, Any]:\n' + if grid_text.count(marker) != 1: + raise SystemExit('grid public entry marker mismatch') + prefix, _ = grid_text.split(marker, 1) + replacement = ''' + def run_public_task_v2_2( + task: Any, + engine: Any, + *, + mechanistic: bool = False, + ) -> dict[str, Any]: + if not _is_grid_task(task): + result = run_public_task_v2(task, engine) + if not mechanistic: + return result + from .grid_mechanistic_runtime import attach_delegate_trace + + return attach_delegate_trace(task, result) + + from .grid_mechanistic_runtime import run_grid_task + + return run_grid_task(task, mechanistic=mechanistic) + ''' + replacement = '\n'.join( + line[10:] if line.startswith(' ') else line + for line in replacement.splitlines() + ) + '\n' + grid_path.write_text(prefix + replacement, encoding='utf-8') + + trace_path = Path('research/universal_core/holonomy_v2_2/mechanistic_trace.py') + trace_text = trace_path.read_text(encoding='utf-8') + old = ''' if run_starts: + hidden_count = run_starts[0].get("hidden_count") + if type(hidden_count) is not int or hidden_count < 0: + raise ValueError("run-start hidden count must be non-negative") + hidden_events = _events_named(events, "hidden_execution") + by_candidate: dict[str, list[Mapping[str, Any]]] = {} + for item in hidden_events: + candidate_id = item.get("candidate_id") + if candidate_id not in representatives: + raise ValueError("hidden execution references non-representative candidate") + by_candidate.setdefault(str(candidate_id), []).append(item) + expected_indices = list(range(hidden_count)) + for candidate_id in sorted(representatives): + indices = sorted(int(item.get("hidden_index", -1)) for item in by_candidate.get(candidate_id, [])) + if indices != expected_indices: + raise ValueError("hidden executions do not cover every hidden input") + ''' + old = '\n'.join( + line[10:] if line.startswith(' ') else line + for line in old.splitlines() + ) + '\n' + new = ''' if run_starts: + demo_count = run_starts[0].get("demo_count") + hidden_count = run_starts[0].get("hidden_count") + if type(demo_count) is not int or demo_count < 0: + raise ValueError("run-start demonstration count must be non-negative") + if type(hidden_count) is not int or hidden_count < 0: + raise ValueError("run-start hidden count must be non-negative") + + demo_events = _events_named(events, "demo_execution") + demo_by_candidate: dict[str, list[Mapping[str, Any]]] = {} + for item in demo_events: + candidate_id = item.get("candidate_id") + if candidate_id not in proposed: + raise ValueError("demonstration execution references unknown candidate") + demo_by_candidate.setdefault(str(candidate_id), []).append(item) + expected_demo_indices = list(range(demo_count)) + for candidate_id in proposed_ids: + indices = sorted( + int(item.get("demo_index", -1)) + for item in demo_by_candidate.get(candidate_id, []) + ) + if indices != expected_demo_indices: + raise ValueError("demonstration executions do not cover every candidate and example") + + hidden_events = _events_named(events, "hidden_execution") + by_candidate: dict[str, list[Mapping[str, Any]]] = {} + for item in hidden_events: + candidate_id = item.get("candidate_id") + if candidate_id not in representatives: + raise ValueError("hidden execution references non-representative candidate") + by_candidate.setdefault(str(candidate_id), []).append(item) + expected_indices = list(range(hidden_count)) + for candidate_id in sorted(representatives): + indices = sorted( + int(item.get("hidden_index", -1)) + for item in by_candidate.get(candidate_id, []) + ) + if indices != expected_indices: + raise ValueError("hidden executions do not cover every hidden input") + ''' + new = '\n'.join( + line[10:] if line.startswith(' ') else line + for line in new.splitlines() + ) + '\n' + if trace_text.count(old) != 1: + raise SystemExit('trace completeness anchor mismatch') + trace_path.write_text(trace_text.replace(old, new, 1), encoding='utf-8') + PY + git diff --check + - name: Run focused mechanistic tests + id: focused + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py \ + research/universal_core/holonomy_v2_2/tests/test_mechanistic_trace.py \ + | tee /tmp/focused.log + - name: Run all V2.2 tests + id: cluster + if: always() + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q research/universal_core/holonomy_v2_2/tests | tee /tmp/v2-2.log + - name: Run frozen V2 integration tests + id: frozen + if: always() + continue-on-error: true + env: + PYTHONPATH: research + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py | tee /tmp/frozen.log + - name: Confirm frozen V2 bytes are unchanged + id: byte_gate + if: always() + continue-on-error: true + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Record result and commit only valid state + if: always() + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + FOCUSED_OUTCOME: ${{ steps.focused.outcome }} + CLUSTER_OUTCOME: ${{ steps.cluster.outcome }} + FROZEN_OUTCOME: ${{ steps.frozen.outcome }} + BYTE_OUTCOME: ${{ steps.byte_gate.outcome }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + outcomes = { + 'focused': os.environ['FOCUSED_OUTCOME'], + 'v2_2_cluster': os.environ['CLUSTER_OUTCOME'], + 'frozen_tests': os.environ['FROZEN_OUTCOME'], + 'frozen_byte_gate': os.environ['BYTE_OUTCOME'], + } + passed = all(value == 'success' for value in outcomes.values()) + evidence = { + 'schema_version': 1, + 'workflow_source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': passed, + 'outcomes': outcomes, + 'focused_output': Path('/tmp/focused.log').read_text(encoding='utf-8') if Path('/tmp/focused.log').exists() else '', + 'v2_2_output': Path('/tmp/v2-2.log').read_text(encoding='utf-8') if Path('/tmp/v2-2.log').exists() else '', + 'frozen_output': Path('/tmp/frozen.log').read_text(encoding='utf-8') if Path('/tmp/frozen.log').exists() else '', + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + } + name = 'GRID_MECHANISTIC_GREEN.json' if passed else 'GRID_MECHANISTIC_DIAGNOSTIC.json' + path = Path('research/universal_core/holonomy_v2_2/evidence') / name + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + Path('/tmp/passed').write_text('1' if passed else '0', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + if [ "$(cat /tmp/passed)" = "1" ]; then + git add \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/mechanistic_trace.py \ + research/universal_core/holonomy_v2_2/evidence/GRID_MECHANISTIC_GREEN.json + git commit -m 'arc: trace complete grid inference path' + else + git restore research/universal_core/holonomy_v2_2/grid_induction.py research/universal_core/holonomy_v2_2/mechanistic_trace.py + git add research/universal_core/holonomy_v2_2/evidence/GRID_MECHANISTIC_DIAGNOSTIC.json + git commit -m 'arc: record grid trace diagnostic failure' + fi + git push origin HEAD:universal-core-arc-training-v2-2 + test "$(cat /tmp/passed)" = "1" diff --git a/.github/workflows/arc-line-connect-corpus.yml b/.github/workflows/arc-line-connect-corpus.yml new file mode 100644 index 00000000..91e76751 --- /dev/null +++ b/.github/workflows/arc-line-connect-corpus.yml @@ -0,0 +1,347 @@ +name: ARC Line Connect Mechanistic Corpus + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-line-connect-corpus.yml' + +permissions: + contents: read + actions: read + +jobs: + training-corpus: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Download immutable benchmark corpus + uses: actions/download-artifact@v4 + with: + name: public-benchmarks-pinned-2026-08-04 + path: /tmp/pinned-public-benchmarks + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30929169392' + - name: Download immutable V2.2 baseline report + uses: actions/download-artifact@v4 + with: + name: arc-v2-2-training-e50965009efb96457fd769d97b42027f8a4e98c0 + path: /tmp/arc-v2-2-baseline + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30941837872' + - name: Verify V2.2 and frozen V2 gates + env: + PYTHONPATH: research + run: | + python -m pip install --disable-pip-version-check --quiet pytest + python -m pytest -q research/universal_core/holonomy_v2_2/tests + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py + git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Run all ARC training tasks with complete traces + run: | + mkdir -p /tmp/arc-v2-2-line/traces + python - <<'PY' | tee /tmp/arc-v2-2-line/console-summary.json + import json + import os + from pathlib import Path + + from research.universal_core.holonomy_v2.engine import HolonomyEngine + from research.universal_core.holonomy_v2.types import SearchConfig + from research.universal_core.holonomy_v2_2.mechanistic_trace import validate_trace + from research.universal_core.holonomy_v2_2.training_eval import run_training_corpus, write_report + from research.universal_core.public_benchmarks.suite import locate_vendor + + output = Path('/tmp/arc-v2-2-line') + trace_dir = output / 'traces' + vendor = locate_vendor(Path('/tmp/pinned-public-benchmarks')) + report = run_training_corpus( + vendor, + HolonomyEngine(SearchConfig()), + run_label='arc-v2-2-line-' + os.environ['GITHUB_SHA'][:12], + candidate_commit=os.environ['GITHUB_SHA'], + mechanistic_dir=trace_dir, + verify_parity=True, + ) + write_report(report, output / 'report.json') + + trace_paths = sorted( + path for path in trace_dir.glob('*.json') + if path.name != 'diagnosis.json' + ) + if len(trace_paths) != 1000: + raise SystemExit(f'expected 1000 task traces, found {len(trace_paths)}') + traces = {} + for path in trace_paths: + trace = json.loads(path.read_text(encoding='utf-8')) + validate_trace(trace) + traces[path.stem] = trace + if report['mechanistic']['trace_count'] != 1000: + raise SystemExit('aggregate trace count mismatch') + if report['mechanistic']['parity_count'] != 1000: + raise SystemExit('compact/mechanistic parity count mismatch') + + baseline = json.loads( + Path('/tmp/arc-v2-2-baseline/report.json').read_text(encoding='utf-8') + ) + baseline_rows = { + item['task_id']: int(item['metrics']['correct']) + for item in baseline['reports'] + } + current_rows = { + item['task_id']: int(item['metrics']['correct']) + for item in report['reports'] + } + lost_rows = { + task_id: {'baseline_correct': correct, 'current_correct': current_rows.get(task_id, 0)} + for task_id, correct in baseline_rows.items() + if current_rows.get(task_id, 0) < correct + } + gained_rows = { + task_id: {'baseline_correct': baseline_rows.get(task_id, 0), 'current_correct': correct} + for task_id, correct in current_rows.items() + if correct > baseline_rows.get(task_id, 0) + } + expected_gain_ids = [ + '0c786b71', '1e0a9b12', '1f876c06', '22168020', '22eb0ac0', + '2dee498d', '3906de3d', '3aa6fb7a', '3af2c5a8', '46442a0e', + '4c4377d9', '5614dbcf', '62c24649', '67e8384a', '68b67ca3', + '6d0aefbc', '6d75e8bb', '6fa7a44f', '746b3537', '7953d61e', + '7b7f7511', '7fe24cdd', '833dafe3', '8be77c9e', 'a59b95c0', + 'aabf363d', 'ac0a08a4', 'b230c067', 'b91ae062', 'c9e6f938', + 'ce8d95cc', 'd4b1c2b1', 'ded97339', 'e1baa8a4', 'e7639916', + 'eb5a1d5d', 'ed98d772', + ] + expected_gains = { + task_id: {'baseline_correct': 0, 'current_correct': 1} + for task_id in expected_gain_ids + } + if lost_rows: + raise SystemExit('lost baseline-correct rows: ' + json.dumps(lost_rows, sort_keys=True)) + if gained_rows != expected_gains: + raise SystemExit( + 'unexpected gained rows: expected ' + + json.dumps(expected_gains, sort_keys=True) + + ' found ' + + json.dumps(gained_rows, sort_keys=True) + ) + + expected_metrics = { + 'rows': 1076, + 'correct': 96, + 'accepted': 96, + 'incorrect_attempts': 0, + 'failed': 0, + } + for key, expected in expected_metrics.items(): + if int(report['metrics'][key]) != expected: + raise SystemExit( + f'metric {key} changed: expected {expected}, found {report["metrics"][key]}' + ) + expected_status = { + 'ACCEPTED': 96, + 'AMBIGUOUS_PROGRAM': 1, + 'GRAMMAR_EXHAUSTED': 979, + } + if report['status_counts'] != expected_status: + raise SystemExit( + 'status counts changed: expected ' + + json.dumps(expected_status, sort_keys=True) + + ' found ' + + json.dumps(report['status_counts'], sort_keys=True) + ) + + line_ids = ('1f876c06', '22168020', '22eb0ac0', 'ded97339') + line_details = {} + for task_id in line_ids: + trace = traces[task_id] + final = [event for event in trace['events'] if event.get('event') == 'final_decision'][0] + successes = [ + event for event in trace['events'] + if event.get('event') == 'hidden_execution' + and event.get('candidate_kind') == 'same_color_line_connect' + and event.get('outcome') == 'SUCCEEDED' + ] + if trace['terminal_status'] != 'ACCEPTED': + raise SystemExit(f'{task_id} did not transition to ACCEPTED') + if not successes: + raise SystemExit(f'{task_id} has no successful line candidate') + selected = str(final.get('selected_candidate_id', '')) + if not selected.startswith('line_connect:'): + raise SystemExit(f'{task_id} did not select line connection: {selected}') + line_details[task_id] = { + 'trace_digest': trace['trace_digest'], + 'successful_line_candidates': len(successes), + 'selected_candidate_id': selected, + } + + primitive_guards = { + 'aabf363d': 'marker_recolor:', + 'b230c067': 'component_outlier:', + '3aa6fb7a': 'bbox_complete:', + '6d75e8bb': 'bbox_complete:', + 'e7639916': 'bbox_complete:', + '4c4377d9': 'mirror_concat:', + '6d0aefbc': 'mirror_concat:', + '6fa7a44f': 'mirror_concat:', + '8be77c9e': 'mirror_concat:', + 'c9e6f938': 'mirror_concat:', + '0c786b71': 'quadrant_mosaic:', + '3af2c5a8': 'quadrant_mosaic:', + '46442a0e': 'quadrant_mosaic:', + '62c24649': 'quadrant_mosaic:', + '67e8384a': 'quadrant_mosaic:', + '7953d61e': 'quadrant_mosaic:', + '7fe24cdd': 'quadrant_mosaic:', + '833dafe3': 'quadrant_mosaic:', + 'ed98d772': 'quadrant_mosaic:', + 'a59b95c0': 'palette_repeat:', + 'ac0a08a4': 'palette_repeat:', + 'b91ae062': 'palette_repeat:', + 'd4b1c2b1': 'palette_repeat:', + '2dee498d': 'axis_collapse:', + '746b3537': 'axis_collapse:', + '7b7f7511': 'axis_collapse:', + 'ce8d95cc': 'axis_collapse:', + 'e1baa8a4': 'axis_collapse:', + 'eb5a1d5d': 'axis_collapse:', + '5614dbcf': 'block_reduce:', + '68b67ca3': 'block_reduce:', + '1e0a9b12': 'gravity_compact:', + '3906de3d': 'gravity_compact:', + } + prior_mechanisms = {} + for task_id, prefix in primitive_guards.items(): + trace = traces[task_id] + final = [event for event in trace['events'] if event.get('event') == 'final_decision'][0] + if trace['terminal_status'] != 'ACCEPTED': + raise SystemExit(f'{task_id} regressed') + selected = str(final.get('selected_candidate_id', '')) + if not selected.startswith(prefix): + raise SystemExit(f'{task_id} selected primitive changed: {selected}') + prior_mechanisms[task_id] = { + 'selected_candidate_id': selected, + 'trace_digest': trace['trace_digest'], + } + + ambiguous = traces['60b61512'] + classes = [event for event in ambiguous['events'] if event.get('event') == 'equivalence_class'] + bbox_classes = [ + event for event in classes + if any(str(candidate_id).startswith('bbox_complete:') for candidate_id in event['candidate_ids']) + ] + if ambiguous['terminal_status'] != 'AMBIGUOUS_PROGRAM': + raise SystemExit('60b61512 did not remain ambiguity-safe') + if len(bbox_classes) < 2: + raise SystemExit('60b61512 lost bounding-box disagreement') + + transitions = { + 'schema_version': 1, + 'candidate_commit': os.environ['GITHUB_SHA'], + 'baseline_commit': 'e50965009efb96457fd769d97b42027f8a4e98c0', + 'lost_rows': lost_rows, + 'gained_rows': gained_rows, + 'expected_gains_matched': True, + 'incorrect_attempts': int(report['metrics']['incorrect_attempts']), + } + mechanisms = { + 'schema_version': 1, + 'line_tasks': line_details, + 'prior_mechanisms': prior_mechanisms, + '60b61512': { + 'terminal_status': ambiguous['terminal_status'], + 'trace_digest': ambiguous['trace_digest'], + 'bbox_equivalence_class_count': len(bbox_classes), + 'safety_action': 'abstain_on_connectivity_disagreement', + }, + } + parity = { + 'schema_version': 1, + 'candidate_commit': os.environ['GITHUB_SHA'], + 'checked_tasks': 1000, + 'mismatches': 0, + 'lost_baseline_rows': lost_rows, + } + for name, data in ( + ('transitions.json', transitions), + ('mechanisms.json', mechanisms), + ('parity.json', parity), + ): + (output / name).write_text( + json.dumps(data, sort_keys=True, indent=2) + '\n', encoding='utf-8' + ) + + print(json.dumps({ + 'report_digest': report['report_digest'], + 'diagnosis_digest': report['mechanistic']['diagnosis_digest'], + 'evaluated_tasks': report['evaluated_tasks'], + 'trace_count': report['mechanistic']['trace_count'], + 'parity_count': report['mechanistic']['parity_count'], + 'metrics': report['metrics'], + 'status_counts': report['status_counts'], + 'transitions': transitions, + 'line_tasks': line_details, + }, sort_keys=True)) + PY + - name: Build checksum manifest after all writers close + run: | + python - <<'PY' + import hashlib + from pathlib import Path + + output = Path('/tmp/arc-v2-2-line') + lines = [] + for path in sorted( + item for item in output.rglob('*') + if item.is_file() and item.name != 'SHA256SUMS' + ): + digest = hashlib.sha256(path.read_bytes()).hexdigest() + lines.append(f'{digest} {path.relative_to(output).as_posix()}') + (output / 'SHA256SUMS').write_text('\n'.join(lines) + '\n', encoding='utf-8') + PY + - name: Verify checksum manifest independently + working-directory: /tmp/arc-v2-2-line + run: sha256sum -c SHA256SUMS + - name: Publish line connection evidence + uses: actions/upload-artifact@v4 + with: + name: arc-v2-2-line-connect-${{ github.sha }} + path: /tmp/arc-v2-2-line + if-no-files-found: error + retention-days: 90 + compression-level: 6 + - name: Write run summary + run: | + python - <<'PY' + import json + from pathlib import Path + + root = Path('/tmp/arc-v2-2-line') + report = json.loads((root / 'report.json').read_text()) + mechanisms = json.loads((root / 'mechanisms.json').read_text()) + lines = [ + '# ARC V2.2 same-color line connection corpus', + '', + f"- Candidate: `{report['identity']['candidate_commit']}`", + f"- Report digest: `{report['report_digest']}`", + f"- Diagnosis digest: `{report['mechanistic']['diagnosis_digest']}`", + f"- Metrics: `{report['metrics']}`", + f"- Status counts: `{report['status_counts']}`", + f"- Line gains: `{sorted(mechanisms['line_tasks'])}`", + f"- 60b61512 action: `{mechanisms['60b61512']['safety_action']}`", + '- Public ARC evaluation rerun: `no`', + ] + Path('/tmp/line-summary.md').write_text('\n'.join(lines) + '\n', encoding='utf-8') + PY + cat /tmp/line-summary.md >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/arc-line-connect-module-green.yml b/.github/workflows/arc-line-connect-module-green.yml new file mode 100644 index 00000000..a358536a --- /dev/null +++ b/.github/workflows/arc-line-connect-module-green.yml @@ -0,0 +1,69 @@ +name: ARC Line Connect Module GREEN + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-line-connect-module-green.yml' + +permissions: + contents: write + +jobs: + verify-module: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Run isolated line tests + run: | + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_line_connect_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_line_connect.py::test_horizontal_connection_fills_between_same_color_anchors \ + research/universal_core/holonomy_v2_2/tests/test_line_connect.py::test_vertical_connection_fills_between_same_color_anchors \ + research/universal_core/holonomy_v2_2/tests/test_line_connect.py::test_diagonal_connection_fills_both_diagonal_families \ + research/universal_core/holonomy_v2_2/tests/test_line_connect.py::test_newly_filled_cells_do_not_become_anchors \ + research/universal_core/holonomy_v2_2/tests/test_line_connect.py::test_connection_rejects_cross_color_overwrite \ + research/universal_core/holonomy_v2_2/tests/test_line_connect.py::test_program_derivation_returns_unique_horizontal_candidate \ + research/universal_core/holonomy_v2_2/tests/test_line_connect.py::test_program_derivation_rejects_shape_change \ + research/universal_core/holonomy_v2_2/tests/test_line_connect.py::test_program_derivation_skips_noop_demonstration \ + | tee /tmp/line-module-green.log + - name: Confirm frozen bytes + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Commit module evidence + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + evidence = { + 'schema_version': 1, + 'source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': True, + 'test_count': 9, + 'test_output': Path('/tmp/line-module-green.log').read_text(encoding='utf-8'), + 'expected_new_exact_tasks': ['1f876c06', '22168020', '22eb0ac0', 'ded97339'], + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'frozen_byte_gate': True, + 'public_arc_evaluation_rerun': False, + } + path = Path('research/universal_core/holonomy_v2_2/evidence/LINE_CONNECT_MODULE_GREEN.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/LINE_CONNECT_MODULE_GREEN.json + git commit -m 'arc: record line connection module GREEN' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-line-connect-pin-fixtures.yml b/.github/workflows/arc-line-connect-pin-fixtures.yml new file mode 100644 index 00000000..c3ae3245 --- /dev/null +++ b/.github/workflows/arc-line-connect-pin-fixtures.yml @@ -0,0 +1,58 @@ +name: Pin ARC Line Connect Fixtures + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-line-connect-pin-fixtures.yml' + +permissions: + contents: write + actions: read + +jobs: + pin: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - name: Download immutable benchmark corpus + uses: actions/download-artifact@v4 + with: + name: public-benchmarks-pinned-2026-08-04 + path: /tmp/pinned-public-benchmarks + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30929169392' + - name: Copy exact immutable payloads + run: | + set -euo pipefail + probe=$(find /tmp/pinned-public-benchmarks -type f -path '*/arc_agi_2/data/training/1f876c06.json' -print -quit) + test -n "$probe" + src=$(dirname "$probe") + dst=research/universal_core/holonomy_v2_2/tests/fixtures/line_connect + mkdir -p "$dst" + cp "$src/1f876c06.json" "$dst/1f876c06.json" + cp "$src/22168020.json" "$dst/22168020.json" + cp "$src/22eb0ac0.json" "$dst/22eb0ac0.json" + cp "$src/ded97339.json" "$dst/ded97339.json" + printf '%s %s\n' \ + b53a1a25685895400323e813a373f0a66fc9512f7dbbb326d52a93695fca8143 "$dst/1f876c06.json" \ + d067b4f83f84120300310199e3a95c65258edcc53d8b0f425288d7ddb71a962e "$dst/22168020.json" \ + cf37cc34e7b0aa123c2806eca77a7d9916b50afea4e6e2db71d5149fe1f1af0d "$dst/22eb0ac0.json" \ + fd14b4ee37bb1539c26a79abed743626b37f5935794ce82cbd0b3332c207906e "$dst/ded97339.json" \ + > /tmp/line-fixtures.sha256 + sha256sum -c /tmp/line-fixtures.sha256 + - name: Confirm frozen V2 bytes + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Commit fixtures + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/tests/fixtures/line_connect + git commit -m 'test: pin same-color line connection fixtures' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-line-connect-red.yml b/.github/workflows/arc-line-connect-red.yml new file mode 100644 index 00000000..c98aa59a --- /dev/null +++ b/.github/workflows/arc-line-connect-red.yml @@ -0,0 +1,85 @@ +name: ARC Line Connect RED + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-line-connect-red.yml' + +permissions: + contents: write + +jobs: + verify-red: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Verify pinned line fixtures + run: python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_line_connect_fixtures.py + - name: Require missing line connection module + run: | + set +e + python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_line_connect.py > /tmp/line-red.log 2>&1 + status=$? + set -e + cat /tmp/line-red.log + test "$status" -ne 0 + grep -F "No module named 'research.universal_core.holonomy_v2_2.line_connect'" /tmp/line-red.log + - name: Verify frozen V2.1 byte identity + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Commit RED evidence + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + evidence = { + 'schema_version': 1, + 'source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'failure_confirmed': True, + 'fixture_integrity': True, + 'expected_failure': "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.line_connect'", + 'expected_new_exact_tasks': ['1f876c06', '22168020', '22eb0ac0', 'ded97339'], + 'training_scan': { + 'compatible_tasks': 4, + 'wrong_completed_hidden_outputs': 0, + 'ambiguous_hidden_output_classes': 0, + 'compatible_accepted_tasks': 0, + }, + 'expected_cumulative_metrics': { + 'rows': 1076, + 'accepted': 96, + 'correct': 96, + 'incorrect_attempts': 0, + 'failed': 0, + }, + 'expected_status_counts': { + 'ACCEPTED': 96, + 'AMBIGUOUS_PROGRAM': 1, + 'GRAMMAR_EXHAUSTED': 979, + }, + 'captured_log': Path('/tmp/line-red.log').read_text(encoding='utf-8'), + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'public_arc_evaluation_rerun': False, + } + path = Path('research/universal_core/holonomy_v2_2/evidence/LINE_CONNECT_RED.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/LINE_CONNECT_RED.json + git commit -m 'arc: record line connection RED evidence' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-line-connect-wire.yml b/.github/workflows/arc-line-connect-wire.yml new file mode 100644 index 00000000..5b031e8c --- /dev/null +++ b/.github/workflows/arc-line-connect-wire.yml @@ -0,0 +1,205 @@ +name: Wire ARC Line Connect + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-line-connect-wire.yml' + +permissions: + contents: write + +jobs: + wire-and-verify: + runs-on: ubuntu-latest + timeout-minutes: 50 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Apply guarded line connection integration + run: | + python - <<'PY' + from pathlib import Path + + grid_path = Path('research/universal_core/holonomy_v2_2/grid_induction.py') + text = grid_path.read_text(encoding='utf-8') + import_anchor = 'from .gravity_compact import apply_gravity_compact\n' + if text.count(import_anchor) != 1 or 'from .line_connect import' in text: + raise SystemExit('line import anchor mismatch') + text = text.replace( + import_anchor, + import_anchor + 'from .line_connect import apply_line_connect\n', + 1, + ) + if text.count('_VERSION = "grid-v2.2-14"') != 1: + raise SystemExit('line version anchor mismatch') + text = text.replace('_VERSION = "grid-v2.2-14"', '_VERSION = "grid-v2.2-15"', 1) + apply_anchor = ' if kind == "axis_gravity_compact":\n' + if text.count(apply_anchor) != 1: + raise SystemExit('line apply anchor mismatch') + text = text.replace( + apply_anchor, + ' if kind == "same_color_line_connect":\n' + ' return apply_line_connect(program, grid)\n' + + apply_anchor, + 1, + ) + grid_path.write_text(text, encoding='utf-8') + + runtime_path = Path('research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py') + text = runtime_path.read_text(encoding='utf-8') + import_anchor = 'from .gravity_compact import gravity_compact_programs\n' + if text.count(import_anchor) != 1 or 'line_connect_programs' in text: + raise SystemExit('line runtime import anchor mismatch') + text = text.replace( + import_anchor, + import_anchor + 'from .line_connect import line_connect_programs\n', + 1, + ) + derive_anchor = ' gravity_compact, gravity_reason = gravity_compact_programs(demos)\n' + if text.count(derive_anchor) != 1: + raise SystemExit('line derivation anchor mismatch') + text = text.replace( + derive_anchor, + derive_anchor + ' line_connect, line_reason = line_connect_programs(demos)\n', + 1, + ) + batch_anchor = ' ("gravity_compact", gravity_compact, gravity_reason),\n' + if text.count(batch_anchor) != 1: + raise SystemExit('line batch anchor mismatch') + text = text.replace( + batch_anchor, + batch_anchor + ' ("line_connect", line_connect, line_reason),\n', + 1, + ) + runtime_path.write_text(text, encoding='utf-8') + PY + python -m py_compile \ + research/universal_core/holonomy_v2_2/line_connect.py \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py + git diff --check + - name: Run focused line connection tests + id: focused + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_line_connect_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_line_connect.py \ + research/universal_core/holonomy_v2_2/tests/test_gravity_compact_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_gravity_compact.py \ + research/universal_core/holonomy_v2_2/tests/test_block_reduce_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_block_reduce.py \ + research/universal_core/holonomy_v2_2/tests/test_axis_collapse_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_axis_collapse.py \ + research/universal_core/holonomy_v2_2/tests/test_palette_repeat_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_palette_repeat.py \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py \ + research/universal_core/holonomy_v2_2/tests/test_mirror_concat.py \ + research/universal_core/holonomy_v2_2/tests/test_bbox_complete.py \ + research/universal_core/holonomy_v2_2/tests/test_component_outlier.py \ + research/universal_core/holonomy_v2_2/tests/test_marker_recolor.py \ + research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py \ + research/universal_core/holonomy_v2_2/tests/test_grid_induction.py::test_cell_scale_and_whole_grid_tile_are_distinct \ + | tee /tmp/line-focused.log + - name: Run all V2.2 tests + id: cluster + if: always() + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q research/universal_core/holonomy_v2_2/tests | tee /tmp/line-v2-2.log + - name: Run frozen V2 integration tests + id: frozen + if: always() + continue-on-error: true + env: + PYTHONPATH: research + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py | tee /tmp/line-frozen.log + - name: Confirm frozen V2 bytes are unchanged + id: byte_gate + if: always() + continue-on-error: true + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Record result and commit only valid state + if: always() + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + FOCUSED_OUTCOME: ${{ steps.focused.outcome }} + CLUSTER_OUTCOME: ${{ steps.cluster.outcome }} + FROZEN_OUTCOME: ${{ steps.frozen.outcome }} + BYTE_OUTCOME: ${{ steps.byte_gate.outcome }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + outcomes = { + 'focused': os.environ['FOCUSED_OUTCOME'], + 'v2_2_cluster': os.environ['CLUSTER_OUTCOME'], + 'frozen_tests': os.environ['FROZEN_OUTCOME'], + 'frozen_byte_gate': os.environ['BYTE_OUTCOME'], + } + passed = all(value == 'success' for value in outcomes.values()) + evidence = { + 'schema_version': 1, + 'workflow_source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': passed, + 'outcomes': outcomes, + 'focused_output': Path('/tmp/line-focused.log').read_text(encoding='utf-8') if Path('/tmp/line-focused.log').exists() else '', + 'v2_2_output': Path('/tmp/line-v2-2.log').read_text(encoding='utf-8') if Path('/tmp/line-v2-2.log').exists() else '', + 'frozen_output': Path('/tmp/line-frozen.log').read_text(encoding='utf-8') if Path('/tmp/line-frozen.log').exists() else '', + 'expected_new_exact_tasks': ['1f876c06', '22168020', '22eb0ac0', 'ded97339'], + 'expected_training_delta': { + 'correct_rows': 4, + 'accepted_rows': 4, + 'grammar_exhausted_rows': -4, + }, + 'expected_cumulative_metrics': { + 'rows': 1076, + 'correct': 96, + 'accepted': 96, + 'incorrect_attempts': 0, + 'failed': 0, + }, + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'public_arc_evaluation_rerun': False, + } + name = 'LINE_CONNECT_GREEN.json' if passed else 'LINE_CONNECT_DIAGNOSTIC.json' + path = Path('research/universal_core/holonomy_v2_2/evidence') / name + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + Path('/tmp/line-passed').write_text('1' if passed else '0', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + if [ "$(cat /tmp/line-passed)" = "1" ]; then + git add \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py \ + research/universal_core/holonomy_v2_2/evidence/LINE_CONNECT_GREEN.json + git commit -m 'arc: wire same-color line connection' + else + git restore \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py + git add research/universal_core/holonomy_v2_2/evidence/LINE_CONNECT_DIAGNOSTIC.json + git commit -m 'arc: record line connection diagnostic failure' + fi + git push origin HEAD:universal-core-arc-training-v2-2 + test "$(cat /tmp/line-passed)" = "1" diff --git a/.github/workflows/arc-marker-recolor-red.yml b/.github/workflows/arc-marker-recolor-red.yml new file mode 100644 index 00000000..b35dc215 --- /dev/null +++ b/.github/workflows/arc-marker-recolor-red.yml @@ -0,0 +1,67 @@ +name: ARC Marker Recolor RED + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-marker-recolor-red.yml' + +permissions: + contents: write + +jobs: + verify-red: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Require missing marker module + run: | + set +e + python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_marker_recolor.py > /tmp/marker-red.log 2>&1 + status=$? + set -e + cat /tmp/marker-red.log + test "$status" -ne 0 + grep -F "No module named 'research.universal_core.holonomy_v2_2.marker_recolor'" /tmp/marker-red.log + - name: Commit RED evidence + env: + RED_SOURCE_SHA: ${{ github.sha }} + RED_RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + evidence = { + 'schema_version': 1, + 'red_commit': os.environ['RED_SOURCE_SHA'], + 'red_run_id': os.environ['RED_RUN_ID'], + 'test_command': 'python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_marker_recolor.py', + 'expected_failure': "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.marker_recolor'", + 'failure_confirmed': True, + 'captured_log': Path('/tmp/marker-red.log').read_text(encoding='utf-8'), + 'source_trace': { + 'task_id': 'aabf363d', + 'trace_digest': '29544f72d9511885b6ea71838162ee86d83437198841c2e4815a5ffb824f54e4', + 'failure': "color_map KeyError: 'unmapped grid cell'", + }, + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + } + path = Path('research/universal_core/holonomy_v2_2/evidence/MARKER_RECOLOR_RED.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/MARKER_RECOLOR_RED.json + git commit -m 'arc: record marker recolor RED evidence' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-marker-recolor-wire.yml b/.github/workflows/arc-marker-recolor-wire.yml new file mode 100644 index 00000000..927cbbbf --- /dev/null +++ b/.github/workflows/arc-marker-recolor-wire.yml @@ -0,0 +1,173 @@ +name: Wire ARC Marker Recolor + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-marker-recolor-wire.yml' + +permissions: + contents: write + +jobs: + wire-and-verify: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Apply guarded marker integration + run: | + python - <<'PY' + from pathlib import Path + + grid_path = Path('research/universal_core/holonomy_v2_2/grid_induction.py') + text = grid_path.read_text(encoding='utf-8') + import_anchor = 'from .component_select import apply_component_select, component_select_candidates\n' + import_value = import_anchor + 'from .marker_recolor import apply_marker_recolor\n' + if text.count(import_anchor) != 1 or 'from .marker_recolor import' in text: + raise SystemExit('marker import anchor mismatch') + text = text.replace(import_anchor, import_value, 1) + + version_anchor = '_VERSION = "grid-v2.2-5"' + if text.count(version_anchor) != 1: + raise SystemExit('marker version anchor mismatch') + text = text.replace(version_anchor, '_VERSION = "grid-v2.2-6"', 1) + + apply_anchor = ' if kind == "color_map":\n' + apply_value = ( + ' if kind == "marker_recolor":\n' + ' return apply_marker_recolor(program, grid)\n' + ) + if text.count(apply_anchor) != 1: + raise SystemExit('marker apply anchor mismatch') + text = text.replace(apply_anchor, apply_value + apply_anchor, 1) + grid_path.write_text(text, encoding='utf-8') + + runtime_path = Path('research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py') + text = runtime_path.read_text(encoding='utf-8') + runtime_import = 'from .mechanistic_trace import TraceRecorder\n' + runtime_import_value = runtime_import + 'from .marker_recolor import marker_recolor_programs\n' + if text.count(runtime_import) != 1 or 'marker_recolor_programs' in text: + raise SystemExit('marker runtime import anchor mismatch') + text = text.replace(runtime_import, runtime_import_value, 1) + + derived_anchor = ' component_select, component_reason = component_select_programs(demos)\n' + derived_value = derived_anchor + ' marker_recolor, marker_reason = marker_recolor_programs(demos)\n' + if text.count(derived_anchor) != 1: + raise SystemExit('marker derivation anchor mismatch') + text = text.replace(derived_anchor, derived_value, 1) + + batch_anchor = ' ("panel", panel, panel_reason),\n' + batch_value = ' ("marker_recolor", marker_recolor, marker_reason),\n' + batch_anchor + if text.count(batch_anchor) != 1: + raise SystemExit('marker batch anchor mismatch') + text = text.replace(batch_anchor, batch_value, 1) + runtime_path.write_text(text, encoding='utf-8') + PY + python -m py_compile \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py \ + research/universal_core/holonomy_v2_2/marker_recolor.py + git diff --check + - name: Run focused marker tests + id: focused + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_marker_recolor.py \ + research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py \ + | tee /tmp/focused.log + - name: Run all V2.2 tests + id: cluster + if: always() + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q research/universal_core/holonomy_v2_2/tests | tee /tmp/v2-2.log + - name: Run frozen V2 integration tests + id: frozen + if: always() + continue-on-error: true + env: + PYTHONPATH: research + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py | tee /tmp/frozen.log + - name: Confirm frozen V2 bytes are unchanged + id: byte_gate + if: always() + continue-on-error: true + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Record result and commit only valid state + if: always() + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + FOCUSED_OUTCOME: ${{ steps.focused.outcome }} + CLUSTER_OUTCOME: ${{ steps.cluster.outcome }} + FROZEN_OUTCOME: ${{ steps.frozen.outcome }} + BYTE_OUTCOME: ${{ steps.byte_gate.outcome }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + outcomes = { + 'focused': os.environ['FOCUSED_OUTCOME'], + 'v2_2_cluster': os.environ['CLUSTER_OUTCOME'], + 'frozen_tests': os.environ['FROZEN_OUTCOME'], + 'frozen_byte_gate': os.environ['BYTE_OUTCOME'], + } + passed = all(value == 'success' for value in outcomes.values()) + evidence = { + 'schema_version': 1, + 'workflow_source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': passed, + 'outcomes': outcomes, + 'focused_output': Path('/tmp/focused.log').read_text(encoding='utf-8') if Path('/tmp/focused.log').exists() else '', + 'v2_2_output': Path('/tmp/v2-2.log').read_text(encoding='utf-8') if Path('/tmp/v2-2.log').exists() else '', + 'frozen_output': Path('/tmp/frozen.log').read_text(encoding='utf-8') if Path('/tmp/frozen.log').exists() else '', + 'source_trace_digest': '29544f72d9511885b6ea71838162ee86d83437198841c2e4815a5ffb824f54e4', + 'expected_training_delta': { + 'task_id': 'aabf363d', + 'correct_rows': 1, + 'accepted_rows': 1, + 'execution_failed_rows': -1, + }, + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + } + name = 'MARKER_RECOLOR_GREEN.json' if passed else 'MARKER_RECOLOR_DIAGNOSTIC.json' + path = Path('research/universal_core/holonomy_v2_2/evidence') / name + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + Path('/tmp/passed').write_text('1' if passed else '0', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + if [ "$(cat /tmp/passed)" = "1" ]; then + git add \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py \ + research/universal_core/holonomy_v2_2/evidence/MARKER_RECOLOR_GREEN.json + git commit -m 'arc: wire relational marker recolor' + else + git restore \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py + git add research/universal_core/holonomy_v2_2/evidence/MARKER_RECOLOR_DIAGNOSTIC.json + git commit -m 'arc: record marker recolor diagnostic failure' + fi + git push origin HEAD:universal-core-arc-training-v2-2 + test "$(cat /tmp/passed)" = "1" diff --git a/.github/workflows/arc-mechanistic-run-verifier.yml b/.github/workflows/arc-mechanistic-run-verifier.yml new file mode 100644 index 00000000..e84a82b8 --- /dev/null +++ b/.github/workflows/arc-mechanistic-run-verifier.yml @@ -0,0 +1,154 @@ +name: Verify ARC Mechanistic Corpus Run + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-mechanistic-run-verifier.yml' + +permissions: + actions: read + contents: write + +jobs: + verify-target-run: + runs-on: ubuntu-latest + timeout-minutes: 110 + env: + GH_TOKEN: ${{ github.token }} + TARGET_SHA: bbb192edd1706b204428041015779c28ded0c4b7 + REPOSITORY: Chrisbryan17/MLE-agent + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - name: Wait for exact mechanistic corpus run + run: | + set -euo pipefail + run_id='' + status='' + conclusion='' + for attempt in $(seq 1 180); do + runs=$(gh api --method GET "repos/$REPOSITORY/actions/runs" \ + -f head_sha="$TARGET_SHA" \ + -f per_page=100) + run_id=$(printf '%s' "$runs" | jq -r ' + [.workflow_runs[] + | select(.name == "ARC V2.2 Mechanistic Training Corpus") + | select(.head_sha == env.TARGET_SHA)] + | sort_by(.created_at) + | last + | .id // empty') + if [ -n "$run_id" ]; then + run=$(gh api "repos/$REPOSITORY/actions/runs/$run_id") + status=$(printf '%s' "$run" | jq -r '.status') + conclusion=$(printf '%s' "$run" | jq -r '.conclusion // ""') + echo "run=$run_id status=$status conclusion=$conclusion attempt=$attempt" + if [ "$status" = 'completed' ]; then + break + fi + else + echo "target run not visible yet; attempt=$attempt" + fi + sleep 30 + done + test -n "$run_id" + test "$status" = 'completed' + printf '%s\n' "$run_id" > /tmp/run-id + printf '%s\n' "$conclusion" > /tmp/conclusion + - name: Capture run, artifact, jobs, and failure logs + if: always() + run: | + set -euo pipefail + run_id=$(cat /tmp/run-id 2>/dev/null || true) + conclusion=$(cat /tmp/conclusion 2>/dev/null || true) + mkdir -p /tmp/run-evidence + if [ -n "$run_id" ]; then + gh api "repos/$REPOSITORY/actions/runs/$run_id" > /tmp/run-evidence/run.json + gh api "repos/$REPOSITORY/actions/runs/$run_id/jobs?per_page=100" > /tmp/run-evidence/jobs.json + gh api "repos/$REPOSITORY/actions/runs/$run_id/artifacts?per_page=100" > /tmp/run-evidence/artifacts.json + if [ "$conclusion" != 'success' ]; then + gh run view "$run_id" --repo "$REPOSITORY" --log-failed > /tmp/run-evidence/failed.log 2>&1 || true + fi + else + printf '%s\n' 'target run was not discovered before verifier timeout' > /tmp/run-evidence/failed.log + fi + - name: Commit immutable run evidence + if: always() + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + root = Path('/tmp/run-evidence') + run = json.loads((root / 'run.json').read_text()) if (root / 'run.json').exists() else {} + jobs = json.loads((root / 'jobs.json').read_text()) if (root / 'jobs.json').exists() else {'jobs': []} + artifacts = json.loads((root / 'artifacts.json').read_text()) if (root / 'artifacts.json').exists() else {'artifacts': []} + target_name = 'arc-v2-2-mechanistic-' + os.environ['TARGET_SHA'] + artifact = next( + (item for item in artifacts.get('artifacts', []) if item.get('name') == target_name), + None, + ) + failed_log = (root / 'failed.log').read_text(encoding='utf-8', errors='replace') if (root / 'failed.log').exists() else '' + evidence = { + 'schema_version': 1, + 'target_commit': os.environ['TARGET_SHA'], + 'workflow_name': run.get('name'), + 'run_id': run.get('id'), + 'run_number': run.get('run_number'), + 'status': run.get('status'), + 'conclusion': run.get('conclusion'), + 'event': run.get('event'), + 'created_at': run.get('created_at'), + 'updated_at': run.get('updated_at'), + 'artifact': None if artifact is None else { + 'id': artifact.get('id'), + 'name': artifact.get('name'), + 'size_in_bytes': artifact.get('size_in_bytes'), + 'digest': artifact.get('digest'), + 'expired': artifact.get('expired'), + }, + 'jobs': [ + { + 'id': item.get('id'), + 'name': item.get('name'), + 'status': item.get('status'), + 'conclusion': item.get('conclusion'), + 'started_at': item.get('started_at'), + 'completed_at': item.get('completed_at'), + 'steps': [ + { + 'number': step.get('number'), + 'name': step.get('name'), + 'status': step.get('status'), + 'conclusion': step.get('conclusion'), + } + for step in item.get('steps', []) + ], + } + for item in jobs.get('jobs', []) + ], + 'failed_log_tail': failed_log[-20000:], + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'public_arc_evaluation_rerun': False, + } + path = Path('docs/evidence/arc_v2_2/FULL_MARKER_RECOLOR_RUN_0007.json') + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add docs/evidence/arc_v2_2/FULL_MARKER_RECOLOR_RUN_0007.json + git commit -m 'arc: record marker recolor corpus run' + git push origin HEAD:universal-core-arc-training-v2-2 + conclusion=$(python -c "import json; print(json.load(open('docs/evidence/arc_v2_2/FULL_MARKER_RECOLOR_RUN_0007.json'))['conclusion'])") + test "$conclusion" = 'success' + python - <<'PY' + import json + evidence = json.load(open('docs/evidence/arc_v2_2/FULL_MARKER_RECOLOR_RUN_0007.json')) + if not evidence['artifact']: + raise SystemExit('successful run did not publish the expected artifact') + PY diff --git a/.github/workflows/arc-mechanistic-trace-green.yml b/.github/workflows/arc-mechanistic-trace-green.yml new file mode 100644 index 00000000..09997641 --- /dev/null +++ b/.github/workflows/arc-mechanistic-trace-green.yml @@ -0,0 +1,74 @@ +name: ARC Mechanistic Trace GREEN + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-mechanistic-trace-green.yml' + +permissions: + contents: write + +jobs: + verify-green: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install test dependency + run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Run focused recorder tests + run: | + python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_mechanistic_trace.py | tee /tmp/focused.log + - name: Run all V2.2 tests + run: | + python -m pytest -q research/universal_core/holonomy_v2_2/tests | tee /tmp/v2-2.log + - name: Run frozen V2 integration tests + env: + PYTHONPATH: research + run: | + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py | tee /tmp/frozen.log + - name: Confirm frozen V2 bytes are unchanged + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Commit GREEN evidence + env: + GREEN_SOURCE_SHA: ${{ github.sha }} + GREEN_RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + evidence = { + 'schema_version': 1, + 'branch': 'universal-core-arc-training-v2-2', + 'green_commit': os.environ['GREEN_SOURCE_SHA'], + 'green_run_id': os.environ['GREEN_RUN_ID'], + 'focused_command': 'python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_mechanistic_trace.py', + 'focused_output': Path('/tmp/focused.log').read_text(encoding='utf-8'), + 'v2_2_command': 'python -m pytest -q research/universal_core/holonomy_v2_2/tests', + 'v2_2_output': Path('/tmp/v2-2.log').read_text(encoding='utf-8'), + 'frozen_command': 'python -m pytest -q research/universal_core/holonomy_v2/tests/test_blind_adapter.py research/universal_core/holonomy_v2/tests/test_engine.py', + 'frozen_output': Path('/tmp/frozen.log').read_text(encoding='utf-8'), + 'frozen_byte_gate': True, + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + } + path = Path('research/universal_core/holonomy_v2_2/evidence/MECHANISTIC_TRACE_GREEN.json') + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/MECHANISTIC_TRACE_GREEN.json + git commit -m 'arc: record mechanistic trace GREEN evidence' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-mechanistic-trace-red.yml b/.github/workflows/arc-mechanistic-trace-red.yml new file mode 100644 index 00000000..34f7ed11 --- /dev/null +++ b/.github/workflows/arc-mechanistic-trace-red.yml @@ -0,0 +1,68 @@ +name: ARC Mechanistic Trace RED + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-mechanistic-trace-red.yml' + +permissions: + contents: write + +jobs: + verify-red: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install test dependency + run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Require exact missing-recorder failure + id: red + shell: bash + run: | + set +e + python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_mechanistic_trace.py > /tmp/mechanistic-trace-red.log 2>&1 + status=$? + set -e + cat /tmp/mechanistic-trace-red.log + test "$status" -ne 0 + grep -F "No module named 'research.universal_core.holonomy_v2_2.mechanistic_trace'" /tmp/mechanistic-trace-red.log + - name: Commit RED evidence + env: + RED_SOURCE_SHA: ${{ github.sha }} + RED_RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + log = Path('/tmp/mechanistic-trace-red.log').read_text(encoding='utf-8') + evidence = { + 'schema_version': 1, + 'branch': 'universal-core-arc-training-v2-2', + 'red_commit': os.environ['RED_SOURCE_SHA'], + 'red_run_id': os.environ['RED_RUN_ID'], + 'test_command': 'python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_mechanistic_trace.py', + 'expected_failure': "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.mechanistic_trace'", + 'failure_confirmed': True, + 'captured_log': log, + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + } + path = Path('research/universal_core/holonomy_v2_2/evidence/MECHANISTIC_TRACE_RED.json') + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/MECHANISTIC_TRACE_RED.json + git commit -m 'arc: record mechanistic trace RED evidence' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-mirror-corpus.yml b/.github/workflows/arc-mirror-corpus.yml new file mode 100644 index 00000000..3ad4a80c --- /dev/null +++ b/.github/workflows/arc-mirror-corpus.yml @@ -0,0 +1,312 @@ +name: ARC Mirror Concatenation Mechanistic Corpus + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-mirror-corpus.yml' + +permissions: + contents: read + actions: read + +jobs: + training-corpus: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Download immutable benchmark corpus + uses: actions/download-artifact@v4 + with: + name: public-benchmarks-pinned-2026-08-04 + path: /tmp/pinned-public-benchmarks + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30929169392' + - name: Download immutable V2.2 baseline report + uses: actions/download-artifact@v4 + with: + name: arc-v2-2-training-e50965009efb96457fd769d97b42027f8a4e98c0 + path: /tmp/arc-v2-2-baseline + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30941837872' + - name: Verify V2.2 and frozen V2 gates + env: + PYTHONPATH: research + run: | + python -m pip install --disable-pip-version-check --quiet pytest + python -m pytest -q research/universal_core/holonomy_v2_2/tests + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py + git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Run all ARC training tasks with complete traces + run: | + mkdir -p /tmp/arc-v2-2-mirror/traces + python - <<'PY' | tee /tmp/arc-v2-2-mirror/console-summary.json + import json + import os + from pathlib import Path + + from research.universal_core.holonomy_v2.engine import HolonomyEngine + from research.universal_core.holonomy_v2.types import SearchConfig + from research.universal_core.holonomy_v2_2.mechanistic_trace import validate_trace + from research.universal_core.holonomy_v2_2.training_eval import run_training_corpus, write_report + from research.universal_core.public_benchmarks.suite import locate_vendor + + output = Path('/tmp/arc-v2-2-mirror') + trace_dir = output / 'traces' + vendor = locate_vendor(Path('/tmp/pinned-public-benchmarks')) + report = run_training_corpus( + vendor, + HolonomyEngine(SearchConfig()), + run_label='arc-v2-2-mirror-' + os.environ['GITHUB_SHA'][:12], + candidate_commit=os.environ['GITHUB_SHA'], + mechanistic_dir=trace_dir, + verify_parity=True, + ) + write_report(report, output / 'report.json') + + trace_paths = sorted( + path for path in trace_dir.glob('*.json') + if path.name != 'diagnosis.json' + ) + if len(trace_paths) != 1000: + raise SystemExit(f'expected 1000 task traces, found {len(trace_paths)}') + traces = {} + for path in trace_paths: + trace = json.loads(path.read_text(encoding='utf-8')) + validate_trace(trace) + traces[path.stem] = trace + if report['mechanistic']['trace_count'] != 1000: + raise SystemExit('aggregate trace count mismatch') + if report['mechanistic']['parity_count'] != 1000: + raise SystemExit('compact/mechanistic parity count mismatch') + + baseline = json.loads( + Path('/tmp/arc-v2-2-baseline/report.json').read_text(encoding='utf-8') + ) + baseline_rows = { + item['task_id']: int(item['metrics']['correct']) + for item in baseline['reports'] + } + current_rows = { + item['task_id']: int(item['metrics']['correct']) + for item in report['reports'] + } + lost_rows = { + task_id: { + 'baseline_correct': correct, + 'current_correct': current_rows.get(task_id, 0), + } + for task_id, correct in baseline_rows.items() + if current_rows.get(task_id, 0) < correct + } + gained_rows = { + task_id: { + 'baseline_correct': baseline_rows.get(task_id, 0), + 'current_correct': correct, + } + for task_id, correct in current_rows.items() + if correct > baseline_rows.get(task_id, 0) + } + expected_gain_ids = [ + '3aa6fb7a', '4c4377d9', '6d0aefbc', '6d75e8bb', '6fa7a44f', + '8be77c9e', 'aabf363d', 'b230c067', 'c9e6f938', 'e7639916', + ] + expected_gains = { + task_id: {'baseline_correct': 0, 'current_correct': 1} + for task_id in expected_gain_ids + } + if lost_rows: + raise SystemExit('lost baseline-correct rows: ' + json.dumps(lost_rows, sort_keys=True)) + if gained_rows != expected_gains: + raise SystemExit( + 'unexpected gained rows: expected ' + + json.dumps(expected_gains, sort_keys=True) + + ' found ' + + json.dumps(gained_rows, sort_keys=True) + ) + + expected_metrics = { + 'rows': 1076, + 'correct': 69, + 'accepted': 69, + 'incorrect_attempts': 0, + 'failed': 0, + } + for key, expected in expected_metrics.items(): + if int(report['metrics'][key]) != expected: + raise SystemExit( + f'metric {key} changed: expected {expected}, found {report["metrics"][key]}' + ) + expected_status = { + 'ACCEPTED': 69, + 'AMBIGUOUS_PROGRAM': 1, + 'GRAMMAR_EXHAUSTED': 1006, + } + if report['status_counts'] != expected_status: + raise SystemExit( + 'status counts changed: expected ' + + json.dumps(expected_status, sort_keys=True) + + ' found ' + + json.dumps(report['status_counts'], sort_keys=True) + ) + + mirror_details = {} + mirror_ids = ('4c4377d9', '6d0aefbc', '6fa7a44f', '8be77c9e', 'c9e6f938') + for task_id in mirror_ids: + trace = traces[task_id] + final = [event for event in trace['events'] if event.get('event') == 'final_decision'][0] + successes = [ + event for event in trace['events'] + if event.get('event') == 'hidden_execution' + and event.get('candidate_kind') == 'mirror_concatenate' + and event.get('outcome') == 'SUCCEEDED' + ] + if trace['terminal_status'] != 'ACCEPTED': + raise SystemExit(f'{task_id} did not transition to ACCEPTED') + if not successes: + raise SystemExit(f'{task_id} has no successful mirror candidate') + if not str(final.get('selected_candidate_id', '')).startswith('mirror_concat:'): + raise SystemExit(f'{task_id} did not select mirror concatenation') + mirror_details[task_id] = { + 'trace_digest': trace['trace_digest'], + 'successful_mirror_candidates': len(successes), + 'selected_candidate_id': final['selected_candidate_id'], + } + + primitive_guards = { + 'aabf363d': 'marker_recolor:', + 'b230c067': 'component_outlier:', + '3aa6fb7a': 'bbox_complete:', + '6d75e8bb': 'bbox_complete:', + 'e7639916': 'bbox_complete:', + } + for task_id, prefix in primitive_guards.items(): + trace = traces[task_id] + final = [event for event in trace['events'] if event.get('event') == 'final_decision'][0] + if trace['terminal_status'] != 'ACCEPTED': + raise SystemExit(f'{task_id} regressed') + if not str(final.get('selected_candidate_id', '')).startswith(prefix): + raise SystemExit(f'{task_id} selected primitive changed') + + ambiguous = traces['60b61512'] + classes = [event for event in ambiguous['events'] if event.get('event') == 'equivalence_class'] + bbox_classes = [ + event for event in classes + if any(str(candidate_id).startswith('bbox_complete:') for candidate_id in event['candidate_ids']) + ] + if ambiguous['terminal_status'] != 'AMBIGUOUS_PROGRAM': + raise SystemExit('60b61512 did not remain ambiguity-safe') + if len(bbox_classes) < 2: + raise SystemExit('60b61512 lost bounding-box disagreement') + + transitions = { + 'schema_version': 1, + 'candidate_commit': os.environ['GITHUB_SHA'], + 'baseline_commit': 'e50965009efb96457fd769d97b42027f8a4e98c0', + 'lost_rows': lost_rows, + 'gained_rows': gained_rows, + 'expected_gains_matched': True, + 'incorrect_attempts': int(report['metrics']['incorrect_attempts']), + } + mechanisms = { + 'schema_version': 1, + 'mirror_tasks': mirror_details, + '60b61512': { + 'terminal_status': ambiguous['terminal_status'], + 'trace_digest': ambiguous['trace_digest'], + 'bbox_equivalence_class_count': len(bbox_classes), + 'safety_action': 'abstain_on_connectivity_disagreement', + }, + } + parity = { + 'schema_version': 1, + 'candidate_commit': os.environ['GITHUB_SHA'], + 'checked_tasks': 1000, + 'mismatches': 0, + 'lost_baseline_rows': lost_rows, + } + for name, data in ( + ('transitions.json', transitions), + ('mechanisms.json', mechanisms), + ('parity.json', parity), + ): + (output / name).write_text( + json.dumps(data, sort_keys=True, indent=2) + '\n', + encoding='utf-8', + ) + + print(json.dumps({ + 'report_digest': report['report_digest'], + 'diagnosis_digest': report['mechanistic']['diagnosis_digest'], + 'evaluated_tasks': report['evaluated_tasks'], + 'trace_count': report['mechanistic']['trace_count'], + 'parity_count': report['mechanistic']['parity_count'], + 'metrics': report['metrics'], + 'status_counts': report['status_counts'], + 'transitions': transitions, + 'mechanisms': mechanisms, + }, sort_keys=True)) + PY + - name: Build checksum manifest after all writers close + run: | + python - <<'PY' + import hashlib + from pathlib import Path + + output = Path('/tmp/arc-v2-2-mirror') + lines = [] + for path in sorted( + item for item in output.rglob('*') + if item.is_file() and item.name != 'SHA256SUMS' + ): + digest = hashlib.sha256(path.read_bytes()).hexdigest() + lines.append(f'{digest} {path.relative_to(output).as_posix()}') + (output / 'SHA256SUMS').write_text('\n'.join(lines) + '\n', encoding='utf-8') + PY + - name: Verify checksum manifest independently + working-directory: /tmp/arc-v2-2-mirror + run: sha256sum -c SHA256SUMS + - name: Publish mirror evidence + uses: actions/upload-artifact@v4 + with: + name: arc-v2-2-mirror-${{ github.sha }} + path: /tmp/arc-v2-2-mirror + if-no-files-found: error + retention-days: 90 + compression-level: 6 + - name: Write run summary + run: | + python - <<'PY' + import json + from pathlib import Path + + root = Path('/tmp/arc-v2-2-mirror') + report = json.loads((root / 'report.json').read_text()) + mechanisms = json.loads((root / 'mechanisms.json').read_text()) + lines = [ + '# ARC V2.2 mirror-concatenation corpus', + '', + f"- Candidate: `{report['identity']['candidate_commit']}`", + f"- Report digest: `{report['report_digest']}`", + f"- Diagnosis digest: `{report['mechanistic']['diagnosis_digest']}`", + f"- Metrics: `{report['metrics']}`", + f"- Status counts: `{report['status_counts']}`", + f"- Mirror gains: `{sorted(mechanisms['mirror_tasks'])}`", + f"- 60b61512 action: `{mechanisms['60b61512']['safety_action']}`", + '- Public ARC evaluation rerun: `no`', + ] + Path('/tmp/mirror-summary.md').write_text('\n'.join(lines) + '\n', encoding='utf-8') + PY + cat /tmp/mirror-summary.md >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/arc-mirror-fixtures.yml b/.github/workflows/arc-mirror-fixtures.yml new file mode 100644 index 00000000..9a8f4c4b --- /dev/null +++ b/.github/workflows/arc-mirror-fixtures.yml @@ -0,0 +1,49 @@ +name: Import ARC Mirror Concatenation Fixtures + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-mirror-fixtures.yml' + +permissions: + contents: write + actions: read + +jobs: + import-fixtures: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - name: Download immutable benchmark corpus + uses: actions/download-artifact@v4 + with: + name: public-benchmarks-pinned-2026-08-04 + path: /tmp/pinned-public-benchmarks + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30929169392' + - name: Copy exact training fixtures + run: | + set -euo pipefail + destination='research/universal_core/holonomy_v2_2/tests/fixtures/mirror_concat' + mkdir -p "$destination" + for task_id in 4c4377d9 6d0aefbc 6fa7a44f 8be77c9e c9e6f938; do + mapfile -t matches < <(find /tmp/pinned-public-benchmarks -type f -path "*/arc_agi_2/data/training/${task_id}.json" | sort) + test "${#matches[@]}" -eq 1 + cp "${matches[0]}" "$destination/${task_id}.json" + python -m json.tool "$destination/${task_id}.json" > /dev/null + done + test "$(find "$destination" -type f -name '*.json' | wc -l)" -eq 5 + - name: Commit immutable fixtures + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/tests/fixtures/mirror_concat + git commit -m 'arc: import mirror concatenation fixtures' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-mirror-red.yml b/.github/workflows/arc-mirror-red.yml new file mode 100644 index 00000000..9ae25118 --- /dev/null +++ b/.github/workflows/arc-mirror-red.yml @@ -0,0 +1,68 @@ +name: ARC Mirror Concatenation RED + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-mirror-red.yml' + +permissions: + contents: write + +jobs: + verify-red: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Require missing mirror module + run: | + set +e + python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_mirror_concat.py > /tmp/mirror-red.log 2>&1 + status=$? + set -e + cat /tmp/mirror-red.log + test "$status" -ne 0 + grep -F "No module named 'research.universal_core.holonomy_v2_2.mirror_concat'" /tmp/mirror-red.log + - name: Commit RED evidence + env: + RED_SOURCE_SHA: ${{ github.sha }} + RED_RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + evidence = { + 'schema_version': 1, + 'red_commit': os.environ['RED_SOURCE_SHA'], + 'red_run_id': os.environ['RED_RUN_ID'], + 'test_command': 'python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_mirror_concat.py', + 'expected_failure': "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.mirror_concat'", + 'failure_confirmed': True, + 'expected_new_exact_tasks': [ + '4c4377d9', '6d0aefbc', '6fa7a44f', '8be77c9e', 'c9e6f938' + ], + 'training_scan': { + 'wrong_completed_hidden_outputs': 0, + 'ambiguous_hidden_outputs': 0, + }, + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + } + path = Path('research/universal_core/holonomy_v2_2/evidence/MIRROR_CONCAT_RED.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/MIRROR_CONCAT_RED.json + git commit -m 'arc: record mirror concatenation RED evidence' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-mirror-wire.yml b/.github/workflows/arc-mirror-wire.yml new file mode 100644 index 00000000..48affe8d --- /dev/null +++ b/.github/workflows/arc-mirror-wire.yml @@ -0,0 +1,177 @@ +name: Wire ARC Mirror Concatenation + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-mirror-wire.yml' + +permissions: + contents: write + +jobs: + wire-and-verify: + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Apply guarded mirror integration + run: | + python - <<'PY' + from pathlib import Path + + grid_path = Path('research/universal_core/holonomy_v2_2/grid_induction.py') + text = grid_path.read_text(encoding='utf-8') + import_anchor = 'from .bbox_complete import apply_bbox_complete\n' + import_value = import_anchor + 'from .mirror_concat import apply_mirror_concat\n' + if text.count(import_anchor) != 1 or 'from .mirror_concat import' in text: + raise SystemExit('mirror import anchor mismatch') + text = text.replace(import_anchor, import_value, 1) + + version_anchor = '_VERSION = "grid-v2.2-8"' + if text.count(version_anchor) != 1: + raise SystemExit('mirror version anchor mismatch') + text = text.replace(version_anchor, '_VERSION = "grid-v2.2-9"', 1) + + apply_anchor = ' if kind == "bounding_box_complete":\n' + apply_value = ( + ' if kind == "mirror_concatenate":\n' + ' return apply_mirror_concat(program, grid)\n' + ) + if text.count(apply_anchor) != 1: + raise SystemExit('mirror apply anchor mismatch') + text = text.replace(apply_anchor, apply_value + apply_anchor, 1) + grid_path.write_text(text, encoding='utf-8') + + runtime_path = Path('research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py') + text = runtime_path.read_text(encoding='utf-8') + runtime_import = 'from .bbox_complete import bbox_complete_programs\n' + runtime_import_value = runtime_import + 'from .mirror_concat import mirror_concat_programs\n' + if text.count(runtime_import) != 1 or 'mirror_concat_programs' in text: + raise SystemExit('mirror runtime import anchor mismatch') + text = text.replace(runtime_import, runtime_import_value, 1) + + derived_anchor = ' bbox_complete, bbox_reason = bbox_complete_programs(demos)\n' + derived_value = derived_anchor + ' mirror_concat, mirror_reason = mirror_concat_programs(demos)\n' + if text.count(derived_anchor) != 1: + raise SystemExit('mirror derivation anchor mismatch') + text = text.replace(derived_anchor, derived_value, 1) + + batch_anchor = ' ("bbox_complete", bbox_complete, bbox_reason),\n' + batch_value = batch_anchor + ' ("mirror_concat", mirror_concat, mirror_reason),\n' + if text.count(batch_anchor) != 1: + raise SystemExit('mirror batch anchor mismatch') + text = text.replace(batch_anchor, batch_value, 1) + runtime_path.write_text(text, encoding='utf-8') + PY + python -m py_compile \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py \ + research/universal_core/holonomy_v2_2/mirror_concat.py + git diff --check + - name: Run focused mirror tests + id: focused + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_mirror_concat.py \ + research/universal_core/holonomy_v2_2/tests/test_bbox_complete.py \ + research/universal_core/holonomy_v2_2/tests/test_component_outlier.py \ + research/universal_core/holonomy_v2_2/tests/test_marker_recolor.py \ + research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py \ + | tee /tmp/focused.log + - name: Run all V2.2 tests + id: cluster + if: always() + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q research/universal_core/holonomy_v2_2/tests | tee /tmp/v2-2.log + - name: Run frozen V2 integration tests + id: frozen + if: always() + continue-on-error: true + env: + PYTHONPATH: research + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py | tee /tmp/frozen.log + - name: Confirm frozen V2 bytes are unchanged + id: byte_gate + if: always() + continue-on-error: true + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Record result and commit only valid state + if: always() + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + FOCUSED_OUTCOME: ${{ steps.focused.outcome }} + CLUSTER_OUTCOME: ${{ steps.cluster.outcome }} + FROZEN_OUTCOME: ${{ steps.frozen.outcome }} + BYTE_OUTCOME: ${{ steps.byte_gate.outcome }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + outcomes = { + 'focused': os.environ['FOCUSED_OUTCOME'], + 'v2_2_cluster': os.environ['CLUSTER_OUTCOME'], + 'frozen_tests': os.environ['FROZEN_OUTCOME'], + 'frozen_byte_gate': os.environ['BYTE_OUTCOME'], + } + passed = all(value == 'success' for value in outcomes.values()) + evidence = { + 'schema_version': 1, + 'workflow_source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': passed, + 'outcomes': outcomes, + 'focused_output': Path('/tmp/focused.log').read_text(encoding='utf-8') if Path('/tmp/focused.log').exists() else '', + 'v2_2_output': Path('/tmp/v2-2.log').read_text(encoding='utf-8') if Path('/tmp/v2-2.log').exists() else '', + 'frozen_output': Path('/tmp/frozen.log').read_text(encoding='utf-8') if Path('/tmp/frozen.log').exists() else '', + 'expected_new_exact_tasks': [ + '4c4377d9', '6d0aefbc', '6fa7a44f', '8be77c9e', 'c9e6f938' + ], + 'expected_training_delta': { + 'correct_rows': 5, + 'accepted_rows': 5, + 'grammar_exhausted_rows': -5, + }, + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + } + name = 'MIRROR_CONCAT_GREEN.json' if passed else 'MIRROR_CONCAT_DIAGNOSTIC.json' + path = Path('research/universal_core/holonomy_v2_2/evidence') / name + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + Path('/tmp/passed').write_text('1' if passed else '0', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + if [ "$(cat /tmp/passed)" = "1" ]; then + git add \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py \ + research/universal_core/holonomy_v2_2/evidence/MIRROR_CONCAT_GREEN.json + git commit -m 'arc: wire mirror concatenation' + else + git restore \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py + git add research/universal_core/holonomy_v2_2/evidence/MIRROR_CONCAT_DIAGNOSTIC.json + git commit -m 'arc: record mirror concatenation diagnostic failure' + fi + git push origin HEAD:universal-core-arc-training-v2-2 + test "$(cat /tmp/passed)" = "1" diff --git a/.github/workflows/arc-palette-fixture-restore.yml b/.github/workflows/arc-palette-fixture-restore.yml new file mode 100644 index 00000000..9bc61c1c --- /dev/null +++ b/.github/workflows/arc-palette-fixture-restore.yml @@ -0,0 +1,81 @@ +name: Restore ARC Palette Fixtures + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-palette-fixture-restore.yml' + +permissions: + contents: write + actions: read + +jobs: + restore-and-verify: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Download immutable benchmark corpus + uses: actions/download-artifact@v4 + with: + name: public-benchmarks-pinned-2026-08-04 + path: /tmp/pinned-public-benchmarks + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30929169392' + - name: Restore exact fixture bytes + run: | + set -euo pipefail + vendor=$(find /tmp/pinned-public-benchmarks -type d -path '*/arc_agi_2/data/training' | head -1) + test -n "$vendor" + destination=research/universal_core/holonomy_v2_2/tests/fixtures/palette_repeat + mkdir -p "$destination" + for task_id in a59b95c0 ac0a08a4 b91ae062 d4b1c2b1; do + cp "$vendor/$task_id.json" "$destination/$task_id.json" + done + printf '%s %s\n' \ + 138f5638ba814834e4f222d6509665ea091c36be6bfc9305fdf41fecd72c41a1 "$destination/a59b95c0.json" \ + 5e68feb78f2f5405be84063528e4beab2b66b8b8cbdac211e305d75e8ac8703d "$destination/ac0a08a4.json" \ + dfb1b3e1b6d275cb1830c1d42320358a1ddc45a925504904ff07fa0dac8a2a53 "$destination/b91ae062.json" \ + 46b0f9fec811ac831b916e44ef8269e45e2b054d59354a4a8f8a1a299716d753 "$destination/d4b1c2b1.json" \ + | sha256sum -c - + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Verify fixture regression + run: python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_palette_repeat_fixtures.py | tee /tmp/fixture-green.log + - name: Commit exact fixtures and evidence + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + evidence = { + 'schema_version': 1, + 'workflow_source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'fixture_count': 4, + 'fixture_integrity': True, + 'source_artifact_run': 30929169392, + 'test_output': Path('/tmp/fixture-green.log').read_text(encoding='utf-8'), + 'public_arc_evaluation_rerun': False, + } + path = Path('research/universal_core/holonomy_v2_2/evidence/PALETTE_FIXTURE_INTEGRITY_GREEN.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add \ + research/universal_core/holonomy_v2_2/tests/fixtures/palette_repeat \ + research/universal_core/holonomy_v2_2/evidence/PALETTE_FIXTURE_INTEGRITY_GREEN.json + git commit -m 'arc: restore pinned palette repeat fixtures' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-palette-repeat-corpus.yml b/.github/workflows/arc-palette-repeat-corpus.yml new file mode 100644 index 00000000..d16eab99 --- /dev/null +++ b/.github/workflows/arc-palette-repeat-corpus.yml @@ -0,0 +1,331 @@ +name: ARC Palette Repeat Mechanistic Corpus + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-palette-repeat-corpus.yml' + +permissions: + contents: read + actions: read + +jobs: + training-corpus: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Download immutable benchmark corpus + uses: actions/download-artifact@v4 + with: + name: public-benchmarks-pinned-2026-08-04 + path: /tmp/pinned-public-benchmarks + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30929169392' + - name: Download immutable V2.2 baseline report + uses: actions/download-artifact@v4 + with: + name: arc-v2-2-training-e50965009efb96457fd769d97b42027f8a4e98c0 + path: /tmp/arc-v2-2-baseline + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30941837872' + - name: Verify V2.2 and frozen V2 gates + env: + PYTHONPATH: research + run: | + python -m pip install --disable-pip-version-check --quiet pytest + python -m pytest -q research/universal_core/holonomy_v2_2/tests + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py + git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Run all ARC training tasks with complete traces + run: | + mkdir -p /tmp/arc-v2-2-palette/traces + python - <<'PY' | tee /tmp/arc-v2-2-palette/console-summary.json + import json + import os + from pathlib import Path + + from research.universal_core.holonomy_v2.engine import HolonomyEngine + from research.universal_core.holonomy_v2.types import SearchConfig + from research.universal_core.holonomy_v2_2.mechanistic_trace import validate_trace + from research.universal_core.holonomy_v2_2.training_eval import run_training_corpus, write_report + from research.universal_core.public_benchmarks.suite import locate_vendor + + output = Path('/tmp/arc-v2-2-palette') + trace_dir = output / 'traces' + vendor = locate_vendor(Path('/tmp/pinned-public-benchmarks')) + report = run_training_corpus( + vendor, + HolonomyEngine(SearchConfig()), + run_label='arc-v2-2-palette-' + os.environ['GITHUB_SHA'][:12], + candidate_commit=os.environ['GITHUB_SHA'], + mechanistic_dir=trace_dir, + verify_parity=True, + ) + write_report(report, output / 'report.json') + + trace_paths = sorted( + path for path in trace_dir.glob('*.json') + if path.name != 'diagnosis.json' + ) + if len(trace_paths) != 1000: + raise SystemExit(f'expected 1000 task traces, found {len(trace_paths)}') + traces = {} + for path in trace_paths: + trace = json.loads(path.read_text(encoding='utf-8')) + validate_trace(trace) + traces[path.stem] = trace + if report['mechanistic']['trace_count'] != 1000: + raise SystemExit('aggregate trace count mismatch') + if report['mechanistic']['parity_count'] != 1000: + raise SystemExit('compact/mechanistic parity count mismatch') + + baseline = json.loads( + Path('/tmp/arc-v2-2-baseline/report.json').read_text(encoding='utf-8') + ) + baseline_rows = { + item['task_id']: int(item['metrics']['correct']) + for item in baseline['reports'] + } + current_rows = { + item['task_id']: int(item['metrics']['correct']) + for item in report['reports'] + } + lost_rows = { + task_id: { + 'baseline_correct': correct, + 'current_correct': current_rows.get(task_id, 0), + } + for task_id, correct in baseline_rows.items() + if current_rows.get(task_id, 0) < correct + } + gained_rows = { + task_id: { + 'baseline_correct': baseline_rows.get(task_id, 0), + 'current_correct': correct, + } + for task_id, correct in current_rows.items() + if correct > baseline_rows.get(task_id, 0) + } + expected_gain_ids = [ + '0c786b71', '3aa6fb7a', '3af2c5a8', '46442a0e', '4c4377d9', + '62c24649', '67e8384a', '6d0aefbc', '6d75e8bb', '6fa7a44f', + '7953d61e', '7fe24cdd', '833dafe3', '8be77c9e', 'a59b95c0', + 'aabf363d', 'ac0a08a4', 'b230c067', 'b91ae062', 'c9e6f938', + 'd4b1c2b1', 'e7639916', 'ed98d772', + ] + expected_gains = { + task_id: {'baseline_correct': 0, 'current_correct': 1} + for task_id in expected_gain_ids + } + if lost_rows: + raise SystemExit('lost baseline-correct rows: ' + json.dumps(lost_rows, sort_keys=True)) + if gained_rows != expected_gains: + raise SystemExit( + 'unexpected gained rows: expected ' + + json.dumps(expected_gains, sort_keys=True) + + ' found ' + + json.dumps(gained_rows, sort_keys=True) + ) + + expected_metrics = { + 'rows': 1076, + 'correct': 82, + 'accepted': 82, + 'incorrect_attempts': 0, + 'failed': 0, + } + for key, expected in expected_metrics.items(): + if int(report['metrics'][key]) != expected: + raise SystemExit( + f'metric {key} changed: expected {expected}, found {report["metrics"][key]}' + ) + expected_status = { + 'ACCEPTED': 82, + 'AMBIGUOUS_PROGRAM': 1, + 'GRAMMAR_EXHAUSTED': 993, + } + if report['status_counts'] != expected_status: + raise SystemExit( + 'status counts changed: expected ' + + json.dumps(expected_status, sort_keys=True) + + ' found ' + + json.dumps(report['status_counts'], sort_keys=True) + ) + + palette_ids = ('a59b95c0', 'ac0a08a4', 'b91ae062', 'd4b1c2b1') + palette_details = {} + for task_id in palette_ids: + trace = traces[task_id] + final = [event for event in trace['events'] if event.get('event') == 'final_decision'][0] + successes = [ + event for event in trace['events'] + if event.get('event') == 'hidden_execution' + and event.get('candidate_kind') == 'palette_cardinality_repeat' + and event.get('outcome') == 'SUCCEEDED' + ] + if trace['terminal_status'] != 'ACCEPTED': + raise SystemExit(f'{task_id} did not transition to ACCEPTED') + if not successes: + raise SystemExit(f'{task_id} has no successful palette-repeat candidate') + if not str(final.get('selected_candidate_id', '')).startswith('palette_repeat:'): + raise SystemExit(f'{task_id} did not select palette repeat') + palette_details[task_id] = { + 'trace_digest': trace['trace_digest'], + 'successful_palette_candidates': len(successes), + 'selected_candidate_id': final['selected_candidate_id'], + } + + primitive_guards = { + 'aabf363d': 'marker_recolor:', + 'b230c067': 'component_outlier:', + '3aa6fb7a': 'bbox_complete:', + '6d75e8bb': 'bbox_complete:', + 'e7639916': 'bbox_complete:', + '4c4377d9': 'mirror_concat:', + '6d0aefbc': 'mirror_concat:', + '6fa7a44f': 'mirror_concat:', + '8be77c9e': 'mirror_concat:', + 'c9e6f938': 'mirror_concat:', + '0c786b71': 'quadrant_mosaic:', + '3af2c5a8': 'quadrant_mosaic:', + '46442a0e': 'quadrant_mosaic:', + '62c24649': 'quadrant_mosaic:', + '67e8384a': 'quadrant_mosaic:', + '7953d61e': 'quadrant_mosaic:', + '7fe24cdd': 'quadrant_mosaic:', + '833dafe3': 'quadrant_mosaic:', + 'ed98d772': 'quadrant_mosaic:', + } + for task_id, prefix in primitive_guards.items(): + trace = traces[task_id] + final = [event for event in trace['events'] if event.get('event') == 'final_decision'][0] + if trace['terminal_status'] != 'ACCEPTED': + raise SystemExit(f'{task_id} regressed') + if not str(final.get('selected_candidate_id', '')).startswith(prefix): + raise SystemExit(f'{task_id} selected primitive changed') + + ambiguous = traces['60b61512'] + classes = [event for event in ambiguous['events'] if event.get('event') == 'equivalence_class'] + bbox_classes = [ + event for event in classes + if any(str(candidate_id).startswith('bbox_complete:') for candidate_id in event['candidate_ids']) + ] + if ambiguous['terminal_status'] != 'AMBIGUOUS_PROGRAM': + raise SystemExit('60b61512 did not remain ambiguity-safe') + if len(bbox_classes) < 2: + raise SystemExit('60b61512 lost bounding-box disagreement') + + transitions = { + 'schema_version': 1, + 'candidate_commit': os.environ['GITHUB_SHA'], + 'baseline_commit': 'e50965009efb96457fd769d97b42027f8a4e98c0', + 'lost_rows': lost_rows, + 'gained_rows': gained_rows, + 'expected_gains_matched': True, + 'incorrect_attempts': int(report['metrics']['incorrect_attempts']), + } + mechanisms = { + 'schema_version': 1, + 'palette_tasks': palette_details, + 'ownership_boundary': 'NO_FACTOR_VARIATION', + '60b61512': { + 'terminal_status': ambiguous['terminal_status'], + 'trace_digest': ambiguous['trace_digest'], + 'bbox_equivalence_class_count': len(bbox_classes), + 'safety_action': 'abstain_on_connectivity_disagreement', + }, + } + parity = { + 'schema_version': 1, + 'candidate_commit': os.environ['GITHUB_SHA'], + 'checked_tasks': 1000, + 'mismatches': 0, + 'lost_baseline_rows': lost_rows, + } + for name, data in ( + ('transitions.json', transitions), + ('mechanisms.json', mechanisms), + ('parity.json', parity), + ): + (output / name).write_text( + json.dumps(data, sort_keys=True, indent=2) + '\n', + encoding='utf-8', + ) + + print(json.dumps({ + 'report_digest': report['report_digest'], + 'diagnosis_digest': report['mechanistic']['diagnosis_digest'], + 'evaluated_tasks': report['evaluated_tasks'], + 'trace_count': report['mechanistic']['trace_count'], + 'parity_count': report['mechanistic']['parity_count'], + 'metrics': report['metrics'], + 'status_counts': report['status_counts'], + 'transitions': transitions, + 'mechanisms': mechanisms, + }, sort_keys=True)) + PY + - name: Build checksum manifest after all writers close + run: | + python - <<'PY' + import hashlib + from pathlib import Path + + output = Path('/tmp/arc-v2-2-palette') + lines = [] + for path in sorted( + item for item in output.rglob('*') + if item.is_file() and item.name != 'SHA256SUMS' + ): + digest = hashlib.sha256(path.read_bytes()).hexdigest() + lines.append(f'{digest} {path.relative_to(output).as_posix()}') + (output / 'SHA256SUMS').write_text('\n'.join(lines) + '\n', encoding='utf-8') + PY + - name: Verify checksum manifest independently + working-directory: /tmp/arc-v2-2-palette + run: sha256sum -c SHA256SUMS + - name: Publish palette-repeat evidence + uses: actions/upload-artifact@v4 + with: + name: arc-v2-2-palette-repeat-${{ github.sha }} + path: /tmp/arc-v2-2-palette + if-no-files-found: error + retention-days: 90 + compression-level: 6 + - name: Write run summary + run: | + python - <<'PY' + import json + from pathlib import Path + + root = Path('/tmp/arc-v2-2-palette') + report = json.loads((root / 'report.json').read_text()) + mechanisms = json.loads((root / 'mechanisms.json').read_text()) + lines = [ + '# ARC V2.2 palette-cardinality repeat corpus', + '', + f"- Candidate: `{report['identity']['candidate_commit']}`", + f"- Report digest: `{report['report_digest']}`", + f"- Diagnosis digest: `{report['mechanistic']['diagnosis_digest']}`", + f"- Metrics: `{report['metrics']}`", + f"- Status counts: `{report['status_counts']}`", + f"- Palette gains: `{sorted(mechanisms['palette_tasks'])}`", + f"- Ownership boundary: `{mechanisms['ownership_boundary']}`", + f"- 60b61512 action: `{mechanisms['60b61512']['safety_action']}`", + '- Public ARC evaluation rerun: `no`', + ] + Path('/tmp/palette-summary.md').write_text('\n'.join(lines) + '\n', encoding='utf-8') + PY + cat /tmp/palette-summary.md >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/arc-palette-repeat-module-green.yml b/.github/workflows/arc-palette-repeat-module-green.yml new file mode 100644 index 00000000..25dc8656 --- /dev/null +++ b/.github/workflows/arc-palette-repeat-module-green.yml @@ -0,0 +1,66 @@ +name: ARC Palette Repeat Module GREEN + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-palette-repeat-module-green.yml' + +permissions: + contents: write + +jobs: + verify-module: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Run isolated palette tests + run: | + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_palette_repeat_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_palette_repeat.py::test_apply_scales_by_total_palette_size \ + research/universal_core/holonomy_v2_2/tests/test_palette_repeat.py::test_apply_tiles_by_total_palette_size \ + research/universal_core/holonomy_v2_2/tests/test_palette_repeat.py::test_apply_scales_by_non_background_palette_size \ + research/universal_core/holonomy_v2_2/tests/test_palette_repeat.py::test_program_derivation_requires_demonstrated_factor_variation \ + research/universal_core/holonomy_v2_2/tests/test_palette_repeat.py::test_program_derivation_defers_constant_factor_to_fixed_grammar \ + research/universal_core/holonomy_v2_2/tests/test_palette_repeat.py::test_apply_rejects_output_beyond_arc_bounds \ + | tee /tmp/module-green.log + - name: Confirm frozen bytes + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Commit module evidence + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + evidence = { + 'schema_version': 1, + 'source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': True, + 'test_count': 7, + 'test_output': Path('/tmp/module-green.log').read_text(encoding='utf-8'), + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'frozen_byte_gate': True, + 'public_arc_evaluation_rerun': False, + } + path = Path('research/universal_core/holonomy_v2_2/evidence/PALETTE_REPEAT_MODULE_GREEN.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/PALETTE_REPEAT_MODULE_GREEN.json + git commit -m 'arc: record palette repeat module GREEN' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-palette-repeat-red-v2.yml b/.github/workflows/arc-palette-repeat-red-v2.yml new file mode 100644 index 00000000..22c55189 --- /dev/null +++ b/.github/workflows/arc-palette-repeat-red-v2.yml @@ -0,0 +1,65 @@ +name: ARC Palette Repeat RED V2 + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-palette-repeat-red-v2.yml' + +permissions: + contents: write + +jobs: + verify-red: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - run: python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_palette_repeat_fixtures.py + - name: Require missing palette module + run: | + set +e + python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_palette_repeat.py > /tmp/palette-red.log 2>&1 + status=$? + set -e + cat /tmp/palette-red.log + test "$status" -ne 0 + grep -F "No module named 'research.universal_core.holonomy_v2_2.palette_repeat'" /tmp/palette-red.log + - name: Commit RED evidence + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + evidence = { + 'schema_version': 2, + 'source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'failure_confirmed': True, + 'fixture_integrity': True, + 'expected_failure': "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.palette_repeat'", + 'expected_new_exact_tasks': ['a59b95c0', 'ac0a08a4', 'b91ae062', 'd4b1c2b1'], + 'test_command': 'python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_palette_repeat.py', + 'captured_log': Path('/tmp/palette-red.log').read_text(encoding='utf-8'), + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'public_arc_evaluation_rerun': False, + } + path = Path('research/universal_core/holonomy_v2_2/evidence/PALETTE_REPEAT_RED.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/PALETTE_REPEAT_RED.json + git commit -m 'arc: record palette repeat RED evidence' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-palette-repeat-red.yml b/.github/workflows/arc-palette-repeat-red.yml new file mode 100644 index 00000000..18fda26f --- /dev/null +++ b/.github/workflows/arc-palette-repeat-red.yml @@ -0,0 +1,73 @@ +name: ARC Palette Repeat RED + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-palette-repeat-red.yml' + +permissions: + contents: write + +jobs: + verify-red: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Verify pinned fixture integrity + run: python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_palette_repeat_fixtures.py + - name: Require missing palette module + run: | + set +e + python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_palette_repeat.py > /tmp/palette-red.log 2>&1 + status=$? + set -e + cat /tmp/palette-red.log + test "$status" -ne 0 + grep -F "No module named 'research.universal_core.holonomy_v2_2.palette_repeat'" /tmp/palette-red.log + - name: Commit RED evidence + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + evidence = { + 'schema_version': 1, + 'source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'failure_confirmed': True, + 'expected_failure': "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.palette_repeat'", + 'fixture_count': 4, + 'fixture_integrity': True, + 'expected_new_exact_tasks': ['a59b95c0', 'ac0a08a4', 'b91ae062', 'd4b1c2b1'], + 'training_scan': { + 'compatible_tasks': 4, + 'wrong_completed_hidden_outputs': 0, + 'ambiguous_hidden_outputs': 0, + 'compatible_accepted_tasks': 0, + }, + 'test_command': 'python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_palette_repeat.py', + 'captured_log': Path('/tmp/palette-red.log').read_text(encoding='utf-8'), + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'public_arc_evaluation_rerun': False, + } + path = Path('research/universal_core/holonomy_v2_2/evidence/PALETTE_REPEAT_RED.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/PALETTE_REPEAT_RED.json + git commit -m 'arc: record palette repeat RED evidence' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-palette-repeat-wire.yml b/.github/workflows/arc-palette-repeat-wire.yml new file mode 100644 index 00000000..a8da02f8 --- /dev/null +++ b/.github/workflows/arc-palette-repeat-wire.yml @@ -0,0 +1,191 @@ +name: Wire ARC Palette Repeat + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-palette-repeat-wire.yml' + +permissions: + contents: write + +jobs: + wire-and-verify: + runs-on: ubuntu-latest + timeout-minutes: 50 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Apply guarded palette integration + run: | + python - <<'PY' + from pathlib import Path + + grid_path = Path('research/universal_core/holonomy_v2_2/grid_induction.py') + text = grid_path.read_text(encoding='utf-8') + import_anchor = 'from .quadrant_mosaic import apply_quadrant_mosaic\n' + if text.count(import_anchor) != 1 or 'from .palette_repeat import' in text: + raise SystemExit('palette import anchor mismatch') + text = text.replace( + import_anchor, + import_anchor + 'from .palette_repeat import apply_palette_repeat\n', + 1, + ) + if text.count('_VERSION = "grid-v2.2-10"') != 1: + raise SystemExit('palette version anchor mismatch') + text = text.replace('_VERSION = "grid-v2.2-10"', '_VERSION = "grid-v2.2-11"', 1) + apply_anchor = ' if kind == "dihedral_quadrant_mosaic":\n' + if text.count(apply_anchor) != 1: + raise SystemExit('palette apply anchor mismatch') + text = text.replace( + apply_anchor, + ' if kind == "palette_cardinality_repeat":\n' + ' return apply_palette_repeat(program, grid)\n' + + apply_anchor, + 1, + ) + grid_path.write_text(text, encoding='utf-8') + + runtime_path = Path('research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py') + text = runtime_path.read_text(encoding='utf-8') + import_anchor = 'from .quadrant_mosaic import quadrant_mosaic_programs\n' + if text.count(import_anchor) != 1 or 'palette_repeat_programs' in text: + raise SystemExit('palette runtime import anchor mismatch') + text = text.replace( + import_anchor, + import_anchor + 'from .palette_repeat import palette_repeat_programs\n', + 1, + ) + derive_anchor = ' quadrant_mosaic, quadrant_reason = quadrant_mosaic_programs(demos)\n' + if text.count(derive_anchor) != 1: + raise SystemExit('palette derivation anchor mismatch') + text = text.replace( + derive_anchor, + derive_anchor + ' palette_repeat, palette_reason = palette_repeat_programs(demos)\n', + 1, + ) + batch_anchor = ' ("quadrant_mosaic", quadrant_mosaic, quadrant_reason),\n' + if text.count(batch_anchor) != 1: + raise SystemExit('palette batch anchor mismatch') + text = text.replace( + batch_anchor, + batch_anchor + ' ("palette_repeat", palette_repeat, palette_reason),\n', + 1, + ) + runtime_path.write_text(text, encoding='utf-8') + PY + python -m py_compile \ + research/universal_core/holonomy_v2_2/palette_repeat.py \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py + git diff --check + - name: Run focused palette tests + id: focused + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_palette_repeat_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_palette_repeat.py \ + research/universal_core/holonomy_v2_2/tests/test_grid_induction.py::test_cell_scale_and_whole_grid_tile_are_distinct \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py \ + research/universal_core/holonomy_v2_2/tests/test_mirror_concat.py \ + research/universal_core/holonomy_v2_2/tests/test_bbox_complete.py \ + research/universal_core/holonomy_v2_2/tests/test_component_outlier.py \ + research/universal_core/holonomy_v2_2/tests/test_marker_recolor.py \ + research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py \ + | tee /tmp/focused.log + - name: Run all V2.2 tests + id: cluster + if: always() + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q research/universal_core/holonomy_v2_2/tests | tee /tmp/v2-2.log + - name: Run frozen V2 integration tests + id: frozen + if: always() + continue-on-error: true + env: + PYTHONPATH: research + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py | tee /tmp/frozen.log + - name: Confirm frozen V2 bytes are unchanged + id: byte_gate + if: always() + continue-on-error: true + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Record result and commit only valid state + if: always() + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + FOCUSED_OUTCOME: ${{ steps.focused.outcome }} + CLUSTER_OUTCOME: ${{ steps.cluster.outcome }} + FROZEN_OUTCOME: ${{ steps.frozen.outcome }} + BYTE_OUTCOME: ${{ steps.byte_gate.outcome }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + outcomes = { + 'focused': os.environ['FOCUSED_OUTCOME'], + 'v2_2_cluster': os.environ['CLUSTER_OUTCOME'], + 'frozen_tests': os.environ['FROZEN_OUTCOME'], + 'frozen_byte_gate': os.environ['BYTE_OUTCOME'], + } + passed = all(value == 'success' for value in outcomes.values()) + evidence = { + 'schema_version': 1, + 'workflow_source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': passed, + 'outcomes': outcomes, + 'focused_output': Path('/tmp/focused.log').read_text(encoding='utf-8') if Path('/tmp/focused.log').exists() else '', + 'v2_2_output': Path('/tmp/v2-2.log').read_text(encoding='utf-8') if Path('/tmp/v2-2.log').exists() else '', + 'frozen_output': Path('/tmp/frozen.log').read_text(encoding='utf-8') if Path('/tmp/frozen.log').exists() else '', + 'ownership_boundary': 'NO_FACTOR_VARIATION', + 'expected_new_exact_tasks': ['a59b95c0', 'ac0a08a4', 'b91ae062', 'd4b1c2b1'], + 'expected_training_delta': { + 'correct_rows': 4, + 'accepted_rows': 4, + 'grammar_exhausted_rows': -4, + }, + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'public_arc_evaluation_rerun': False, + } + name = 'PALETTE_REPEAT_GREEN.json' if passed else 'PALETTE_REPEAT_DIAGNOSTIC.json' + path = Path('research/universal_core/holonomy_v2_2/evidence') / name + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + Path('/tmp/passed').write_text('1' if passed else '0', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + if [ "$(cat /tmp/passed)" = "1" ]; then + git add \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py \ + research/universal_core/holonomy_v2_2/evidence/PALETTE_REPEAT_GREEN.json + git commit -m 'arc: wire palette-cardinality repeat' + else + git restore \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py + git add research/universal_core/holonomy_v2_2/evidence/PALETTE_REPEAT_DIAGNOSTIC.json + git commit -m 'arc: record palette repeat diagnostic failure' + fi + git push origin HEAD:universal-core-arc-training-v2-2 + test "$(cat /tmp/passed)" = "1" diff --git a/.github/workflows/arc-panel-wire.yml b/.github/workflows/arc-panel-wire.yml new file mode 100644 index 00000000..b1280d09 --- /dev/null +++ b/.github/workflows/arc-panel-wire.yml @@ -0,0 +1,65 @@ +name: Wire ARC Panel Candidates + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-panel-wire.yml' + +permissions: + contents: write + +jobs: + wire: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - name: Apply guarded integration patch + run: | + python - <<'PY' + from pathlib import Path + + path = Path('research/universal_core/holonomy_v2_2/grid_induction.py') + text = path.read_text(encoding='utf-8') + + old_import = 'from research.universal_core.holonomy_v2.blind_adapter import run_public_task_v2\n' + new_import = old_import + '\nfrom .panel_combine import apply_panel, panel_candidates\n' + if text.count(old_import) != 1 or 'from .panel_combine import' in text: + raise SystemExit('panel import anchor mismatch') + text = text.replace(old_import, new_import, 1) + + old_version = '_VERSION = "grid-v2.2-2"' + if text.count(old_version) != 1: + raise SystemExit('version anchor mismatch') + text = text.replace(old_version, '_VERSION = "grid-v2.2-3"', 1) + + apply_anchor = ' if kind == "color_map":\n' + apply_insert = ( + ' if kind == "panel_boolean_combine":\n' + ' return apply_panel(program, grid)\n' + ) + if text.count(apply_anchor) != 1: + raise SystemExit('apply anchor mismatch') + text = text.replace(apply_anchor, apply_insert + apply_anchor, 1) + + candidate_anchor = ' by_digest: dict[str, Program] = {}\n' + candidate_insert = ' candidates.extend(panel_candidates(demos))\n\n' + if text.count(candidate_anchor) != 1: + raise SystemExit('candidate anchor mismatch') + text = text.replace(candidate_anchor, candidate_insert + candidate_anchor, 1) + + path.write_text(text, encoding='utf-8') + PY + git diff --check + git diff -- research/universal_core/holonomy_v2_2/grid_induction.py + - name: Commit integration + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/grid_induction.py + git commit -m 'arc: wire panel boolean candidates' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-quadrant-fixture-green.yml b/.github/workflows/arc-quadrant-fixture-green.yml new file mode 100644 index 00000000..8b568875 --- /dev/null +++ b/.github/workflows/arc-quadrant-fixture-green.yml @@ -0,0 +1,65 @@ +name: ARC Quadrant Fixture Integrity GREEN + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-quadrant-fixture-green.yml' + +permissions: + contents: write + +jobs: + verify-green: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Verify pinned fixture bytes and isolated module + run: | + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py::test_apply_standard_reflected_quadrant_mosaic \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py::test_apply_rejects_output_beyond_arc_bounds \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py::test_program_derivation_returns_one_canonical_program \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py::test_program_derivation_reports_non_double_output_shape \ + | tee /tmp/fixture-green.log + - name: Confirm frozen bytes + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Commit GREEN evidence + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + evidence = { + 'schema_version': 1, + 'source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': True, + 'fixture_count': 9, + 'test_count': 5, + 'test_output': Path('/tmp/fixture-green.log').read_text(encoding='utf-8'), + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'frozen_byte_gate': True, + 'public_arc_evaluation_rerun': False, + } + path = Path('research/universal_core/holonomy_v2_2/evidence/QUADRANT_FIXTURE_INTEGRITY_GREEN.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/QUADRANT_FIXTURE_INTEGRITY_GREEN.json + git commit -m 'arc: record quadrant fixture integrity GREEN' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-quadrant-fixture-red.yml b/.github/workflows/arc-quadrant-fixture-red.yml new file mode 100644 index 00000000..749acb37 --- /dev/null +++ b/.github/workflows/arc-quadrant-fixture-red.yml @@ -0,0 +1,61 @@ +name: ARC Quadrant Fixture Integrity RED + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-quadrant-fixture-red.yml' + +permissions: + contents: write + +jobs: + verify-red: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Require fixture digest mismatch + run: | + set +e + python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic_fixtures.py > /tmp/fixture-red.log 2>&1 + status=$? + set -e + cat /tmp/fixture-red.log + test "$status" -ne 0 + grep -F "test_quadrant_mosaic_fixtures_match_pinned_training_payloads" /tmp/fixture-red.log + - name: Commit fixture RED evidence + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + evidence = { + 'schema_version': 1, + 'source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'failure_confirmed': True, + 'root_cause': 'fixture files were manually transcribed instead of copied byte-for-byte from the pinned ARC training corpus', + 'test_command': 'python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic_fixtures.py', + 'captured_log': Path('/tmp/fixture-red.log').read_text(encoding='utf-8'), + 'public_arc_evaluation_rerun': False, + } + path = Path('research/universal_core/holonomy_v2_2/evidence/QUADRANT_FIXTURE_INTEGRITY_RED.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/QUADRANT_FIXTURE_INTEGRITY_RED.json + git commit -m 'arc: record quadrant fixture integrity RED' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-quadrant-mosaic-corpus.yml b/.github/workflows/arc-quadrant-mosaic-corpus.yml new file mode 100644 index 00000000..de569027 --- /dev/null +++ b/.github/workflows/arc-quadrant-mosaic-corpus.yml @@ -0,0 +1,324 @@ +name: ARC Quadrant Mosaic Mechanistic Corpus + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-quadrant-mosaic-corpus.yml' + +permissions: + contents: read + actions: read + +jobs: + training-corpus: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Download immutable benchmark corpus + uses: actions/download-artifact@v4 + with: + name: public-benchmarks-pinned-2026-08-04 + path: /tmp/pinned-public-benchmarks + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30929169392' + - name: Download immutable V2.2 baseline report + uses: actions/download-artifact@v4 + with: + name: arc-v2-2-training-e50965009efb96457fd769d97b42027f8a4e98c0 + path: /tmp/arc-v2-2-baseline + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30941837872' + - name: Verify V2.2 and frozen V2 gates + env: + PYTHONPATH: research + run: | + python -m pip install --disable-pip-version-check --quiet pytest + python -m pytest -q research/universal_core/holonomy_v2_2/tests + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py + git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Run all ARC training tasks with complete traces + run: | + mkdir -p /tmp/arc-v2-2-quadrant/traces + python - <<'PY' | tee /tmp/arc-v2-2-quadrant/console-summary.json + import json + import os + from pathlib import Path + + from research.universal_core.holonomy_v2.engine import HolonomyEngine + from research.universal_core.holonomy_v2.types import SearchConfig + from research.universal_core.holonomy_v2_2.mechanistic_trace import validate_trace + from research.universal_core.holonomy_v2_2.training_eval import run_training_corpus, write_report + from research.universal_core.public_benchmarks.suite import locate_vendor + + output = Path('/tmp/arc-v2-2-quadrant') + trace_dir = output / 'traces' + vendor = locate_vendor(Path('/tmp/pinned-public-benchmarks')) + report = run_training_corpus( + vendor, + HolonomyEngine(SearchConfig()), + run_label='arc-v2-2-quadrant-' + os.environ['GITHUB_SHA'][:12], + candidate_commit=os.environ['GITHUB_SHA'], + mechanistic_dir=trace_dir, + verify_parity=True, + ) + write_report(report, output / 'report.json') + + trace_paths = sorted( + path for path in trace_dir.glob('*.json') + if path.name != 'diagnosis.json' + ) + if len(trace_paths) != 1000: + raise SystemExit(f'expected 1000 task traces, found {len(trace_paths)}') + traces = {} + for path in trace_paths: + trace = json.loads(path.read_text(encoding='utf-8')) + validate_trace(trace) + traces[path.stem] = trace + if report['mechanistic']['trace_count'] != 1000: + raise SystemExit('aggregate trace count mismatch') + if report['mechanistic']['parity_count'] != 1000: + raise SystemExit('compact/mechanistic parity count mismatch') + + baseline = json.loads( + Path('/tmp/arc-v2-2-baseline/report.json').read_text(encoding='utf-8') + ) + baseline_rows = { + item['task_id']: int(item['metrics']['correct']) + for item in baseline['reports'] + } + current_rows = { + item['task_id']: int(item['metrics']['correct']) + for item in report['reports'] + } + lost_rows = { + task_id: { + 'baseline_correct': correct, + 'current_correct': current_rows.get(task_id, 0), + } + for task_id, correct in baseline_rows.items() + if current_rows.get(task_id, 0) < correct + } + gained_rows = { + task_id: { + 'baseline_correct': baseline_rows.get(task_id, 0), + 'current_correct': correct, + } + for task_id, correct in current_rows.items() + if correct > baseline_rows.get(task_id, 0) + } + expected_gain_ids = [ + '0c786b71', '3aa6fb7a', '3af2c5a8', '46442a0e', '4c4377d9', + '62c24649', '67e8384a', '6d0aefbc', '6d75e8bb', '6fa7a44f', + '7953d61e', '7fe24cdd', '833dafe3', '8be77c9e', 'aabf363d', + 'b230c067', 'c9e6f938', 'e7639916', 'ed98d772', + ] + expected_gains = { + task_id: {'baseline_correct': 0, 'current_correct': 1} + for task_id in expected_gain_ids + } + if lost_rows: + raise SystemExit('lost baseline-correct rows: ' + json.dumps(lost_rows, sort_keys=True)) + if gained_rows != expected_gains: + raise SystemExit( + 'unexpected gained rows: expected ' + + json.dumps(expected_gains, sort_keys=True) + + ' found ' + + json.dumps(gained_rows, sort_keys=True) + ) + + expected_metrics = { + 'rows': 1076, + 'correct': 78, + 'accepted': 78, + 'incorrect_attempts': 0, + 'failed': 0, + } + for key, expected in expected_metrics.items(): + if int(report['metrics'][key]) != expected: + raise SystemExit( + f'metric {key} changed: expected {expected}, found {report["metrics"][key]}' + ) + expected_status = { + 'ACCEPTED': 78, + 'AMBIGUOUS_PROGRAM': 1, + 'GRAMMAR_EXHAUSTED': 997, + } + if report['status_counts'] != expected_status: + raise SystemExit( + 'status counts changed: expected ' + + json.dumps(expected_status, sort_keys=True) + + ' found ' + + json.dumps(report['status_counts'], sort_keys=True) + ) + + quadrant_ids = ( + '0c786b71', '3af2c5a8', '46442a0e', '62c24649', '67e8384a', + '7953d61e', '7fe24cdd', '833dafe3', 'ed98d772', + ) + quadrant_details = {} + for task_id in quadrant_ids: + trace = traces[task_id] + final = [event for event in trace['events'] if event.get('event') == 'final_decision'][0] + successes = [ + event for event in trace['events'] + if event.get('event') == 'hidden_execution' + and event.get('candidate_kind') == 'dihedral_quadrant_mosaic' + and event.get('outcome') == 'SUCCEEDED' + ] + if trace['terminal_status'] != 'ACCEPTED': + raise SystemExit(f'{task_id} did not transition to ACCEPTED') + if not successes: + raise SystemExit(f'{task_id} has no successful quadrant candidate') + if not str(final.get('selected_candidate_id', '')).startswith('quadrant_mosaic:'): + raise SystemExit(f'{task_id} did not select quadrant mosaic') + quadrant_details[task_id] = { + 'trace_digest': trace['trace_digest'], + 'successful_quadrant_candidates': len(successes), + 'selected_candidate_id': final['selected_candidate_id'], + } + + primitive_guards = { + 'aabf363d': 'marker_recolor:', + 'b230c067': 'component_outlier:', + '3aa6fb7a': 'bbox_complete:', + '6d75e8bb': 'bbox_complete:', + 'e7639916': 'bbox_complete:', + '4c4377d9': 'mirror_concat:', + '6d0aefbc': 'mirror_concat:', + '6fa7a44f': 'mirror_concat:', + '8be77c9e': 'mirror_concat:', + 'c9e6f938': 'mirror_concat:', + } + for task_id, prefix in primitive_guards.items(): + trace = traces[task_id] + final = [event for event in trace['events'] if event.get('event') == 'final_decision'][0] + if trace['terminal_status'] != 'ACCEPTED': + raise SystemExit(f'{task_id} regressed') + if not str(final.get('selected_candidate_id', '')).startswith(prefix): + raise SystemExit(f'{task_id} selected primitive changed') + + ambiguous = traces['60b61512'] + classes = [event for event in ambiguous['events'] if event.get('event') == 'equivalence_class'] + bbox_classes = [ + event for event in classes + if any(str(candidate_id).startswith('bbox_complete:') for candidate_id in event['candidate_ids']) + ] + if ambiguous['terminal_status'] != 'AMBIGUOUS_PROGRAM': + raise SystemExit('60b61512 did not remain ambiguity-safe') + if len(bbox_classes) < 2: + raise SystemExit('60b61512 lost bounding-box disagreement') + + transitions = { + 'schema_version': 1, + 'candidate_commit': os.environ['GITHUB_SHA'], + 'baseline_commit': 'e50965009efb96457fd769d97b42027f8a4e98c0', + 'lost_rows': lost_rows, + 'gained_rows': gained_rows, + 'expected_gains_matched': True, + 'incorrect_attempts': int(report['metrics']['incorrect_attempts']), + } + mechanisms = { + 'schema_version': 1, + 'quadrant_tasks': quadrant_details, + 'role_boundary': 'NO_DISTINCT_QUADRANT_ROLES', + '60b61512': { + 'terminal_status': ambiguous['terminal_status'], + 'trace_digest': ambiguous['trace_digest'], + 'bbox_equivalence_class_count': len(bbox_classes), + 'safety_action': 'abstain_on_connectivity_disagreement', + }, + } + parity = { + 'schema_version': 1, + 'candidate_commit': os.environ['GITHUB_SHA'], + 'checked_tasks': 1000, + 'mismatches': 0, + 'lost_baseline_rows': lost_rows, + } + for name, data in ( + ('transitions.json', transitions), + ('mechanisms.json', mechanisms), + ('parity.json', parity), + ): + (output / name).write_text( + json.dumps(data, sort_keys=True, indent=2) + '\n', + encoding='utf-8', + ) + + print(json.dumps({ + 'report_digest': report['report_digest'], + 'diagnosis_digest': report['mechanistic']['diagnosis_digest'], + 'evaluated_tasks': report['evaluated_tasks'], + 'trace_count': report['mechanistic']['trace_count'], + 'parity_count': report['mechanistic']['parity_count'], + 'metrics': report['metrics'], + 'status_counts': report['status_counts'], + 'transitions': transitions, + 'mechanisms': mechanisms, + }, sort_keys=True)) + PY + - name: Build checksum manifest after all writers close + run: | + python - <<'PY' + import hashlib + from pathlib import Path + + output = Path('/tmp/arc-v2-2-quadrant') + lines = [] + for path in sorted( + item for item in output.rglob('*') + if item.is_file() and item.name != 'SHA256SUMS' + ): + digest = hashlib.sha256(path.read_bytes()).hexdigest() + lines.append(f'{digest} {path.relative_to(output).as_posix()}') + (output / 'SHA256SUMS').write_text('\n'.join(lines) + '\n', encoding='utf-8') + PY + - name: Verify checksum manifest independently + working-directory: /tmp/arc-v2-2-quadrant + run: sha256sum -c SHA256SUMS + - name: Publish quadrant mosaic evidence + uses: actions/upload-artifact@v4 + with: + name: arc-v2-2-quadrant-mosaic-${{ github.sha }} + path: /tmp/arc-v2-2-quadrant + if-no-files-found: error + retention-days: 90 + compression-level: 6 + - name: Write run summary + run: | + python - <<'PY' + import json + from pathlib import Path + + root = Path('/tmp/arc-v2-2-quadrant') + report = json.loads((root / 'report.json').read_text()) + mechanisms = json.loads((root / 'mechanisms.json').read_text()) + lines = [ + '# ARC V2.2 dihedral quadrant mosaic corpus', + '', + f"- Candidate: `{report['identity']['candidate_commit']}`", + f"- Report digest: `{report['report_digest']}`", + f"- Diagnosis digest: `{report['mechanistic']['diagnosis_digest']}`", + f"- Metrics: `{report['metrics']}`", + f"- Status counts: `{report['status_counts']}`", + f"- Quadrant gains: `{sorted(mechanisms['quadrant_tasks'])}`", + f"- Role boundary: `{mechanisms['role_boundary']}`", + f"- 60b61512 action: `{mechanisms['60b61512']['safety_action']}`", + '- Public ARC evaluation rerun: `no`', + ] + Path('/tmp/quadrant-summary.md').write_text('\n'.join(lines) + '\n', encoding='utf-8') + PY + cat /tmp/quadrant-summary.md >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/arc-quadrant-mosaic-module-green.yml b/.github/workflows/arc-quadrant-mosaic-module-green.yml new file mode 100644 index 00000000..1ff87254 --- /dev/null +++ b/.github/workflows/arc-quadrant-mosaic-module-green.yml @@ -0,0 +1,63 @@ +name: ARC Quadrant Mosaic Module GREEN + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-quadrant-mosaic-module-green.yml' + +permissions: + contents: write + +jobs: + verify-module: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Run isolated module tests + run: | + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py::test_apply_standard_reflected_quadrant_mosaic \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py::test_apply_rejects_output_beyond_arc_bounds \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py::test_program_derivation_returns_one_canonical_program \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py::test_program_derivation_reports_non_double_output_shape \ + | tee /tmp/module-green.log + - name: Confirm frozen bytes + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Commit module evidence + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + evidence = { + 'schema_version': 1, + 'source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': True, + 'test_count': 4, + 'test_output': Path('/tmp/module-green.log').read_text(encoding='utf-8'), + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'frozen_byte_gate': True, + 'public_arc_evaluation_rerun': False, + } + path = Path('research/universal_core/holonomy_v2_2/evidence/QUADRANT_MOSAIC_MODULE_GREEN.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/QUADRANT_MOSAIC_MODULE_GREEN.json + git commit -m 'arc: record quadrant mosaic module GREEN evidence' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-quadrant-mosaic-red.yml b/.github/workflows/arc-quadrant-mosaic-red.yml new file mode 100644 index 00000000..9e13bcde --- /dev/null +++ b/.github/workflows/arc-quadrant-mosaic-red.yml @@ -0,0 +1,70 @@ +name: ARC Quadrant Mosaic RED + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-quadrant-mosaic-red.yml' + +permissions: + contents: write + +jobs: + verify-red: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Require missing quadrant module + run: | + set +e + python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py > /tmp/quadrant-red.log 2>&1 + status=$? + set -e + cat /tmp/quadrant-red.log + test "$status" -ne 0 + grep -F "No module named 'research.universal_core.holonomy_v2_2.quadrant_mosaic'" /tmp/quadrant-red.log + - name: Commit RED evidence + env: + RED_SOURCE_SHA: ${{ github.sha }} + RED_RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + evidence = { + 'schema_version': 1, + 'red_commit': os.environ['RED_SOURCE_SHA'], + 'red_run_id': os.environ['RED_RUN_ID'], + 'test_command': 'python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py', + 'expected_failure': "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.quadrant_mosaic'", + 'failure_confirmed': True, + 'expected_new_exact_tasks': [ + '0c786b71', '3af2c5a8', '46442a0e', '62c24649', '67e8384a', + '7953d61e', '7fe24cdd', '833dafe3', 'ed98d772', + ], + 'training_scan': { + 'wrong_completed_hidden_outputs': 0, + 'ambiguous_hidden_outputs': 0, + }, + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'public_arc_evaluation_rerun': False, + } + path = Path('research/universal_core/holonomy_v2_2/evidence/QUADRANT_MOSAIC_RED.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/QUADRANT_MOSAIC_RED.json + git commit -m 'arc: record quadrant mosaic RED evidence' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-quadrant-mosaic-wire-v2.yml b/.github/workflows/arc-quadrant-mosaic-wire-v2.yml new file mode 100644 index 00000000..4fc8d6e5 --- /dev/null +++ b/.github/workflows/arc-quadrant-mosaic-wire-v2.yml @@ -0,0 +1,192 @@ +name: Wire ARC Quadrant Mosaic V2 + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-quadrant-mosaic-wire-v2.yml' + +permissions: + contents: write + +jobs: + wire-and-verify: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Apply guarded quadrant integration + run: | + python - <<'PY' + from pathlib import Path + + grid_path = Path('research/universal_core/holonomy_v2_2/grid_induction.py') + text = grid_path.read_text(encoding='utf-8') + import_anchor = 'from .mirror_concat import apply_mirror_concat\n' + if text.count(import_anchor) != 1 or 'from .quadrant_mosaic import' in text: + raise SystemExit('quadrant import anchor mismatch') + text = text.replace( + import_anchor, + import_anchor + 'from .quadrant_mosaic import apply_quadrant_mosaic\n', + 1, + ) + if text.count('_VERSION = "grid-v2.2-9"') != 1: + raise SystemExit('quadrant version anchor mismatch') + text = text.replace('_VERSION = "grid-v2.2-9"', '_VERSION = "grid-v2.2-10"', 1) + apply_anchor = ' if kind == "mirror_concatenate":\n' + if text.count(apply_anchor) != 1: + raise SystemExit('quadrant apply anchor mismatch') + text = text.replace( + apply_anchor, + ' if kind == "dihedral_quadrant_mosaic":\n' + ' return apply_quadrant_mosaic(program, grid)\n' + + apply_anchor, + 1, + ) + grid_path.write_text(text, encoding='utf-8') + + runtime_path = Path('research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py') + text = runtime_path.read_text(encoding='utf-8') + import_anchor = 'from .mirror_concat import mirror_concat_programs\n' + if text.count(import_anchor) != 1 or 'quadrant_mosaic_programs' in text: + raise SystemExit('quadrant runtime import anchor mismatch') + text = text.replace( + import_anchor, + import_anchor + 'from .quadrant_mosaic import quadrant_mosaic_programs\n', + 1, + ) + derive_anchor = ' mirror_concat, mirror_reason = mirror_concat_programs(demos)\n' + if text.count(derive_anchor) != 1: + raise SystemExit('quadrant derivation anchor mismatch') + text = text.replace( + derive_anchor, + derive_anchor + ' quadrant_mosaic, quadrant_reason = quadrant_mosaic_programs(demos)\n', + 1, + ) + batch_anchor = ' ("mirror_concat", mirror_concat, mirror_reason),\n' + if text.count(batch_anchor) != 1: + raise SystemExit('quadrant batch anchor mismatch') + text = text.replace( + batch_anchor, + batch_anchor + ' ("quadrant_mosaic", quadrant_mosaic, quadrant_reason),\n', + 1, + ) + runtime_path.write_text(text, encoding='utf-8') + PY + python -m py_compile \ + research/universal_core/holonomy_v2_2/quadrant_mosaic.py \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py + git diff --check + - name: Run focused quadrant tests + id: focused + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py \ + research/universal_core/holonomy_v2_2/tests/test_grid_induction.py::test_cell_scale_and_whole_grid_tile_are_distinct \ + research/universal_core/holonomy_v2_2/tests/test_mirror_concat.py \ + research/universal_core/holonomy_v2_2/tests/test_bbox_complete.py \ + research/universal_core/holonomy_v2_2/tests/test_component_outlier.py \ + research/universal_core/holonomy_v2_2/tests/test_marker_recolor.py \ + research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py \ + | tee /tmp/focused.log + - name: Run all V2.2 tests + id: cluster + if: always() + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q research/universal_core/holonomy_v2_2/tests | tee /tmp/v2-2.log + - name: Run frozen V2 integration tests + id: frozen + if: always() + continue-on-error: true + env: + PYTHONPATH: research + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py | tee /tmp/frozen.log + - name: Confirm frozen V2 bytes are unchanged + id: byte_gate + if: always() + continue-on-error: true + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Record result and commit only valid state + if: always() + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + FOCUSED_OUTCOME: ${{ steps.focused.outcome }} + CLUSTER_OUTCOME: ${{ steps.cluster.outcome }} + FROZEN_OUTCOME: ${{ steps.frozen.outcome }} + BYTE_OUTCOME: ${{ steps.byte_gate.outcome }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + outcomes = { + 'focused': os.environ['FOCUSED_OUTCOME'], + 'v2_2_cluster': os.environ['CLUSTER_OUTCOME'], + 'frozen_tests': os.environ['FROZEN_OUTCOME'], + 'frozen_byte_gate': os.environ['BYTE_OUTCOME'], + } + passed = all(value == 'success' for value in outcomes.values()) + evidence = { + 'schema_version': 2, + 'workflow_source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': passed, + 'outcomes': outcomes, + 'focused_output': Path('/tmp/focused.log').read_text(encoding='utf-8') if Path('/tmp/focused.log').exists() else '', + 'v2_2_output': Path('/tmp/v2-2.log').read_text(encoding='utf-8') if Path('/tmp/v2-2.log').exists() else '', + 'frozen_output': Path('/tmp/frozen.log').read_text(encoding='utf-8') if Path('/tmp/frozen.log').exists() else '', + 'role_boundary': 'NO_DISTINCT_QUADRANT_ROLES', + 'expected_new_exact_tasks': [ + '0c786b71', '3af2c5a8', '46442a0e', '62c24649', '67e8384a', + '7953d61e', '7fe24cdd', '833dafe3', 'ed98d772', + ], + 'expected_training_delta': { + 'correct_rows': 9, + 'accepted_rows': 9, + 'grammar_exhausted_rows': -9, + }, + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'public_arc_evaluation_rerun': False, + } + name = 'QUADRANT_MOSAIC_GREEN.json' if passed else 'QUADRANT_MOSAIC_DIAGNOSTIC_V2.json' + path = Path('research/universal_core/holonomy_v2_2/evidence') / name + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + Path('/tmp/passed').write_text('1' if passed else '0', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + if [ "$(cat /tmp/passed)" = "1" ]; then + git add \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py \ + research/universal_core/holonomy_v2_2/evidence/QUADRANT_MOSAIC_GREEN.json + git commit -m 'arc: wire dihedral quadrant mosaic' + else + git restore \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py + git add research/universal_core/holonomy_v2_2/evidence/QUADRANT_MOSAIC_DIAGNOSTIC_V2.json + git commit -m 'arc: record quadrant mosaic V2 diagnostic failure' + fi + git push origin HEAD:universal-core-arc-training-v2-2 + test "$(cat /tmp/passed)" = "1" diff --git a/.github/workflows/arc-quadrant-mosaic-wire.yml b/.github/workflows/arc-quadrant-mosaic-wire.yml new file mode 100644 index 00000000..a089fdaa --- /dev/null +++ b/.github/workflows/arc-quadrant-mosaic-wire.yml @@ -0,0 +1,181 @@ +name: Wire ARC Quadrant Mosaic + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-quadrant-mosaic-wire.yml' + +permissions: + contents: write + +jobs: + wire-and-verify: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Apply guarded quadrant integration + run: | + python - <<'PY' + from pathlib import Path + + grid_path = Path('research/universal_core/holonomy_v2_2/grid_induction.py') + text = grid_path.read_text(encoding='utf-8') + import_anchor = 'from .mirror_concat import apply_mirror_concat\n' + import_value = import_anchor + 'from .quadrant_mosaic import apply_quadrant_mosaic\n' + if text.count(import_anchor) != 1 or 'from .quadrant_mosaic import' in text: + raise SystemExit('quadrant import anchor mismatch') + text = text.replace(import_anchor, import_value, 1) + + version_anchor = '_VERSION = "grid-v2.2-9"' + if text.count(version_anchor) != 1: + raise SystemExit('quadrant version anchor mismatch') + text = text.replace(version_anchor, '_VERSION = "grid-v2.2-10"', 1) + + apply_anchor = ' if kind == "mirror_concatenate":\n' + apply_value = ( + ' if kind == "dihedral_quadrant_mosaic":\n' + ' return apply_quadrant_mosaic(program, grid)\n' + ) + if text.count(apply_anchor) != 1: + raise SystemExit('quadrant apply anchor mismatch') + text = text.replace(apply_anchor, apply_value + apply_anchor, 1) + grid_path.write_text(text, encoding='utf-8') + + runtime_path = Path('research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py') + text = runtime_path.read_text(encoding='utf-8') + runtime_import = 'from .mirror_concat import mirror_concat_programs\n' + runtime_import_value = runtime_import + 'from .quadrant_mosaic import quadrant_mosaic_programs\n' + if text.count(runtime_import) != 1 or 'quadrant_mosaic_programs' in text: + raise SystemExit('quadrant runtime import anchor mismatch') + text = text.replace(runtime_import, runtime_import_value, 1) + + derived_anchor = ' mirror_concat, mirror_reason = mirror_concat_programs(demos)\n' + derived_value = derived_anchor + ' quadrant_mosaic, quadrant_reason = quadrant_mosaic_programs(demos)\n' + if text.count(derived_anchor) != 1: + raise SystemExit('quadrant derivation anchor mismatch') + text = text.replace(derived_anchor, derived_value, 1) + + batch_anchor = ' ("mirror_concat", mirror_concat, mirror_reason),\n' + batch_value = batch_anchor + ' ("quadrant_mosaic", quadrant_mosaic, quadrant_reason),\n' + if text.count(batch_anchor) != 1: + raise SystemExit('quadrant batch anchor mismatch') + text = text.replace(batch_anchor, batch_value, 1) + runtime_path.write_text(text, encoding='utf-8') + PY + python -m py_compile \ + research/universal_core/holonomy_v2_2/quadrant_mosaic.py \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py + git diff --check + - name: Run focused quadrant tests + id: focused + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py \ + research/universal_core/holonomy_v2_2/tests/test_mirror_concat.py \ + research/universal_core/holonomy_v2_2/tests/test_bbox_complete.py \ + research/universal_core/holonomy_v2_2/tests/test_component_outlier.py \ + research/universal_core/holonomy_v2_2/tests/test_marker_recolor.py \ + research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py \ + | tee /tmp/focused.log + - name: Run all V2.2 tests + id: cluster + if: always() + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q research/universal_core/holonomy_v2_2/tests | tee /tmp/v2-2.log + - name: Run frozen V2 integration tests + id: frozen + if: always() + continue-on-error: true + env: + PYTHONPATH: research + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py | tee /tmp/frozen.log + - name: Confirm frozen V2 bytes are unchanged + id: byte_gate + if: always() + continue-on-error: true + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Record result and commit only valid state + if: always() + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + FOCUSED_OUTCOME: ${{ steps.focused.outcome }} + CLUSTER_OUTCOME: ${{ steps.cluster.outcome }} + FROZEN_OUTCOME: ${{ steps.frozen.outcome }} + BYTE_OUTCOME: ${{ steps.byte_gate.outcome }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + outcomes = { + 'focused': os.environ['FOCUSED_OUTCOME'], + 'v2_2_cluster': os.environ['CLUSTER_OUTCOME'], + 'frozen_tests': os.environ['FROZEN_OUTCOME'], + 'frozen_byte_gate': os.environ['BYTE_OUTCOME'], + } + passed = all(value == 'success' for value in outcomes.values()) + evidence = { + 'schema_version': 1, + 'workflow_source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': passed, + 'outcomes': outcomes, + 'focused_output': Path('/tmp/focused.log').read_text(encoding='utf-8') if Path('/tmp/focused.log').exists() else '', + 'v2_2_output': Path('/tmp/v2-2.log').read_text(encoding='utf-8') if Path('/tmp/v2-2.log').exists() else '', + 'frozen_output': Path('/tmp/frozen.log').read_text(encoding='utf-8') if Path('/tmp/frozen.log').exists() else '', + 'expected_new_exact_tasks': [ + '0c786b71', '3af2c5a8', '46442a0e', '62c24649', '67e8384a', + '7953d61e', '7fe24cdd', '833dafe3', 'ed98d772', + ], + 'expected_training_delta': { + 'correct_rows': 9, + 'accepted_rows': 9, + 'grammar_exhausted_rows': -9, + }, + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'public_arc_evaluation_rerun': False, + } + name = 'QUADRANT_MOSAIC_GREEN.json' if passed else 'QUADRANT_MOSAIC_DIAGNOSTIC.json' + path = Path('research/universal_core/holonomy_v2_2/evidence') / name + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + Path('/tmp/passed').write_text('1' if passed else '0', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + if [ "$(cat /tmp/passed)" = "1" ]; then + git add \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py \ + research/universal_core/holonomy_v2_2/evidence/QUADRANT_MOSAIC_GREEN.json + git commit -m 'arc: wire dihedral quadrant mosaic' + else + git restore \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py + git add research/universal_core/holonomy_v2_2/evidence/QUADRANT_MOSAIC_DIAGNOSTIC.json + git commit -m 'arc: record quadrant mosaic diagnostic failure' + fi + git push origin HEAD:universal-core-arc-training-v2-2 + test "$(cat /tmp/passed)" = "1" diff --git a/.github/workflows/arc-quadrant-role-red.yml b/.github/workflows/arc-quadrant-role-red.yml new file mode 100644 index 00000000..9c6d7fc9 --- /dev/null +++ b/.github/workflows/arc-quadrant-role-red.yml @@ -0,0 +1,64 @@ +name: ARC Quadrant Role Boundary RED + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-quadrant-role-red.yml' + +permissions: + contents: write + +jobs: + verify-red: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Require missing role boundary + run: | + set +e + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py::test_program_derivation_defers_uniform_quadrant_roles_to_tile \ + > /tmp/role-red.log 2>&1 + status=$? + set -e + cat /tmp/role-red.log + test "$status" -ne 0 + grep -F 'NO_DISTINCT_QUADRANT_ROLES' /tmp/role-red.log + - name: Commit RED evidence + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + evidence = { + 'schema_version': 1, + 'source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'failure_confirmed': True, + 'root_cause': 'a transform supported in all four quadrant roles overlaps whole-grid tiling and can create hidden ambiguity on shape changes', + 'required_boundary': 'NO_DISTINCT_QUADRANT_ROLES', + 'test_command': 'python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py::test_program_derivation_defers_uniform_quadrant_roles_to_tile', + 'captured_log': Path('/tmp/role-red.log').read_text(encoding='utf-8'), + 'public_arc_evaluation_rerun': False, + } + path = Path('research/universal_core/holonomy_v2_2/evidence/QUADRANT_ROLE_BOUNDARY_RED.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/QUADRANT_ROLE_BOUNDARY_RED.json + git commit -m 'arc: record quadrant role boundary RED' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-ray-extend-corpus.yml b/.github/workflows/arc-ray-extend-corpus.yml new file mode 100644 index 00000000..db064bb6 --- /dev/null +++ b/.github/workflows/arc-ray-extend-corpus.yml @@ -0,0 +1,364 @@ +name: ARC Ray Extension Mechanistic Corpus + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-ray-extend-corpus.yml' + +permissions: + contents: read + actions: read + +jobs: + training-corpus: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Download immutable benchmark corpus + uses: actions/download-artifact@v4 + with: + name: public-benchmarks-pinned-2026-08-04 + path: /tmp/pinned-public-benchmarks + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30929169392' + - name: Download immutable V2.2 baseline report + uses: actions/download-artifact@v4 + with: + name: arc-v2-2-training-e50965009efb96457fd769d97b42027f8a4e98c0 + path: /tmp/arc-v2-2-baseline + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30941837872' + - name: Verify V2.2 and frozen V2 gates + env: + PYTHONPATH: research + run: | + python -m pip install --disable-pip-version-check --quiet pytest + python -m pytest -q research/universal_core/holonomy_v2_2/tests + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py + git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Run all ARC training tasks with complete traces + run: | + mkdir -p /tmp/arc-v2-2-ray/traces + python - <<'PY' | tee /tmp/arc-v2-2-ray/console-summary.json + import json + import os + from pathlib import Path + + from research.universal_core.holonomy_v2.engine import HolonomyEngine + from research.universal_core.holonomy_v2.types import SearchConfig + from research.universal_core.holonomy_v2_2.mechanistic_trace import validate_trace + from research.universal_core.holonomy_v2_2.training_eval import run_training_corpus, write_report + from research.universal_core.public_benchmarks.suite import locate_vendor + + output = Path('/tmp/arc-v2-2-ray') + trace_dir = output / 'traces' + vendor = locate_vendor(Path('/tmp/pinned-public-benchmarks')) + report = run_training_corpus( + vendor, + HolonomyEngine(SearchConfig()), + run_label='arc-v2-2-ray-' + os.environ['GITHUB_SHA'][:12], + candidate_commit=os.environ['GITHUB_SHA'], + mechanistic_dir=trace_dir, + verify_parity=True, + ) + write_report(report, output / 'report.json') + + trace_paths = sorted( + path for path in trace_dir.glob('*.json') + if path.name != 'diagnosis.json' + ) + if len(trace_paths) != 1000: + raise SystemExit(f'expected 1000 task traces, found {len(trace_paths)}') + traces = {} + for path in trace_paths: + trace = json.loads(path.read_text(encoding='utf-8')) + validate_trace(trace) + traces[path.stem] = trace + if report['mechanistic']['trace_count'] != 1000: + raise SystemExit('aggregate trace count mismatch') + if report['mechanistic']['parity_count'] != 1000: + raise SystemExit('compact/mechanistic parity count mismatch') + + baseline = json.loads( + Path('/tmp/arc-v2-2-baseline/report.json').read_text(encoding='utf-8') + ) + baseline_rows = { + item['task_id']: int(item['metrics']['correct']) + for item in baseline['reports'] + } + current_rows = { + item['task_id']: int(item['metrics']['correct']) + for item in report['reports'] + } + lost_rows = { + task_id: {'baseline_correct': correct, 'current_correct': current_rows.get(task_id, 0)} + for task_id, correct in baseline_rows.items() + if current_rows.get(task_id, 0) < correct + } + gained_rows = { + task_id: {'baseline_correct': baseline_rows.get(task_id, 0), 'current_correct': correct} + for task_id, correct in current_rows.items() + if correct > baseline_rows.get(task_id, 0) + } + expected_gain_ids = [ + '0c786b71', '1e0a9b12', '1f876c06', '22168020', '22eb0ac0', + '2dee498d', '3906de3d', '3aa6fb7a', '3af2c5a8', '46442a0e', + '496994bd', '4c4377d9', '5614dbcf', '623ea044', '62c24649', + '67e8384a', '68b67ca3', '6d0aefbc', '6d75e8bb', '6fa7a44f', + '746b3537', '7953d61e', '7b7f7511', '7fe24cdd', '833dafe3', + '8be77c9e', 'a59b95c0', 'aabf363d', 'ac0a08a4', 'b230c067', + 'b91ae062', 'c9e6f938', 'ce8d95cc', 'd037b0a7', 'd4b1c2b1', + 'ded97339', 'e1baa8a4', 'e729b7be', 'e7639916', 'eb5a1d5d', + 'ed98d772', 'f25ffba3', + ] + expected_gains = { + task_id: {'baseline_correct': 0, 'current_correct': 1} + for task_id in expected_gain_ids + } + if lost_rows: + raise SystemExit('lost baseline-correct rows: ' + json.dumps(lost_rows, sort_keys=True)) + if gained_rows != expected_gains: + raise SystemExit( + 'unexpected gained rows: expected ' + + json.dumps(expected_gains, sort_keys=True) + + ' found ' + + json.dumps(gained_rows, sort_keys=True) + ) + + expected_metrics = { + 'rows': 1076, + 'correct': 101, + 'accepted': 101, + 'incorrect_attempts': 0, + 'failed': 0, + } + for key, expected in expected_metrics.items(): + if int(report['metrics'][key]) != expected: + raise SystemExit( + f'metric {key} changed: expected {expected}, found {report["metrics"][key]}' + ) + expected_status = { + 'ACCEPTED': 101, + 'AMBIGUOUS_PROGRAM': 2, + 'GRAMMAR_EXHAUSTED': 973, + } + if report['status_counts'] != expected_status: + raise SystemExit( + 'status counts changed: expected ' + + json.dumps(expected_status, sort_keys=True) + + ' found ' + + json.dumps(report['status_counts'], sort_keys=True) + ) + + ray_ids = ('623ea044', 'd037b0a7') + ray_details = {} + for task_id in ray_ids: + trace = traces[task_id] + final = [event for event in trace['events'] if event.get('event') == 'final_decision'][0] + successes = [ + event for event in trace['events'] + if event.get('event') == 'hidden_execution' + and event.get('candidate_kind') == 'anchor_ray_extend' + and event.get('outcome') == 'SUCCEEDED' + ] + if trace['terminal_status'] != 'ACCEPTED': + raise SystemExit(f'{task_id} did not transition to ACCEPTED') + if not successes: + raise SystemExit(f'{task_id} has no successful ray candidate') + selected = str(final.get('selected_candidate_id', '')) + if not selected.startswith('ray_extend:'): + raise SystemExit(f'{task_id} did not select ray extension: {selected}') + ray_details[task_id] = { + 'trace_digest': trace['trace_digest'], + 'successful_ray_candidates': len(successes), + 'selected_candidate_id': selected, + } + + primitive_guards = { + 'aabf363d': 'marker_recolor:', + 'b230c067': 'component_outlier:', + '3aa6fb7a': 'bbox_complete:', + '6d75e8bb': 'bbox_complete:', + 'e7639916': 'bbox_complete:', + '4c4377d9': 'mirror_concat:', + '6d0aefbc': 'mirror_concat:', + '6fa7a44f': 'mirror_concat:', + '8be77c9e': 'mirror_concat:', + 'c9e6f938': 'mirror_concat:', + '0c786b71': 'quadrant_mosaic:', + '3af2c5a8': 'quadrant_mosaic:', + '46442a0e': 'quadrant_mosaic:', + '62c24649': 'quadrant_mosaic:', + '67e8384a': 'quadrant_mosaic:', + '7953d61e': 'quadrant_mosaic:', + '7fe24cdd': 'quadrant_mosaic:', + '833dafe3': 'quadrant_mosaic:', + 'ed98d772': 'quadrant_mosaic:', + 'a59b95c0': 'palette_repeat:', + 'ac0a08a4': 'palette_repeat:', + 'b91ae062': 'palette_repeat:', + 'd4b1c2b1': 'palette_repeat:', + '2dee498d': 'axis_collapse:', + '746b3537': 'axis_collapse:', + '7b7f7511': 'axis_collapse:', + 'ce8d95cc': 'axis_collapse:', + 'e1baa8a4': 'axis_collapse:', + 'eb5a1d5d': 'axis_collapse:', + '5614dbcf': 'block_reduce:', + '68b67ca3': 'block_reduce:', + '1e0a9b12': 'gravity_compact:', + '3906de3d': 'gravity_compact:', + '1f876c06': 'line_connect:', + '22168020': 'line_connect:', + '22eb0ac0': 'line_connect:', + 'ded97339': 'line_connect:', + '496994bd': 'symmetry_complete:', + 'e729b7be': 'symmetry_complete:', + 'f25ffba3': 'symmetry_complete:', + } + prior_mechanisms = {} + for task_id, prefix in primitive_guards.items(): + trace = traces[task_id] + final = [event for event in trace['events'] if event.get('event') == 'final_decision'][0] + if trace['terminal_status'] != 'ACCEPTED': + raise SystemExit(f'{task_id} regressed') + selected = str(final.get('selected_candidate_id', '')) + if not selected.startswith(prefix): + raise SystemExit(f'{task_id} selected primitive changed: {selected}') + prior_mechanisms[task_id] = { + 'selected_candidate_id': selected, + 'trace_digest': trace['trace_digest'], + } + + b882 = traces['b8825c91'] + b882_classes = [ + event for event in b882['events'] + if event.get('event') == 'equivalence_class' + and any(str(candidate_id).startswith('symmetry_complete:') for candidate_id in event['candidate_ids']) + ] + if b882['terminal_status'] != 'AMBIGUOUS_PROGRAM' or len(b882_classes) < 2: + raise SystemExit('b8825c91 symmetry ambiguity regressed') + + old_ambiguous = traces['60b61512'] + old_classes = [event for event in old_ambiguous['events'] if event.get('event') == 'equivalence_class'] + bbox_classes = [ + event for event in old_classes + if any(str(candidate_id).startswith('bbox_complete:') for candidate_id in event['candidate_ids']) + ] + if old_ambiguous['terminal_status'] != 'AMBIGUOUS_PROGRAM' or len(bbox_classes) < 2: + raise SystemExit('60b61512 bbox ambiguity regressed') + + transitions = { + 'schema_version': 1, + 'candidate_commit': os.environ['GITHUB_SHA'], + 'baseline_commit': 'e50965009efb96457fd769d97b42027f8a4e98c0', + 'lost_rows': lost_rows, + 'gained_rows': gained_rows, + 'expected_gains_matched': True, + 'incorrect_attempts': int(report['metrics']['incorrect_attempts']), + } + mechanisms = { + 'schema_version': 1, + 'ray_tasks': ray_details, + 'prior_mechanisms': prior_mechanisms, + 'b8825c91': { + 'terminal_status': b882['terminal_status'], + 'trace_digest': b882['trace_digest'], + 'symmetry_equivalence_class_count': len(b882_classes), + 'safety_action': 'abstain_on_symmetry_disagreement', + }, + '60b61512': { + 'terminal_status': old_ambiguous['terminal_status'], + 'trace_digest': old_ambiguous['trace_digest'], + 'bbox_equivalence_class_count': len(bbox_classes), + 'safety_action': 'abstain_on_connectivity_disagreement', + }, + } + parity = { + 'schema_version': 1, + 'candidate_commit': os.environ['GITHUB_SHA'], + 'checked_tasks': 1000, + 'mismatches': 0, + 'lost_baseline_rows': lost_rows, + } + for name, data in ( + ('transitions.json', transitions), + ('mechanisms.json', mechanisms), + ('parity.json', parity), + ): + (output / name).write_text( + json.dumps(data, sort_keys=True, indent=2) + '\n', encoding='utf-8' + ) + + print(json.dumps({ + 'report_digest': report['report_digest'], + 'diagnosis_digest': report['mechanistic']['diagnosis_digest'], + 'evaluated_tasks': report['evaluated_tasks'], + 'trace_count': report['mechanistic']['trace_count'], + 'parity_count': report['mechanistic']['parity_count'], + 'metrics': report['metrics'], + 'status_counts': report['status_counts'], + 'transitions': transitions, + 'ray_tasks': ray_details, + }, sort_keys=True)) + PY + - name: Build checksum manifest after all writers close + run: | + python - <<'PY' + import hashlib + from pathlib import Path + output = Path('/tmp/arc-v2-2-ray') + lines = [] + for path in sorted(item for item in output.rglob('*') if item.is_file() and item.name != 'SHA256SUMS'): + digest = hashlib.sha256(path.read_bytes()).hexdigest() + lines.append(f'{digest} {path.relative_to(output).as_posix()}') + (output / 'SHA256SUMS').write_text('\n'.join(lines) + '\n', encoding='utf-8') + PY + - name: Verify checksum manifest independently + working-directory: /tmp/arc-v2-2-ray + run: sha256sum -c SHA256SUMS + - name: Publish ray extension evidence + uses: actions/upload-artifact@v4 + with: + name: arc-v2-2-ray-extend-${{ github.sha }} + path: /tmp/arc-v2-2-ray + if-no-files-found: error + retention-days: 90 + compression-level: 6 + - name: Write run summary + run: | + python - <<'PY' + import json + from pathlib import Path + root = Path('/tmp/arc-v2-2-ray') + report = json.loads((root / 'report.json').read_text()) + mechanisms = json.loads((root / 'mechanisms.json').read_text()) + lines = [ + '# ARC V2.2 anchor ray extension corpus', + '', + f"- Candidate: `{report['identity']['candidate_commit']}`", + f"- Report digest: `{report['report_digest']}`", + f"- Diagnosis digest: `{report['mechanistic']['diagnosis_digest']}`", + f"- Metrics: `{report['metrics']}`", + f"- Status counts: `{report['status_counts']}`", + f"- Ray gains: `{sorted(mechanisms['ray_tasks'])}`", + f"- b8825c91 action: `{mechanisms['b8825c91']['safety_action']}`", + f"- 60b61512 action: `{mechanisms['60b61512']['safety_action']}`", + '- Public ARC evaluation rerun: `no`', + ] + Path('/tmp/ray-summary.md').write_text('\n'.join(lines) + '\n', encoding='utf-8') + PY + cat /tmp/ray-summary.md >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/arc-ray-extend-module-green.yml b/.github/workflows/arc-ray-extend-module-green.yml new file mode 100644 index 00000000..ee19f39e --- /dev/null +++ b/.github/workflows/arc-ray-extend-module-green.yml @@ -0,0 +1,70 @@ +name: ARC Ray Extension Module GREEN + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-ray-extend-module-green.yml' + +permissions: + contents: write + +jobs: + verify-module: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Run isolated ray tests + run: | + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_ray_extend_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_ray_extend.py::test_down_ray_extends_each_anchor_to_bottom_border \ + research/universal_core/holonomy_v2_2/tests/test_ray_extend.py::test_diagonal_ray_extends_anchor_to_four_diagonal_borders \ + research/universal_core/holonomy_v2_2/tests/test_ray_extend.py::test_orthogonal_ray_extends_anchor_in_four_cardinal_directions \ + research/universal_core/holonomy_v2_2/tests/test_ray_extend.py::test_ray_extension_preserves_original_anchor_cells \ + research/universal_core/holonomy_v2_2/tests/test_ray_extend.py::test_ray_extension_rejects_cross_color_conflict \ + research/universal_core/holonomy_v2_2/tests/test_ray_extend.py::test_ray_extension_rejects_unknown_mode \ + research/universal_core/holonomy_v2_2/tests/test_ray_extend.py::test_program_derivation_returns_unique_down_candidate \ + research/universal_core/holonomy_v2_2/tests/test_ray_extend.py::test_program_derivation_rejects_shape_change \ + research/universal_core/holonomy_v2_2/tests/test_ray_extend.py::test_program_derivation_skips_noop_demonstration \ + | tee /tmp/ray-module-green.log + - name: Confirm frozen bytes + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Commit module evidence + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + evidence = { + 'schema_version': 1, + 'source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': True, + 'test_count': 10, + 'test_output': Path('/tmp/ray-module-green.log').read_text(encoding='utf-8'), + 'expected_new_exact_tasks': ['623ea044', 'd037b0a7'], + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'frozen_byte_gate': True, + 'public_arc_evaluation_rerun': False, + } + path = Path('research/universal_core/holonomy_v2_2/evidence/RAY_EXTEND_MODULE_GREEN.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/RAY_EXTEND_MODULE_GREEN.json + git commit -m 'arc: record ray extension module GREEN' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-ray-extend-pin-fixtures.yml b/.github/workflows/arc-ray-extend-pin-fixtures.yml new file mode 100644 index 00000000..f263df4f --- /dev/null +++ b/.github/workflows/arc-ray-extend-pin-fixtures.yml @@ -0,0 +1,54 @@ +name: Pin ARC Ray Extension Fixtures + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-ray-extend-pin-fixtures.yml' + +permissions: + contents: write + actions: read + +jobs: + pin: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - name: Download immutable benchmark corpus + uses: actions/download-artifact@v4 + with: + name: public-benchmarks-pinned-2026-08-04 + path: /tmp/pinned-public-benchmarks + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30929169392' + - name: Copy exact immutable payloads + run: | + set -euo pipefail + probe=$(find /tmp/pinned-public-benchmarks -type f -path '*/arc_agi_2/data/training/623ea044.json' -print -quit) + test -n "$probe" + src=$(dirname "$probe") + dst=research/universal_core/holonomy_v2_2/tests/fixtures/ray_extend + mkdir -p "$dst" + cp "$src/623ea044.json" "$dst/623ea044.json" + cp "$src/d037b0a7.json" "$dst/d037b0a7.json" + printf '%s %s\n' \ + 578ab7e47a2d67489f1103715615f719ef81e9faf773be85586803d6da44cbdd "$dst/623ea044.json" \ + 9e39a4acdb3e3bff7dd54bed5f4ab77973f4027d48beb62a9c740d8a7a0b0aee "$dst/d037b0a7.json" \ + > /tmp/ray-fixtures.sha256 + sha256sum -c /tmp/ray-fixtures.sha256 + - name: Confirm frozen V2 bytes + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Commit fixtures + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/tests/fixtures/ray_extend + git commit -m 'test: pin anchor ray extension fixtures' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-ray-extend-red.yml b/.github/workflows/arc-ray-extend-red.yml new file mode 100644 index 00000000..a932b67d --- /dev/null +++ b/.github/workflows/arc-ray-extend-red.yml @@ -0,0 +1,79 @@ +name: ARC Ray Extension RED + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-ray-extend-red.yml' + +permissions: + contents: write + +jobs: + verify-red: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Verify pinned ray fixtures + run: python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_ray_extend_fixtures.py + - name: Require missing ray module + run: | + set +e + python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_ray_extend.py > /tmp/ray-red.log 2>&1 + status=$? + set -e + cat /tmp/ray-red.log + test "$status" -ne 0 + grep -F "No module named 'research.universal_core.holonomy_v2_2.ray_extend'" /tmp/ray-red.log + - name: Verify frozen V2.1 byte identity + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Commit RED evidence + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + evidence = { + 'schema_version': 1, + 'source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'failure_confirmed': True, + 'fixture_integrity': True, + 'expected_failure': "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.ray_extend'", + 'expected_new_exact_tasks': ['623ea044', 'd037b0a7'], + 'expected_cumulative_metrics': { + 'rows': 1076, + 'accepted': 101, + 'correct': 101, + 'incorrect_attempts': 0, + 'failed': 0, + }, + 'expected_status_counts': { + 'ACCEPTED': 101, + 'AMBIGUOUS_PROGRAM': 2, + 'GRAMMAR_EXHAUSTED': 973, + }, + 'captured_log': Path('/tmp/ray-red.log').read_text(encoding='utf-8'), + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'public_arc_evaluation_rerun': False, + } + path = Path('research/universal_core/holonomy_v2_2/evidence/RAY_EXTEND_RED.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/RAY_EXTEND_RED.json + git commit -m 'arc: record ray extension RED evidence' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-ray-extend-wire.yml b/.github/workflows/arc-ray-extend-wire.yml new file mode 100644 index 00000000..616377e4 --- /dev/null +++ b/.github/workflows/arc-ray-extend-wire.yml @@ -0,0 +1,208 @@ +name: Wire ARC Ray Extension + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-ray-extend-wire.yml' + +permissions: + contents: write + +jobs: + wire-and-verify: + runs-on: ubuntu-latest + timeout-minutes: 55 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Apply guarded ray integration + run: | + python - <<'PY' + from pathlib import Path + + grid_path = Path('research/universal_core/holonomy_v2_2/grid_induction.py') + text = grid_path.read_text(encoding='utf-8') + import_anchor = 'from .symmetry_complete import apply_symmetry_complete\n' + if text.count(import_anchor) != 1 or 'from .ray_extend import' in text: + raise SystemExit('ray import anchor mismatch') + text = text.replace( + import_anchor, + import_anchor + 'from .ray_extend import apply_ray_extend\n', + 1, + ) + if text.count('_VERSION = "grid-v2.2-16"') != 1: + raise SystemExit('ray version anchor mismatch') + text = text.replace('_VERSION = "grid-v2.2-16"', '_VERSION = "grid-v2.2-17"', 1) + apply_anchor = ' if kind == "reflective_symmetry_complete":\n' + if text.count(apply_anchor) != 1: + raise SystemExit('ray apply anchor mismatch') + text = text.replace( + apply_anchor, + ' if kind == "anchor_ray_extend":\n' + ' return apply_ray_extend(program, grid)\n' + + apply_anchor, + 1, + ) + grid_path.write_text(text, encoding='utf-8') + + runtime_path = Path('research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py') + text = runtime_path.read_text(encoding='utf-8') + import_anchor = 'from .symmetry_complete import symmetry_complete_programs\n' + if text.count(import_anchor) != 1 or 'ray_extend_programs' in text: + raise SystemExit('ray runtime import anchor mismatch') + text = text.replace( + import_anchor, + import_anchor + 'from .ray_extend import ray_extend_programs\n', + 1, + ) + derive_anchor = ' symmetry_complete, symmetry_reason = symmetry_complete_programs(demos)\n' + if text.count(derive_anchor) != 1: + raise SystemExit('ray derivation anchor mismatch') + text = text.replace( + derive_anchor, + derive_anchor + ' ray_extend, ray_reason = ray_extend_programs(demos)\n', + 1, + ) + batch_anchor = ' ("symmetry_complete", symmetry_complete, symmetry_reason),\n' + if text.count(batch_anchor) != 1: + raise SystemExit('ray batch anchor mismatch') + text = text.replace( + batch_anchor, + batch_anchor + ' ("ray_extend", ray_extend, ray_reason),\n', + 1, + ) + runtime_path.write_text(text, encoding='utf-8') + PY + python -m py_compile \ + research/universal_core/holonomy_v2_2/ray_extend.py \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py + git diff --check + - name: Run focused ray regression set + id: focused + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_ray_extend_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_ray_extend.py \ + research/universal_core/holonomy_v2_2/tests/test_symmetry_complete_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_symmetry_complete.py \ + research/universal_core/holonomy_v2_2/tests/test_line_connect_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_line_connect.py \ + research/universal_core/holonomy_v2_2/tests/test_gravity_compact_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_gravity_compact.py \ + research/universal_core/holonomy_v2_2/tests/test_block_reduce_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_block_reduce.py \ + research/universal_core/holonomy_v2_2/tests/test_axis_collapse_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_axis_collapse.py \ + research/universal_core/holonomy_v2_2/tests/test_palette_repeat_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_palette_repeat.py \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py \ + research/universal_core/holonomy_v2_2/tests/test_mirror_concat.py \ + research/universal_core/holonomy_v2_2/tests/test_bbox_complete.py \ + research/universal_core/holonomy_v2_2/tests/test_component_outlier.py \ + research/universal_core/holonomy_v2_2/tests/test_marker_recolor.py \ + research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py \ + research/universal_core/holonomy_v2_2/tests/test_grid_induction.py::test_cell_scale_and_whole_grid_tile_are_distinct \ + | tee /tmp/ray-focused.log + - name: Run all V2.2 tests + id: cluster + if: always() + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q research/universal_core/holonomy_v2_2/tests | tee /tmp/ray-v2-2.log + - name: Run frozen V2 integration tests + id: frozen + if: always() + continue-on-error: true + env: + PYTHONPATH: research + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py | tee /tmp/ray-frozen.log + - name: Confirm frozen V2 bytes are unchanged + id: byte_gate + if: always() + continue-on-error: true + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Record result and commit only valid state + if: always() + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + FOCUSED_OUTCOME: ${{ steps.focused.outcome }} + CLUSTER_OUTCOME: ${{ steps.cluster.outcome }} + FROZEN_OUTCOME: ${{ steps.frozen.outcome }} + BYTE_OUTCOME: ${{ steps.byte_gate.outcome }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + outcomes = { + 'focused': os.environ['FOCUSED_OUTCOME'], + 'v2_2_cluster': os.environ['CLUSTER_OUTCOME'], + 'frozen_tests': os.environ['FROZEN_OUTCOME'], + 'frozen_byte_gate': os.environ['BYTE_OUTCOME'], + } + passed = all(value == 'success' for value in outcomes.values()) + evidence = { + 'schema_version': 1, + 'workflow_source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': passed, + 'outcomes': outcomes, + 'focused_output': Path('/tmp/ray-focused.log').read_text(encoding='utf-8') if Path('/tmp/ray-focused.log').exists() else '', + 'v2_2_output': Path('/tmp/ray-v2-2.log').read_text(encoding='utf-8') if Path('/tmp/ray-v2-2.log').exists() else '', + 'frozen_output': Path('/tmp/ray-frozen.log').read_text(encoding='utf-8') if Path('/tmp/ray-frozen.log').exists() else '', + 'expected_new_exact_tasks': ['623ea044', 'd037b0a7'], + 'expected_cumulative_metrics': { + 'rows': 1076, + 'correct': 101, + 'accepted': 101, + 'incorrect_attempts': 0, + 'failed': 0, + }, + 'expected_status_counts': { + 'ACCEPTED': 101, + 'AMBIGUOUS_PROGRAM': 2, + 'GRAMMAR_EXHAUSTED': 973, + }, + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'public_arc_evaluation_rerun': False, + } + name = 'RAY_EXTEND_GREEN.json' if passed else 'RAY_EXTEND_DIAGNOSTIC.json' + path = Path('research/universal_core/holonomy_v2_2/evidence') / name + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + Path('/tmp/ray-passed').write_text('1' if passed else '0', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + if [ "$(cat /tmp/ray-passed)" = "1" ]; then + git add \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py \ + research/universal_core/holonomy_v2_2/evidence/RAY_EXTEND_GREEN.json + git commit -m 'arc: wire anchor ray extension' + else + git restore \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py + git add research/universal_core/holonomy_v2_2/evidence/RAY_EXTEND_DIAGNOSTIC.json + git commit -m 'arc: record ray extension diagnostic failure' + fi + git push origin HEAD:universal-core-arc-training-v2-2 + test "$(cat /tmp/ray-passed)" = "1" diff --git a/.github/workflows/arc-region-wire.yml b/.github/workflows/arc-region-wire.yml new file mode 100644 index 00000000..e0ca5c9a --- /dev/null +++ b/.github/workflows/arc-region-wire.yml @@ -0,0 +1,76 @@ +name: Wire ARC Enclosed Regions + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-region-wire.yml' + +permissions: + contents: write + +jobs: + wire: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - name: Apply guarded integration patch + run: | + python - <<'PY' + from pathlib import Path + + path = Path('research/universal_core/holonomy_v2_2/grid_induction.py') + text = path.read_text(encoding='utf-8') + + import_anchor = 'from .panel_combine import apply_panel, panel_candidates\n' + import_value = import_anchor + 'from .region_fill import apply_region, region_candidates\n' + if text.count(import_anchor) != 1 or 'from .region_fill import' in text: + raise SystemExit('region import anchor mismatch') + text = text.replace(import_anchor, import_value, 1) + + version_anchor = '_VERSION = "grid-v2.2-3"' + if text.count(version_anchor) != 1: + raise SystemExit('version anchor mismatch') + text = text.replace(version_anchor, '_VERSION = "grid-v2.2-4"', 1) + + apply_anchor = ' if kind == "color_map":\n' + apply_value = ( + ' if kind == "enclosed_region_fill":\n' + ' return apply_region(program, grid)\n' + ) + if text.count(apply_anchor) != 1: + raise SystemExit('region apply anchor mismatch') + text = text.replace(apply_anchor, apply_value + apply_anchor, 1) + + candidate_anchor = ' candidates.extend(panel_candidates(demos))\n' + candidate_value = candidate_anchor + ' candidates.extend(region_candidates(demos))\n' + if text.count(candidate_anchor) != 1: + raise SystemExit('region candidate anchor mismatch') + text = text.replace(candidate_anchor, candidate_value, 1) + + background_loop = ' for background in backgrounds:\n' + background_value = ( + ' common_backgrounds = set(cell for row in demos[0][0] for cell in row)\n' + ' for source, _ in demos[1:]:\n' + ' common_backgrounds.intersection_update(cell for row in source for cell in row)\n' + ' for background in sorted(common_backgrounds):\n' + ) + if text.count(background_loop) != 1: + raise SystemExit('component background loop mismatch') + text = text.replace(background_loop, background_value, 1) + + path.write_text(text, encoding='utf-8') + PY + git diff --check + git diff -- research/universal_core/holonomy_v2_2/grid_induction.py + - name: Commit integration + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/grid_induction.py + git commit -m 'arc: wire enclosed region candidates' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-structural-enumeration-green.yml b/.github/workflows/arc-structural-enumeration-green.yml new file mode 100644 index 00000000..7a55e676 --- /dev/null +++ b/.github/workflows/arc-structural-enumeration-green.yml @@ -0,0 +1,71 @@ +name: ARC Structural Enumeration GREEN + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-structural-enumeration-green.yml' + +permissions: + contents: write + +jobs: + verify-green: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Run focused structural tests + run: python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_structural_enumeration.py | tee /tmp/focused.log + - name: Run all V2.2 tests + run: python -m pytest -q research/universal_core/holonomy_v2_2/tests | tee /tmp/v2-2.log + - name: Run frozen V2 integration tests + env: + PYTHONPATH: research + run: | + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py | tee /tmp/frozen.log + - name: Confirm frozen V2 bytes are unchanged + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Commit GREEN evidence + env: + GREEN_SOURCE_SHA: ${{ github.sha }} + GREEN_RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + evidence = { + 'schema_version': 1, + 'green_commit': os.environ['GREEN_SOURCE_SHA'], + 'green_run_id': os.environ['GREEN_RUN_ID'], + 'focused_output': Path('/tmp/focused.log').read_text(encoding='utf-8'), + 'v2_2_output': Path('/tmp/v2-2.log').read_text(encoding='utf-8'), + 'frozen_output': Path('/tmp/frozen.log').read_text(encoding='utf-8'), + 'frozen_byte_gate': True, + 'interfaces': [ + 'panel_programs', + 'region_programs', + 'component_select_programs', + ], + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + } + path = Path('research/universal_core/holonomy_v2_2/evidence/STRUCTURAL_ENUMERATION_GREEN.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/STRUCTURAL_ENUMERATION_GREEN.json + git commit -m 'arc: record structural enumeration GREEN evidence' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-structural-enumeration-red.yml b/.github/workflows/arc-structural-enumeration-red.yml new file mode 100644 index 00000000..6b0a0bc2 --- /dev/null +++ b/.github/workflows/arc-structural-enumeration-red.yml @@ -0,0 +1,69 @@ +name: ARC Structural Enumeration RED + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-structural-enumeration-red.yml' + +permissions: + contents: write + +jobs: + verify-red: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Require missing-enumerator failures + run: | + set +e + python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_structural_enumeration.py > /tmp/structural-red.log 2>&1 + status=$? + set -e + cat /tmp/structural-red.log + test "$status" -ne 0 + grep -F "has no attribute 'panel_programs'" /tmp/structural-red.log + grep -F "has no attribute 'region_programs'" /tmp/structural-red.log + grep -F "has no attribute 'component_select_programs'" /tmp/structural-red.log + - name: Commit RED evidence + env: + RED_SOURCE_SHA: ${{ github.sha }} + RED_RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + evidence = { + 'schema_version': 1, + 'red_commit': os.environ['RED_SOURCE_SHA'], + 'red_run_id': os.environ['RED_RUN_ID'], + 'test_command': 'python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_structural_enumeration.py', + 'expected_missing_interfaces': [ + 'panel_programs', + 'region_programs', + 'component_select_programs', + ], + 'failure_confirmed': True, + 'captured_log': Path('/tmp/structural-red.log').read_text(encoding='utf-8'), + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + } + path = Path('research/universal_core/holonomy_v2_2/evidence/STRUCTURAL_ENUMERATION_RED.json') + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/STRUCTURAL_ENUMERATION_RED.json + git commit -m 'arc: record structural enumeration RED evidence' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-structural-enumeration-wire.yml b/.github/workflows/arc-structural-enumeration-wire.yml new file mode 100644 index 00000000..bdf69d69 --- /dev/null +++ b/.github/workflows/arc-structural-enumeration-wire.yml @@ -0,0 +1,254 @@ +name: Wire ARC Structural Enumeration + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-structural-enumeration-wire.yml' + +permissions: + contents: write + +jobs: + wire-and-verify: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Apply guarded enumeration patch + run: | + python - <<'PY' + from pathlib import Path + + panel = Path('research/universal_core/holonomy_v2_2/panel_combine.py') + text = panel.read_text(encoding='utf-8') + old = '''def panel_candidates(demos: Sequence[tuple[Grid, Grid]]) -> tuple[Program, ...]: + target_colors = sorted( + {cell for _, target in demos for row in target for cell in row} + ) + if len(target_colors) != 2: + return () + candidates: list[Program] = [] + for axis in ("vertical", "horizontal"): + for gap in (0, 1, 2): + for background in _background_candidates(demos): + for predicate in _PREDICATES: + for true_color, false_color in ( + (target_colors[0], target_colors[1]), + (target_colors[1], target_colors[0]), + ): + program = { + "kind": "panel_boolean_combine", + "axis": axis, + "gap": gap, + "background": background, + "predicate": predicate, + "true_color": true_color, + "false_color": false_color, + } + if _fits(program, demos): + candidates.append(program) + return tuple(candidates) + ''' + old = '\n'.join(line[14:] if line.startswith(' ') else line for line in old.splitlines()) + '\n' + new = '''def panel_programs( + demos: Sequence[tuple[Grid, Grid]], + ) -> tuple[tuple[Program, ...], str | None]: + target_colors = sorted( + {cell for _, target in demos for row in target for cell in row} + ) + if len(target_colors) != 2: + return (), "TARGET_COLOR_CARDINALITY_NOT_TWO" + programs: list[Program] = [] + for axis in ("vertical", "horizontal"): + for gap in (0, 1, 2): + for background in _background_candidates(demos): + for predicate in _PREDICATES: + for true_color, false_color in ( + (target_colors[0], target_colors[1]), + (target_colors[1], target_colors[0]), + ): + programs.append({ + "kind": "panel_boolean_combine", + "axis": axis, + "gap": gap, + "background": background, + "predicate": predicate, + "true_color": true_color, + "false_color": false_color, + }) + if not programs: + return (), "NO_STRUCTURAL_PROGRAM" + return tuple(programs), None + + + def panel_candidates(demos: Sequence[tuple[Grid, Grid]]) -> tuple[Program, ...]: + programs, _ = panel_programs(demos) + return tuple(program for program in programs if _fits(program, demos)) + ''' + new = '\n'.join(line[10:] if line.startswith(' ') else line for line in new.splitlines()) + '\n' + if text.count(old) != 1: + raise SystemExit('panel enumeration anchor mismatch') + panel.write_text(text.replace(old, new, 1), encoding='utf-8') + + region = Path('research/universal_core/holonomy_v2_2/region_fill.py') + text = region.read_text(encoding='utf-8') + old = '''def region_candidates(demos: Sequence[tuple[Grid, Grid]]) -> tuple[Program, ...]: + target_colors = sorted({cell for _, target in demos for row in target for cell in row}) + candidates: list[Program] = [] + for background in _background_candidates(demos): + for connectivity in (4, 8): + for mode in ("fill_copy", "highlight", "solidify"): + for fill_color in target_colors: + program = { + "kind": "enclosed_region_fill", + "background": background, + "connectivity": connectivity, + "mode": mode, + "fill_color": fill_color, + } + if _fits(program, demos): + candidates.append(program) + return tuple(candidates) + ''' + old = '\n'.join(line[14:] if line.startswith(' ') else line for line in old.splitlines()) + '\n' + new = '''def region_programs( + demos: Sequence[tuple[Grid, Grid]], + ) -> tuple[tuple[Program, ...], str | None]: + backgrounds = _background_candidates(demos) + if not backgrounds: + return (), "NO_BACKGROUND_CANDIDATE" + target_colors = sorted({cell for _, target in demos for row in target for cell in row}) + programs: list[Program] = [] + for background in backgrounds: + for connectivity in (4, 8): + for mode in ("fill_copy", "highlight", "solidify"): + for fill_color in target_colors: + programs.append({ + "kind": "enclosed_region_fill", + "background": background, + "connectivity": connectivity, + "mode": mode, + "fill_color": fill_color, + }) + if not programs: + return (), "NO_STRUCTURAL_PROGRAM" + return tuple(programs), None + + + def region_candidates(demos: Sequence[tuple[Grid, Grid]]) -> tuple[Program, ...]: + programs, _ = region_programs(demos) + return tuple(program for program in programs if _fits(program, demos)) + ''' + new = '\n'.join(line[10:] if line.startswith(' ') else line for line in new.splitlines()) + '\n' + if text.count(old) != 1: + raise SystemExit('region enumeration anchor mismatch') + region.write_text(text.replace(old, new, 1), encoding='utf-8') + + component = Path('research/universal_core/holonomy_v2_2/component_select.py') + text = component.read_text(encoding='utf-8') + old = '''def component_select_candidates(demos: Sequence[tuple[Grid, Grid]]) -> tuple[Program, ...]: + candidates: list[Program] = [] + for background in _background_candidates(demos): + for connectivity in (4, 8): + for selector in ("minimum", "maximum"): + program = { + "kind": "component_area_select", + "background": background, + "connectivity": connectivity, + "selector": selector, + } + if _fits(program, demos): + candidates.append(program) + return tuple(candidates) + ''' + old = '\n'.join(line[14:] if line.startswith(' ') else line for line in old.splitlines()) + '\n' + new = '''def component_select_programs( + demos: Sequence[tuple[Grid, Grid]], + ) -> tuple[tuple[Program, ...], str | None]: + backgrounds = _background_candidates(demos) + if not backgrounds: + return (), "NO_BACKGROUND_CANDIDATE" + programs: list[Program] = [] + for background in backgrounds: + for connectivity in (4, 8): + for selector in ("minimum", "maximum"): + programs.append({ + "kind": "component_area_select", + "background": background, + "connectivity": connectivity, + "selector": selector, + }) + if not programs: + return (), "NO_STRUCTURAL_PROGRAM" + return tuple(programs), None + + + def component_select_candidates(demos: Sequence[tuple[Grid, Grid]]) -> tuple[Program, ...]: + programs, _ = component_select_programs(demos) + return tuple(program for program in programs if _fits(program, demos)) + ''' + new = '\n'.join(line[10:] if line.startswith(' ') else line for line in new.splitlines()) + '\n' + if text.count(old) != 1: + raise SystemExit('component enumeration anchor mismatch') + component.write_text(text.replace(old, new, 1), encoding='utf-8') + PY + git diff --check + - name: Run focused structural tests + run: python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_structural_enumeration.py | tee /tmp/focused.log + - name: Run all V2.2 tests + run: python -m pytest -q research/universal_core/holonomy_v2_2/tests | tee /tmp/v2-2.log + - name: Run frozen V2 integration tests + env: + PYTHONPATH: research + run: | + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py | tee /tmp/frozen.log + - name: Confirm frozen V2 bytes are unchanged + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Write evidence and commit implementation + env: + GREEN_SOURCE_SHA: ${{ github.sha }} + GREEN_RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + evidence = { + 'schema_version': 1, + 'workflow_source_commit': os.environ['GREEN_SOURCE_SHA'], + 'green_run_id': os.environ['GREEN_RUN_ID'], + 'focused_output': Path('/tmp/focused.log').read_text(encoding='utf-8'), + 'v2_2_output': Path('/tmp/v2-2.log').read_text(encoding='utf-8'), + 'frozen_output': Path('/tmp/frozen.log').read_text(encoding='utf-8'), + 'frozen_byte_gate': True, + 'interfaces': [ + 'panel_programs', + 'region_programs', + 'component_select_programs', + ], + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + } + path = Path('research/universal_core/holonomy_v2_2/evidence/STRUCTURAL_ENUMERATION_GREEN.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add \ + research/universal_core/holonomy_v2_2/panel_combine.py \ + research/universal_core/holonomy_v2_2/region_fill.py \ + research/universal_core/holonomy_v2_2/component_select.py \ + research/universal_core/holonomy_v2_2/evidence/STRUCTURAL_ENUMERATION_GREEN.json + git commit -m 'arc: expose structural grid hypotheses' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-symmetry-complete-corpus.yml b/.github/workflows/arc-symmetry-complete-corpus.yml new file mode 100644 index 00000000..429d4391 --- /dev/null +++ b/.github/workflows/arc-symmetry-complete-corpus.yml @@ -0,0 +1,371 @@ +name: ARC Symmetry Complete Mechanistic Corpus + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-symmetry-complete-corpus.yml' + +permissions: + contents: read + actions: read + +jobs: + training-corpus: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Download immutable benchmark corpus + uses: actions/download-artifact@v4 + with: + name: public-benchmarks-pinned-2026-08-04 + path: /tmp/pinned-public-benchmarks + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30929169392' + - name: Download immutable V2.2 baseline report + uses: actions/download-artifact@v4 + with: + name: arc-v2-2-training-e50965009efb96457fd769d97b42027f8a4e98c0 + path: /tmp/arc-v2-2-baseline + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30941837872' + - name: Verify V2.2 and frozen V2 gates + env: + PYTHONPATH: research + run: | + python -m pip install --disable-pip-version-check --quiet pytest + python -m pytest -q research/universal_core/holonomy_v2_2/tests + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py + git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Run all ARC training tasks with complete traces + run: | + mkdir -p /tmp/arc-v2-2-symmetry/traces + python - <<'PY' | tee /tmp/arc-v2-2-symmetry/console-summary.json + import json + import os + from pathlib import Path + + from research.universal_core.holonomy_v2.engine import HolonomyEngine + from research.universal_core.holonomy_v2.types import SearchConfig + from research.universal_core.holonomy_v2_2.mechanistic_trace import validate_trace + from research.universal_core.holonomy_v2_2.training_eval import run_training_corpus, write_report + from research.universal_core.public_benchmarks.suite import locate_vendor + + output = Path('/tmp/arc-v2-2-symmetry') + trace_dir = output / 'traces' + vendor = locate_vendor(Path('/tmp/pinned-public-benchmarks')) + report = run_training_corpus( + vendor, + HolonomyEngine(SearchConfig()), + run_label='arc-v2-2-symmetry-' + os.environ['GITHUB_SHA'][:12], + candidate_commit=os.environ['GITHUB_SHA'], + mechanistic_dir=trace_dir, + verify_parity=True, + ) + write_report(report, output / 'report.json') + + trace_paths = sorted( + path for path in trace_dir.glob('*.json') + if path.name != 'diagnosis.json' + ) + if len(trace_paths) != 1000: + raise SystemExit(f'expected 1000 task traces, found {len(trace_paths)}') + traces = {} + for path in trace_paths: + trace = json.loads(path.read_text(encoding='utf-8')) + validate_trace(trace) + traces[path.stem] = trace + if report['mechanistic']['trace_count'] != 1000: + raise SystemExit('aggregate trace count mismatch') + if report['mechanistic']['parity_count'] != 1000: + raise SystemExit('compact/mechanistic parity count mismatch') + + baseline = json.loads( + Path('/tmp/arc-v2-2-baseline/report.json').read_text(encoding='utf-8') + ) + baseline_rows = { + item['task_id']: int(item['metrics']['correct']) + for item in baseline['reports'] + } + current_rows = { + item['task_id']: int(item['metrics']['correct']) + for item in report['reports'] + } + lost_rows = { + task_id: {'baseline_correct': correct, 'current_correct': current_rows.get(task_id, 0)} + for task_id, correct in baseline_rows.items() + if current_rows.get(task_id, 0) < correct + } + gained_rows = { + task_id: {'baseline_correct': baseline_rows.get(task_id, 0), 'current_correct': correct} + for task_id, correct in current_rows.items() + if correct > baseline_rows.get(task_id, 0) + } + expected_gain_ids = [ + '0c786b71', '1e0a9b12', '1f876c06', '22168020', '22eb0ac0', + '2dee498d', '3906de3d', '3aa6fb7a', '3af2c5a8', '46442a0e', + '496994bd', '4c4377d9', '5614dbcf', '62c24649', '67e8384a', + '68b67ca3', '6d0aefbc', '6d75e8bb', '6fa7a44f', '746b3537', + '7953d61e', '7b7f7511', '7fe24cdd', '833dafe3', '8be77c9e', + 'a59b95c0', 'aabf363d', 'ac0a08a4', 'b230c067', 'b91ae062', + 'c9e6f938', 'ce8d95cc', 'd4b1c2b1', 'ded97339', 'e1baa8a4', + 'e729b7be', 'e7639916', 'eb5a1d5d', 'ed98d772', 'f25ffba3', + ] + expected_gains = { + task_id: {'baseline_correct': 0, 'current_correct': 1} + for task_id in expected_gain_ids + } + if lost_rows: + raise SystemExit('lost baseline-correct rows: ' + json.dumps(lost_rows, sort_keys=True)) + if gained_rows != expected_gains: + raise SystemExit( + 'unexpected gained rows: expected ' + + json.dumps(expected_gains, sort_keys=True) + + ' found ' + + json.dumps(gained_rows, sort_keys=True) + ) + + expected_metrics = { + 'rows': 1076, + 'correct': 99, + 'accepted': 99, + 'incorrect_attempts': 0, + 'failed': 0, + } + for key, expected in expected_metrics.items(): + if int(report['metrics'][key]) != expected: + raise SystemExit( + f'metric {key} changed: expected {expected}, found {report["metrics"][key]}' + ) + expected_status = { + 'ACCEPTED': 99, + 'AMBIGUOUS_PROGRAM': 2, + 'GRAMMAR_EXHAUSTED': 975, + } + if report['status_counts'] != expected_status: + raise SystemExit( + 'status counts changed: expected ' + + json.dumps(expected_status, sort_keys=True) + + ' found ' + + json.dumps(report['status_counts'], sort_keys=True) + ) + + symmetry_ids = ('496994bd', 'e729b7be', 'f25ffba3') + symmetry_details = {} + for task_id in symmetry_ids: + trace = traces[task_id] + final = [event for event in trace['events'] if event.get('event') == 'final_decision'][0] + successes = [ + event for event in trace['events'] + if event.get('event') == 'hidden_execution' + and event.get('candidate_kind') == 'reflective_symmetry_complete' + and event.get('outcome') == 'SUCCEEDED' + ] + if trace['terminal_status'] != 'ACCEPTED': + raise SystemExit(f'{task_id} did not transition to ACCEPTED') + if not successes: + raise SystemExit(f'{task_id} has no successful symmetry candidate') + selected = str(final.get('selected_candidate_id', '')) + if not selected.startswith('symmetry_complete:'): + raise SystemExit(f'{task_id} did not select symmetry completion: {selected}') + symmetry_details[task_id] = { + 'trace_digest': trace['trace_digest'], + 'successful_symmetry_candidates': len(successes), + 'selected_candidate_id': selected, + } + + b882 = traces['b8825c91'] + b882_classes = [ + event for event in b882['events'] + if event.get('event') == 'equivalence_class' + and any(str(candidate_id).startswith('symmetry_complete:') for candidate_id in event['candidate_ids']) + ] + if b882['terminal_status'] != 'AMBIGUOUS_PROGRAM': + raise SystemExit('b8825c91 did not remain ambiguity-safe') + if len(b882_classes) < 2: + raise SystemExit('b8825c91 lost symmetry disagreement') + + primitive_guards = { + 'aabf363d': 'marker_recolor:', + 'b230c067': 'component_outlier:', + '3aa6fb7a': 'bbox_complete:', + '6d75e8bb': 'bbox_complete:', + 'e7639916': 'bbox_complete:', + '4c4377d9': 'mirror_concat:', + '6d0aefbc': 'mirror_concat:', + '6fa7a44f': 'mirror_concat:', + '8be77c9e': 'mirror_concat:', + 'c9e6f938': 'mirror_concat:', + '0c786b71': 'quadrant_mosaic:', + '3af2c5a8': 'quadrant_mosaic:', + '46442a0e': 'quadrant_mosaic:', + '62c24649': 'quadrant_mosaic:', + '67e8384a': 'quadrant_mosaic:', + '7953d61e': 'quadrant_mosaic:', + '7fe24cdd': 'quadrant_mosaic:', + '833dafe3': 'quadrant_mosaic:', + 'ed98d772': 'quadrant_mosaic:', + 'a59b95c0': 'palette_repeat:', + 'ac0a08a4': 'palette_repeat:', + 'b91ae062': 'palette_repeat:', + 'd4b1c2b1': 'palette_repeat:', + '2dee498d': 'axis_collapse:', + '746b3537': 'axis_collapse:', + '7b7f7511': 'axis_collapse:', + 'ce8d95cc': 'axis_collapse:', + 'e1baa8a4': 'axis_collapse:', + 'eb5a1d5d': 'axis_collapse:', + '5614dbcf': 'block_reduce:', + '68b67ca3': 'block_reduce:', + '1e0a9b12': 'gravity_compact:', + '3906de3d': 'gravity_compact:', + '1f876c06': 'line_connect:', + '22168020': 'line_connect:', + '22eb0ac0': 'line_connect:', + 'ded97339': 'line_connect:', + } + prior_mechanisms = {} + for task_id, prefix in primitive_guards.items(): + trace = traces[task_id] + final = [event for event in trace['events'] if event.get('event') == 'final_decision'][0] + if trace['terminal_status'] != 'ACCEPTED': + raise SystemExit(f'{task_id} regressed') + selected = str(final.get('selected_candidate_id', '')) + if not selected.startswith(prefix): + raise SystemExit(f'{task_id} selected primitive changed: {selected}') + prior_mechanisms[task_id] = { + 'selected_candidate_id': selected, + 'trace_digest': trace['trace_digest'], + } + + old_ambiguous = traces['60b61512'] + old_classes = [event for event in old_ambiguous['events'] if event.get('event') == 'equivalence_class'] + bbox_classes = [ + event for event in old_classes + if any(str(candidate_id).startswith('bbox_complete:') for candidate_id in event['candidate_ids']) + ] + if old_ambiguous['terminal_status'] != 'AMBIGUOUS_PROGRAM': + raise SystemExit('60b61512 did not remain ambiguity-safe') + if len(bbox_classes) < 2: + raise SystemExit('60b61512 lost bounding-box disagreement') + + transitions = { + 'schema_version': 1, + 'candidate_commit': os.environ['GITHUB_SHA'], + 'baseline_commit': 'e50965009efb96457fd769d97b42027f8a4e98c0', + 'lost_rows': lost_rows, + 'gained_rows': gained_rows, + 'expected_gains_matched': True, + 'incorrect_attempts': int(report['metrics']['incorrect_attempts']), + 'new_ambiguous_tasks': ['b8825c91'], + } + mechanisms = { + 'schema_version': 1, + 'symmetry_tasks': symmetry_details, + 'prior_mechanisms': prior_mechanisms, + 'b8825c91': { + 'terminal_status': b882['terminal_status'], + 'trace_digest': b882['trace_digest'], + 'symmetry_equivalence_class_count': len(b882_classes), + 'safety_action': 'abstain_on_symmetry_disagreement', + }, + '60b61512': { + 'terminal_status': old_ambiguous['terminal_status'], + 'trace_digest': old_ambiguous['trace_digest'], + 'bbox_equivalence_class_count': len(bbox_classes), + 'safety_action': 'abstain_on_connectivity_disagreement', + }, + } + parity = { + 'schema_version': 1, + 'candidate_commit': os.environ['GITHUB_SHA'], + 'checked_tasks': 1000, + 'mismatches': 0, + 'lost_baseline_rows': lost_rows, + } + for name, data in ( + ('transitions.json', transitions), + ('mechanisms.json', mechanisms), + ('parity.json', parity), + ): + (output / name).write_text( + json.dumps(data, sort_keys=True, indent=2) + '\n', encoding='utf-8' + ) + + print(json.dumps({ + 'report_digest': report['report_digest'], + 'diagnosis_digest': report['mechanistic']['diagnosis_digest'], + 'evaluated_tasks': report['evaluated_tasks'], + 'trace_count': report['mechanistic']['trace_count'], + 'parity_count': report['mechanistic']['parity_count'], + 'metrics': report['metrics'], + 'status_counts': report['status_counts'], + 'transitions': transitions, + 'symmetry_tasks': symmetry_details, + 'b8825c91_classes': len(b882_classes), + }, sort_keys=True)) + PY + - name: Build checksum manifest after all writers close + run: | + python - <<'PY' + import hashlib + from pathlib import Path + + output = Path('/tmp/arc-v2-2-symmetry') + lines = [] + for path in sorted( + item for item in output.rglob('*') + if item.is_file() and item.name != 'SHA256SUMS' + ): + digest = hashlib.sha256(path.read_bytes()).hexdigest() + lines.append(f'{digest} {path.relative_to(output).as_posix()}') + (output / 'SHA256SUMS').write_text('\n'.join(lines) + '\n', encoding='utf-8') + PY + - name: Verify checksum manifest independently + working-directory: /tmp/arc-v2-2-symmetry + run: sha256sum -c SHA256SUMS + - name: Publish symmetry completion evidence + uses: actions/upload-artifact@v4 + with: + name: arc-v2-2-symmetry-complete-${{ github.sha }} + path: /tmp/arc-v2-2-symmetry + if-no-files-found: error + retention-days: 90 + compression-level: 6 + - name: Write run summary + run: | + python - <<'PY' + import json + from pathlib import Path + + root = Path('/tmp/arc-v2-2-symmetry') + report = json.loads((root / 'report.json').read_text()) + mechanisms = json.loads((root / 'mechanisms.json').read_text()) + lines = [ + '# ARC V2.2 reflective symmetry completion corpus', + '', + f"- Candidate: `{report['identity']['candidate_commit']}`", + f"- Report digest: `{report['report_digest']}`", + f"- Diagnosis digest: `{report['mechanistic']['diagnosis_digest']}`", + f"- Metrics: `{report['metrics']}`", + f"- Status counts: `{report['status_counts']}`", + f"- Symmetry gains: `{sorted(mechanisms['symmetry_tasks'])}`", + f"- b8825c91 action: `{mechanisms['b8825c91']['safety_action']}`", + f"- 60b61512 action: `{mechanisms['60b61512']['safety_action']}`", + '- Public ARC evaluation rerun: `no`', + ] + Path('/tmp/symmetry-summary.md').write_text('\n'.join(lines) + '\n', encoding='utf-8') + PY + cat /tmp/symmetry-summary.md >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/arc-symmetry-complete-module-green.yml b/.github/workflows/arc-symmetry-complete-module-green.yml new file mode 100644 index 00000000..a0593998 --- /dev/null +++ b/.github/workflows/arc-symmetry-complete-module-green.yml @@ -0,0 +1,71 @@ +name: ARC Symmetry Complete Module GREEN + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-symmetry-complete-module-green.yml' + +permissions: + contents: write + +jobs: + verify-module: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Run isolated symmetry tests + run: | + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_symmetry_complete_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_symmetry_complete.py::test_vertical_completion_reflects_columns \ + research/universal_core/holonomy_v2_2/tests/test_symmetry_complete.py::test_horizontal_completion_reflects_rows \ + research/universal_core/holonomy_v2_2/tests/test_symmetry_complete.py::test_rotate_180_completion_preserves_original_cells \ + research/universal_core/holonomy_v2_2/tests/test_symmetry_complete.py::test_diagonal_completion_uses_original_anchors_only \ + research/universal_core/holonomy_v2_2/tests/test_symmetry_complete.py::test_symmetry_completion_rejects_cross_color_overwrite \ + research/universal_core/holonomy_v2_2/tests/test_symmetry_complete.py::test_diagonal_completion_rejects_non_square_grid \ + research/universal_core/holonomy_v2_2/tests/test_symmetry_complete.py::test_program_derivation_returns_unique_horizontal_candidate \ + research/universal_core/holonomy_v2_2/tests/test_symmetry_complete.py::test_program_derivation_rejects_shape_change \ + research/universal_core/holonomy_v2_2/tests/test_symmetry_complete.py::test_program_derivation_skips_noop_demonstration \ + | tee /tmp/symmetry-module-green.log + - name: Confirm frozen bytes + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Commit module evidence + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + evidence = { + 'schema_version': 1, + 'source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': True, + 'test_count': 10, + 'test_output': Path('/tmp/symmetry-module-green.log').read_text(encoding='utf-8'), + 'expected_new_exact_tasks': ['496994bd', 'e729b7be', 'f25ffba3'], + 'expected_new_ambiguous_tasks': ['b8825c91'], + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'frozen_byte_gate': True, + 'public_arc_evaluation_rerun': False, + } + path = Path('research/universal_core/holonomy_v2_2/evidence/SYMMETRY_COMPLETE_MODULE_GREEN.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/SYMMETRY_COMPLETE_MODULE_GREEN.json + git commit -m 'arc: record symmetry completion module GREEN' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-symmetry-complete-red.yml b/.github/workflows/arc-symmetry-complete-red.yml new file mode 100644 index 00000000..77fbb36b --- /dev/null +++ b/.github/workflows/arc-symmetry-complete-red.yml @@ -0,0 +1,80 @@ +name: ARC Symmetry Complete RED + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-symmetry-complete-red.yml' + +permissions: + contents: write + +jobs: + verify-red: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Verify pinned symmetry fixtures + run: python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_symmetry_complete_fixtures.py + - name: Require missing symmetry module + run: | + set +e + python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_symmetry_complete.py > /tmp/symmetry-red.log 2>&1 + status=$? + set -e + cat /tmp/symmetry-red.log + test "$status" -ne 0 + grep -F "No module named 'research.universal_core.holonomy_v2_2.symmetry_complete'" /tmp/symmetry-red.log + - name: Verify frozen V2.1 byte identity + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Commit RED evidence + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + evidence = { + 'schema_version': 1, + 'source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'failure_confirmed': True, + 'fixture_integrity': True, + 'expected_failure': "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.symmetry_complete'", + 'expected_new_exact_tasks': ['496994bd', 'e729b7be', 'f25ffba3'], + 'expected_new_ambiguous_tasks': ['b8825c91'], + 'expected_cumulative_metrics': { + 'rows': 1076, + 'accepted': 99, + 'correct': 99, + 'incorrect_attempts': 0, + 'failed': 0, + }, + 'expected_status_counts': { + 'ACCEPTED': 99, + 'AMBIGUOUS_PROGRAM': 2, + 'GRAMMAR_EXHAUSTED': 975, + }, + 'captured_log': Path('/tmp/symmetry-red.log').read_text(encoding='utf-8'), + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'public_arc_evaluation_rerun': False, + } + path = Path('research/universal_core/holonomy_v2_2/evidence/SYMMETRY_COMPLETE_RED.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/SYMMETRY_COMPLETE_RED.json + git commit -m 'arc: record symmetry completion RED evidence' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-symmetry-complete-wire.yml b/.github/workflows/arc-symmetry-complete-wire.yml new file mode 100644 index 00000000..7bacf46d --- /dev/null +++ b/.github/workflows/arc-symmetry-complete-wire.yml @@ -0,0 +1,208 @@ +name: Wire ARC Symmetry Complete + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-symmetry-complete-wire.yml' + +permissions: + contents: write + +jobs: + wire-and-verify: + runs-on: ubuntu-latest + timeout-minutes: 50 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Apply guarded symmetry integration + run: | + python - <<'PY' + from pathlib import Path + + grid_path = Path('research/universal_core/holonomy_v2_2/grid_induction.py') + text = grid_path.read_text(encoding='utf-8') + import_anchor = 'from .line_connect import apply_line_connect\n' + if text.count(import_anchor) != 1 or 'from .symmetry_complete import' in text: + raise SystemExit('symmetry import anchor mismatch') + text = text.replace( + import_anchor, + import_anchor + 'from .symmetry_complete import apply_symmetry_complete\n', + 1, + ) + if text.count('_VERSION = "grid-v2.2-15"') != 1: + raise SystemExit('symmetry version anchor mismatch') + text = text.replace('_VERSION = "grid-v2.2-15"', '_VERSION = "grid-v2.2-16"', 1) + apply_anchor = ' if kind == "same_color_line_connect":\n' + if text.count(apply_anchor) != 1: + raise SystemExit('symmetry apply anchor mismatch') + text = text.replace( + apply_anchor, + ' if kind == "reflective_symmetry_complete":\n' + ' return apply_symmetry_complete(program, grid)\n' + + apply_anchor, + 1, + ) + grid_path.write_text(text, encoding='utf-8') + + runtime_path = Path('research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py') + text = runtime_path.read_text(encoding='utf-8') + import_anchor = 'from .line_connect import line_connect_programs\n' + if text.count(import_anchor) != 1 or 'symmetry_complete_programs' in text: + raise SystemExit('symmetry runtime import anchor mismatch') + text = text.replace( + import_anchor, + import_anchor + 'from .symmetry_complete import symmetry_complete_programs\n', + 1, + ) + derive_anchor = ' line_connect, line_reason = line_connect_programs(demos)\n' + if text.count(derive_anchor) != 1: + raise SystemExit('symmetry derivation anchor mismatch') + text = text.replace( + derive_anchor, + derive_anchor + ' symmetry_complete, symmetry_reason = symmetry_complete_programs(demos)\n', + 1, + ) + batch_anchor = ' ("line_connect", line_connect, line_reason),\n' + if text.count(batch_anchor) != 1: + raise SystemExit('symmetry batch anchor mismatch') + text = text.replace( + batch_anchor, + batch_anchor + ' ("symmetry_complete", symmetry_complete, symmetry_reason),\n', + 1, + ) + runtime_path.write_text(text, encoding='utf-8') + PY + python -m py_compile \ + research/universal_core/holonomy_v2_2/symmetry_complete.py \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py + git diff --check + - name: Run focused symmetry tests + id: focused + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_symmetry_complete_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_symmetry_complete.py \ + research/universal_core/holonomy_v2_2/tests/test_line_connect_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_line_connect.py \ + research/universal_core/holonomy_v2_2/tests/test_gravity_compact_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_gravity_compact.py \ + research/universal_core/holonomy_v2_2/tests/test_block_reduce_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_block_reduce.py \ + research/universal_core/holonomy_v2_2/tests/test_axis_collapse_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_axis_collapse.py \ + research/universal_core/holonomy_v2_2/tests/test_palette_repeat_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_palette_repeat.py \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic_fixtures.py \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py \ + research/universal_core/holonomy_v2_2/tests/test_mirror_concat.py \ + research/universal_core/holonomy_v2_2/tests/test_bbox_complete.py \ + research/universal_core/holonomy_v2_2/tests/test_component_outlier.py \ + research/universal_core/holonomy_v2_2/tests/test_marker_recolor.py \ + research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py \ + research/universal_core/holonomy_v2_2/tests/test_grid_induction.py::test_cell_scale_and_whole_grid_tile_are_distinct \ + | tee /tmp/symmetry-focused.log + - name: Run all V2.2 tests + id: cluster + if: always() + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q research/universal_core/holonomy_v2_2/tests | tee /tmp/symmetry-v2-2.log + - name: Run frozen V2 integration tests + id: frozen + if: always() + continue-on-error: true + env: + PYTHONPATH: research + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py | tee /tmp/symmetry-frozen.log + - name: Confirm frozen V2 bytes are unchanged + id: byte_gate + if: always() + continue-on-error: true + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Record result and commit only valid state + if: always() + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + FOCUSED_OUTCOME: ${{ steps.focused.outcome }} + CLUSTER_OUTCOME: ${{ steps.cluster.outcome }} + FROZEN_OUTCOME: ${{ steps.frozen.outcome }} + BYTE_OUTCOME: ${{ steps.byte_gate.outcome }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + outcomes = { + 'focused': os.environ['FOCUSED_OUTCOME'], + 'v2_2_cluster': os.environ['CLUSTER_OUTCOME'], + 'frozen_tests': os.environ['FROZEN_OUTCOME'], + 'frozen_byte_gate': os.environ['BYTE_OUTCOME'], + } + passed = all(value == 'success' for value in outcomes.values()) + evidence = { + 'schema_version': 1, + 'workflow_source_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': passed, + 'outcomes': outcomes, + 'focused_output': Path('/tmp/symmetry-focused.log').read_text(encoding='utf-8') if Path('/tmp/symmetry-focused.log').exists() else '', + 'v2_2_output': Path('/tmp/symmetry-v2-2.log').read_text(encoding='utf-8') if Path('/tmp/symmetry-v2-2.log').exists() else '', + 'frozen_output': Path('/tmp/symmetry-frozen.log').read_text(encoding='utf-8') if Path('/tmp/symmetry-frozen.log').exists() else '', + 'expected_new_exact_tasks': ['496994bd', 'e729b7be', 'f25ffba3'], + 'expected_new_ambiguous_tasks': ['b8825c91'], + 'expected_cumulative_metrics': { + 'rows': 1076, + 'correct': 99, + 'accepted': 99, + 'incorrect_attempts': 0, + 'failed': 0, + }, + 'expected_status_counts': { + 'ACCEPTED': 99, + 'AMBIGUOUS_PROGRAM': 2, + 'GRAMMAR_EXHAUSTED': 975, + }, + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + 'public_arc_evaluation_rerun': False, + } + name = 'SYMMETRY_COMPLETE_GREEN.json' if passed else 'SYMMETRY_COMPLETE_DIAGNOSTIC.json' + path = Path('research/universal_core/holonomy_v2_2/evidence') / name + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + Path('/tmp/symmetry-passed').write_text('1' if passed else '0', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + if [ "$(cat /tmp/symmetry-passed)" = "1" ]; then + git add \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py \ + research/universal_core/holonomy_v2_2/evidence/SYMMETRY_COMPLETE_GREEN.json + git commit -m 'arc: wire reflective symmetry completion' + else + git restore \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py + git add research/universal_core/holonomy_v2_2/evidence/SYMMETRY_COMPLETE_DIAGNOSTIC.json + git commit -m 'arc: record symmetry completion diagnostic failure' + fi + git push origin HEAD:universal-core-arc-training-v2-2 + test "$(cat /tmp/symmetry-passed)" = "1" diff --git a/.github/workflows/arc-symmetry-pin-fixtures.yml b/.github/workflows/arc-symmetry-pin-fixtures.yml new file mode 100644 index 00000000..f7943689 --- /dev/null +++ b/.github/workflows/arc-symmetry-pin-fixtures.yml @@ -0,0 +1,55 @@ +name: Pin ARC Symmetry Fixtures + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-symmetry-pin-fixtures.yml' + +permissions: + contents: write + actions: read + +jobs: + pin: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - name: Download immutable benchmark corpus + uses: actions/download-artifact@v4 + with: + name: public-benchmarks-pinned-2026-08-04 + path: /tmp/pinned-public-benchmarks + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30929169392' + - name: Copy exact immutable payloads + run: | + set -euo pipefail + probe=$(find /tmp/pinned-public-benchmarks -type f -path '*/arc_agi_2/data/training/496994bd.json' -print -quit) + test -n "$probe" + src=$(dirname "$probe") + dst=research/universal_core/holonomy_v2_2/tests/fixtures/symmetry_complete + mkdir -p "$dst" + for id in 496994bd b8825c91 e729b7be f25ffba3; do cp "$src/$id.json" "$dst/$id.json"; done + printf '%s %s\n' \ + ecc40bb943911e33cdfc1e668a5bb49b22ad55a662aa37d68ba57e884897dc25 "$dst/496994bd.json" \ + ccf892a7ba8797ab4c69b241d80dd389cde0b9c7b8c0667a8b3a273873c4d49d "$dst/b8825c91.json" \ + d2565e3c8e6482a1a3cec2ccf604032c223ee6d6f579dfee98a72b4cbdc12c99 "$dst/e729b7be.json" \ + 98bd5da81220ac72cf0a2d8ae093ed602066e14cb28ffce6db3110d91382067c "$dst/f25ffba3.json" \ + > /tmp/symmetry-fixtures.sha256 + sha256sum -c /tmp/symmetry-fixtures.sha256 + - name: Confirm frozen V2 bytes + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Commit fixtures + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/tests/fixtures/symmetry_complete + git commit -m 'test: pin reflective symmetry fixtures' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-training-corpus-v2-2.yml b/.github/workflows/arc-training-corpus-v2-2.yml new file mode 100644 index 00000000..730c7f88 --- /dev/null +++ b/.github/workflows/arc-training-corpus-v2-2.yml @@ -0,0 +1,346 @@ +name: ARC V2.2 Mechanistic Training Corpus + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-training-corpus-v2-2.yml' + - 'research/universal_core/holonomy_v2_2/**' + workflow_dispatch: + +permissions: + contents: read + actions: read + +concurrency: + group: arc-v2-2-training-${{ github.sha }} + cancel-in-progress: false + +jobs: + training-corpus: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Download immutable benchmark corpus + uses: actions/download-artifact@v4 + with: + name: public-benchmarks-pinned-2026-08-04 + path: /tmp/pinned-public-benchmarks + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30929169392' + - name: Download immutable V2.2 baseline report + uses: actions/download-artifact@v4 + with: + name: arc-v2-2-training-e50965009efb96457fd769d97b42027f8a4e98c0 + path: /tmp/arc-v2-2-baseline + github-token: ${{ github.token }} + repository: Chrisbryan17/MLE-agent + run-id: '30941837872' + - name: Verify V2.2 and frozen V2 gates + env: + PYTHONPATH: research + run: | + python -m pip install --disable-pip-version-check --quiet pytest + python -m pytest -q research/universal_core/holonomy_v2_2/tests + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py + git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Run all ARC training tasks with complete traces + run: | + mkdir -p /tmp/arc-v2-2-training/traces + python - <<'PY' | tee /tmp/arc-v2-2-training/console-summary.json + import json + import os + from pathlib import Path + + from research.universal_core.holonomy_v2.engine import HolonomyEngine + from research.universal_core.holonomy_v2.types import SearchConfig + from research.universal_core.holonomy_v2_2.mechanistic_trace import validate_trace + from research.universal_core.holonomy_v2_2.training_eval import ( + run_training_corpus, + write_report, + ) + from research.universal_core.public_benchmarks.suite import locate_vendor + + output = Path('/tmp/arc-v2-2-training') + trace_dir = output / 'traces' + vendor = locate_vendor(Path('/tmp/pinned-public-benchmarks')) + report = run_training_corpus( + vendor, + HolonomyEngine(SearchConfig()), + run_label='arc-v2-2-mechanistic-' + os.environ['GITHUB_SHA'][:12], + candidate_commit=os.environ['GITHUB_SHA'], + mechanistic_dir=trace_dir, + verify_parity=True, + ) + write_report(report, output / 'report.json') + + trace_paths = sorted( + path for path in trace_dir.glob('*.json') + if path.name != 'diagnosis.json' + ) + if len(trace_paths) != 1000: + raise SystemExit(f'expected 1000 task traces, found {len(trace_paths)}') + traces = {} + for path in trace_paths: + trace = json.loads(path.read_text(encoding='utf-8')) + validate_trace(trace) + traces[path.stem] = trace + if report['mechanistic']['trace_count'] != 1000: + raise SystemExit('aggregate trace count mismatch') + if report['mechanistic']['parity_count'] != 1000: + raise SystemExit('compact/mechanistic parity count mismatch') + + baseline = json.loads( + Path('/tmp/arc-v2-2-baseline/report.json').read_text(encoding='utf-8') + ) + baseline_rows = { + item['task_id']: int(item['metrics']['correct']) + for item in baseline['reports'] + } + current_rows = { + item['task_id']: int(item['metrics']['correct']) + for item in report['reports'] + } + lost_rows = { + task_id: { + 'baseline_correct': correct, + 'current_correct': current_rows.get(task_id, 0), + } + for task_id, correct in baseline_rows.items() + if current_rows.get(task_id, 0) < correct + } + gained_rows = { + task_id: { + 'baseline_correct': baseline_rows.get(task_id, 0), + 'current_correct': correct, + } + for task_id, correct in current_rows.items() + if correct > baseline_rows.get(task_id, 0) + } + expected_gains = { + 'aabf363d': { + 'baseline_correct': 0, + 'current_correct': 1, + } + } + if lost_rows: + raise SystemExit('lost baseline-correct rows: ' + json.dumps(lost_rows, sort_keys=True)) + if gained_rows != expected_gains: + raise SystemExit( + 'unexpected gained rows: expected ' + + json.dumps(expected_gains, sort_keys=True) + + ' found ' + + json.dumps(gained_rows, sort_keys=True) + ) + + metrics = report['metrics'] + expected_metrics = { + 'rows': 1076, + 'correct': 60, + 'accepted': 60, + 'incorrect_attempts': 0, + } + for key, expected in expected_metrics.items(): + if int(metrics[key]) != expected: + raise SystemExit( + f'metric {key} changed: expected {expected}, found {metrics[key]}' + ) + expected_status = { + 'ACCEPTED': 60, + 'EXECUTION_FAILED': 1, + 'GRAMMAR_EXHAUSTED': 1015, + } + if report['status_counts'] != expected_status: + raise SystemExit( + 'status counts changed: expected ' + + json.dumps(expected_status, sort_keys=True) + + ' found ' + + json.dumps(report['status_counts'], sort_keys=True) + ) + + aab_trace = traces['aabf363d'] + aab_events = aab_trace['events'] + color_failures = [ + event + for event in aab_events + if event.get('event') == 'hidden_execution' + and event.get('candidate_kind') == 'color_map' + and event.get('outcome') == 'FAILED' + and event.get('exception_type') == 'KeyError' + and 'unmapped grid cell' in event.get('exception_message', '') + ] + marker_successes = [ + event + for event in aab_events + if event.get('event') == 'hidden_execution' + and event.get('candidate_kind') == 'marker_recolor' + and event.get('outcome') == 'SUCCEEDED' + ] + aab_final = [ + event for event in aab_events + if event.get('event') == 'final_decision' + ][0] + if aab_trace['terminal_status'] != 'ACCEPTED': + raise SystemExit('aabf363d did not transition to ACCEPTED') + if len(color_failures) != 1: + raise SystemExit( + f'aabf363d expected one preserved color-map failure, found {len(color_failures)}' + ) + if len(marker_successes) != 2: + raise SystemExit( + f'aabf363d expected two marker successes, found {len(marker_successes)}' + ) + if not str(aab_final.get('selected_candidate_id', '')).startswith('marker_recolor:'): + raise SystemExit('aabf363d did not select a marker-recolor candidate') + + b230_trace = traces['b230c067'] + b230_events = b230_trace['events'] + rank_failures = [ + event + for event in b230_events + if event.get('event') == 'hidden_execution' + and event.get('candidate_kind') == 'component_rank_color' + and event.get('outcome') == 'FAILED' + and event.get('exception_type') == 'KeyError' + and 'unmapped component rank' in event.get('exception_message', '') + ] + if b230_trace['terminal_status'] != 'EXECUTION_FAILED': + raise SystemExit('b230c067 changed before its dedicated repair') + if len(rank_failures) != 2: + raise SystemExit( + f'b230c067 expected two preserved rank failures, found {len(rank_failures)}' + ) + + transitions = { + 'schema_version': 1, + 'candidate_commit': os.environ['GITHUB_SHA'], + 'baseline_commit': 'e50965009efb96457fd769d97b42027f8a4e98c0', + 'lost_rows': lost_rows, + 'gained_rows': gained_rows, + 'expected_gains_matched': True, + 'incorrect_attempts': int(metrics['incorrect_attempts']), + } + (output / 'transitions.json').write_text( + json.dumps(transitions, sort_keys=True, indent=2) + '\n', + encoding='utf-8', + ) + parity = { + 'schema_version': 1, + 'candidate_commit': os.environ['GITHUB_SHA'], + 'checked_tasks': 1000, + 'mismatches': 0, + 'baseline_commit': 'e50965009efb96457fd769d97b42027f8a4e98c0', + 'lost_baseline_rows': lost_rows, + } + (output / 'parity.json').write_text( + json.dumps(parity, sort_keys=True, indent=2) + '\n', + encoding='utf-8', + ) + root_causes = { + 'schema_version': 2, + 'aabf363d': { + 'terminal_status': aab_trace['terminal_status'], + 'trace_digest': aab_trace['trace_digest'], + 'preserved_failure': 'color_map hidden execution KeyError: unmapped grid cell', + 'preserved_failure_count': len(color_failures), + 'repair': 'dynamic singleton marker recolors all other foreground and is erased', + 'repair_success_count': len(marker_successes), + 'selected_candidate_id': aab_final['selected_candidate_id'], + }, + 'b230c067': { + 'terminal_status': b230_trace['terminal_status'], + 'trace_digest': b230_trace['trace_digest'], + 'mechanism': 'component_rank_color hidden execution KeyError: unmapped component rank', + 'matching_failure_count': len(rank_failures), + }, + } + (output / 'root-cause-confirmation.json').write_text( + json.dumps(root_causes, sort_keys=True, indent=2) + '\n', + encoding='utf-8', + ) + + print(json.dumps({ + 'report_digest': report['report_digest'], + 'diagnosis_digest': report['mechanistic']['diagnosis_digest'], + 'evaluated_tasks': report['evaluated_tasks'], + 'trace_count': report['mechanistic']['trace_count'], + 'parity_count': report['mechanistic']['parity_count'], + 'metrics': report['metrics'], + 'status_counts': report['status_counts'], + 'transitions': transitions, + 'root_causes': root_causes, + }, sort_keys=True)) + PY + - name: Build checksum manifest after all writers close + run: | + python - <<'PY' + import hashlib + from pathlib import Path + + output = Path('/tmp/arc-v2-2-training') + manifest_lines = [] + for path in sorted( + item for item in output.rglob('*') + if item.is_file() and item.name != 'SHA256SUMS' + ): + digest = hashlib.sha256(path.read_bytes()).hexdigest() + manifest_lines.append(f'{digest} {path.relative_to(output).as_posix()}') + (output / 'SHA256SUMS').write_text( + '\n'.join(manifest_lines) + '\n', + encoding='utf-8', + ) + PY + - name: Verify checksum manifest independently + working-directory: /tmp/arc-v2-2-training + run: sha256sum -c SHA256SUMS + - name: Publish mechanistic training evidence + uses: actions/upload-artifact@v4 + with: + name: arc-v2-2-mechanistic-${{ github.sha }} + path: /tmp/arc-v2-2-training + if-no-files-found: error + retention-days: 90 + compression-level: 6 + - name: Write training summary + run: | + python - <<'PY' + import json + from pathlib import Path + + report = json.loads(Path('/tmp/arc-v2-2-training/report.json').read_text()) + root_causes = json.loads( + Path('/tmp/arc-v2-2-training/root-cause-confirmation.json').read_text() + ) + transitions = json.loads( + Path('/tmp/arc-v2-2-training/transitions.json').read_text() + ) + lines = [ + '# ARC V2.2 mechanistic training corpus', + '', + f"- Candidate: `{report['identity']['candidate_commit']}`", + f"- Report digest: `{report['report_digest']}`", + f"- Diagnosis digest: `{report['mechanistic']['diagnosis_digest']}`", + f"- Tasks/traces/parity: `{report['evaluated_tasks']}` / `{report['mechanistic']['trace_count']}` / `{report['mechanistic']['parity_count']}`", + f"- Metrics: `{report['metrics']}`", + f"- Status counts: `{report['status_counts']}`", + f"- Gains: `{transitions['gained_rows']}`", + f"- aabf363d repair: `{root_causes['aabf363d']['repair']}`", + f"- b230c067 remaining mechanism: `{root_causes['b230c067']['mechanism']}`", + '- Public ARC evaluation rerun: `no`', + ] + Path('/tmp/arc-v2-2-training-summary.md').write_text( + '\n'.join(lines) + '\n', + encoding='utf-8', + ) + PY + cat /tmp/arc-v2-2-training-summary.md >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/arc-training-mechanistic-green.yml b/.github/workflows/arc-training-mechanistic-green.yml new file mode 100644 index 00000000..f81e10b0 --- /dev/null +++ b/.github/workflows/arc-training-mechanistic-green.yml @@ -0,0 +1,107 @@ +name: ARC Training Mechanistic GREEN + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-training-mechanistic-green.yml' + +permissions: + contents: write + +jobs: + verify-green: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Run focused corpus trace tests + id: focused + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_trace_diagnosis.py \ + research/universal_core/holonomy_v2_2/tests/test_training_mechanistic.py \ + research/universal_core/holonomy_v2_2/tests/test_training_eval.py \ + | tee /tmp/focused.log + - name: Run all V2.2 tests + id: cluster + if: always() + continue-on-error: true + run: | + set -o pipefail + python -m pytest -q research/universal_core/holonomy_v2_2/tests | tee /tmp/v2-2.log + - name: Run frozen V2 integration tests + id: frozen + if: always() + continue-on-error: true + env: + PYTHONPATH: research + run: | + set -o pipefail + python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py | tee /tmp/frozen.log + - name: Confirm frozen V2 bytes are unchanged + id: byte_gate + if: always() + continue-on-error: true + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 + - name: Record GREEN or diagnostics + if: always() + env: + SOURCE_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + FOCUSED_OUTCOME: ${{ steps.focused.outcome }} + CLUSTER_OUTCOME: ${{ steps.cluster.outcome }} + FROZEN_OUTCOME: ${{ steps.frozen.outcome }} + BYTE_OUTCOME: ${{ steps.byte_gate.outcome }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + outcomes = { + 'focused': os.environ['FOCUSED_OUTCOME'], + 'v2_2_cluster': os.environ['CLUSTER_OUTCOME'], + 'frozen_tests': os.environ['FROZEN_OUTCOME'], + 'frozen_byte_gate': os.environ['BYTE_OUTCOME'], + } + passed = all(value == 'success' for value in outcomes.values()) + evidence = { + 'schema_version': 1, + 'green_commit': os.environ['SOURCE_SHA'], + 'run_id': os.environ['RUN_ID'], + 'passed': passed, + 'outcomes': outcomes, + 'focused_output': Path('/tmp/focused.log').read_text(encoding='utf-8') if Path('/tmp/focused.log').exists() else '', + 'v2_2_output': Path('/tmp/v2-2.log').read_text(encoding='utf-8') if Path('/tmp/v2-2.log').exists() else '', + 'frozen_output': Path('/tmp/frozen.log').read_text(encoding='utf-8') if Path('/tmp/frozen.log').exists() else '', + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + } + name = 'TRAINING_MECHANISTIC_GREEN.json' if passed else 'TRAINING_MECHANISTIC_DIAGNOSTIC.json' + path = Path('research/universal_core/holonomy_v2_2/evidence') / name + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + Path('/tmp/passed').write_text('1' if passed else '0', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + if [ "$(cat /tmp/passed)" = "1" ]; then + git add research/universal_core/holonomy_v2_2/evidence/TRAINING_MECHANISTIC_GREEN.json + git commit -m 'arc: record mechanistic corpus GREEN evidence' + else + git add research/universal_core/holonomy_v2_2/evidence/TRAINING_MECHANISTIC_DIAGNOSTIC.json + git commit -m 'arc: record mechanistic corpus diagnostic failure' + fi + git push origin HEAD:universal-core-arc-training-v2-2 + test "$(cat /tmp/passed)" = "1" diff --git a/.github/workflows/arc-training-mechanistic-red.yml b/.github/workflows/arc-training-mechanistic-red.yml new file mode 100644 index 00000000..b229914d --- /dev/null +++ b/.github/workflows/arc-training-mechanistic-red.yml @@ -0,0 +1,65 @@ +name: ARC Training Mechanistic RED + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-training-mechanistic-red.yml' + +permissions: + contents: write + +jobs: + verify-red: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Require missing diagnosis interface + run: | + set +e + python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_trace_diagnosis.py \ + research/universal_core/holonomy_v2_2/tests/test_training_mechanistic.py \ + > /tmp/training-mechanistic-red.log 2>&1 + status=$? + set -e + cat /tmp/training-mechanistic-red.log + test "$status" -ne 0 + grep -F "No module named 'research.universal_core.holonomy_v2_2.trace_diagnosis'" /tmp/training-mechanistic-red.log + - name: Commit RED evidence + env: + RED_SOURCE_SHA: ${{ github.sha }} + RED_RUN_ID: ${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + evidence = { + 'schema_version': 1, + 'red_commit': os.environ['RED_SOURCE_SHA'], + 'red_run_id': os.environ['RED_RUN_ID'], + 'test_command': 'python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_trace_diagnosis.py research/universal_core/holonomy_v2_2/tests/test_training_mechanistic.py', + 'expected_failure': "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.trace_diagnosis'", + 'failure_confirmed': True, + 'captured_log': Path('/tmp/training-mechanistic-red.log').read_text(encoding='utf-8'), + 'frozen_core_commit': '1d7c489bcdbff755d638b35ad47ce62bd4fe829d', + } + path = Path('research/universal_core/holonomy_v2_2/evidence/TRAINING_MECHANISTIC_RED.json') + path.write_text(json.dumps(evidence, sort_keys=True, indent=2) + '\n', encoding='utf-8') + PY + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add research/universal_core/holonomy_v2_2/evidence/TRAINING_MECHANISTIC_RED.json + git commit -m 'arc: record mechanistic corpus RED evidence' + git push origin HEAD:universal-core-arc-training-v2-2 diff --git a/.github/workflows/arc-training-v2-2.yml b/.github/workflows/arc-training-v2-2.yml new file mode 100644 index 00000000..836596be --- /dev/null +++ b/.github/workflows/arc-training-v2-2.yml @@ -0,0 +1,39 @@ +name: ARC Training V2.2 + +on: + push: + branches: + - universal-core-arc-training-v2-2 + paths: + - '.github/workflows/arc-training-v2-2.yml' + - 'research/universal_core/holonomy_v2_2/**' + workflow_dispatch: + +permissions: + contents: read + +jobs: + grid-tests: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + ref: universal-core-arc-training-v2-2 + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install test dependency + run: python -m pip install --disable-pip-version-check --quiet pytest + - name: Run V2.2 grid tests + run: python -m pytest -q research/universal_core/holonomy_v2_2/tests + - name: Run frozen V2 integration cluster + env: + PYTHONPATH: research + run: >- + python -m pytest -q + research/universal_core/holonomy_v2/tests/test_blind_adapter.py + research/universal_core/holonomy_v2/tests/test_engine.py + - name: Confirm frozen V2 bytes are unchanged + run: git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 diff --git a/docs/evidence/arc_v2_2/FULL_COMPONENT_OUTLIER_RUN_0008.json b/docs/evidence/arc_v2_2/FULL_COMPONENT_OUTLIER_RUN_0008.json new file mode 100644 index 00000000..90715e11 --- /dev/null +++ b/docs/evidence/arc_v2_2/FULL_COMPONENT_OUTLIER_RUN_0008.json @@ -0,0 +1,118 @@ +{ + "artifact": { + "digest": "sha256:96a6b3aabbeae4742e4df129fb76a1ad6fe3ce52a2580d9b1d35def4ee6e215d", + "expired": false, + "id": 8909609720, + "name": "arc-v2-2-outlier-0439bd2e8611bc2ad0549249b417c01034ce546e", + "size_in_bytes": 18028856 + }, + "conclusion": "success", + "created_at": "2026-08-04T21:22:14Z", + "event": "push", + "failed_log_tail": "", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "jobs": [ + { + "completed_at": "2026-08-04T21:26:38Z", + "conclusion": "success", + "id": 92136212802, + "name": "training-corpus", + "started_at": "2026-08-04T21:22:23Z", + "status": "completed", + "steps": [ + { + "conclusion": "success", + "name": "Set up job", + "number": 1, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Run actions/checkout@v4", + "number": 2, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Run actions/setup-python@v5", + "number": 3, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Download immutable benchmark corpus", + "number": 4, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Download immutable V2.2 baseline report", + "number": 5, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Verify V2.2 and frozen V2 gates", + "number": 6, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Run all ARC training tasks with complete traces", + "number": 7, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Build checksum manifest after all writers close", + "number": 8, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Verify checksum manifest independently", + "number": 9, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish component-outlier evidence", + "number": 10, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Write run summary", + "number": 11, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Post Run actions/setup-python@v5", + "number": 21, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Post Run actions/checkout@v4", + "number": 22, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Complete job", + "number": 23, + "status": "completed" + } + ] + } + ], + "public_arc_evaluation_rerun": false, + "run_id": 30952042813, + "run_number": 1, + "schema_version": 1, + "status": "completed", + "target_commit": "0439bd2e8611bc2ad0549249b417c01034ce546e", + "updated_at": "2026-08-04T21:26:38Z", + "workflow_name": "ARC Component Outlier Mechanistic Corpus" +} diff --git a/docs/evidence/arc_v2_2/FULL_MARKER_RECOLOR_RUN_0007.json b/docs/evidence/arc_v2_2/FULL_MARKER_RECOLOR_RUN_0007.json new file mode 100644 index 00000000..ac56bedc --- /dev/null +++ b/docs/evidence/arc_v2_2/FULL_MARKER_RECOLOR_RUN_0007.json @@ -0,0 +1,118 @@ +{ + "artifact": { + "digest": "sha256:eb62c32eddeaf04817ee17d076469160ca9a653d6fd910b8d02844754c3f8edf", + "expired": false, + "id": 8909206467, + "name": "arc-v2-2-mechanistic-bbb192edd1706b204428041015779c28ded0c4b7", + "size_in_bytes": 17989697 + }, + "conclusion": "success", + "created_at": "2026-08-04T21:08:41Z", + "event": "push", + "failed_log_tail": "", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "jobs": [ + { + "completed_at": "2026-08-04T21:12:16Z", + "conclusion": "success", + "id": 92132961395, + "name": "training-corpus", + "started_at": "2026-08-04T21:08:44Z", + "status": "completed", + "steps": [ + { + "conclusion": "success", + "name": "Set up job", + "number": 1, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Run actions/checkout@v4", + "number": 2, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Run actions/setup-python@v5", + "number": 3, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Download immutable benchmark corpus", + "number": 4, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Download immutable V2.2 baseline report", + "number": 5, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Verify V2.2 and frozen V2 gates", + "number": 6, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Run all ARC training tasks with complete traces", + "number": 7, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Build checksum manifest after all writers close", + "number": 8, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Verify checksum manifest independently", + "number": 9, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish mechanistic training evidence", + "number": 10, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Write training summary", + "number": 11, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Post Run actions/setup-python@v5", + "number": 21, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Post Run actions/checkout@v4", + "number": 22, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Complete job", + "number": 23, + "status": "completed" + } + ] + } + ], + "public_arc_evaluation_rerun": false, + "run_id": 30951065593, + "run_number": 39, + "schema_version": 1, + "status": "completed", + "target_commit": "bbb192edd1706b204428041015779c28ded0c4b7", + "updated_at": "2026-08-04T21:12:17Z", + "workflow_name": "ARC V2.2 Mechanistic Training Corpus" +} diff --git a/docs/evidence/arc_v2_2/FULL_MECHANISTIC_RUN_0006.json b/docs/evidence/arc_v2_2/FULL_MECHANISTIC_RUN_0006.json new file mode 100644 index 00000000..e8db9cdc --- /dev/null +++ b/docs/evidence/arc_v2_2/FULL_MECHANISTIC_RUN_0006.json @@ -0,0 +1,118 @@ +{ + "artifact": { + "digest": "sha256:83b835a5e58293a33d554875db1495061c1a3dd1dc004d1af79b11210269cbc5", + "expired": false, + "id": 8908817384, + "name": "arc-v2-2-mechanistic-29d64251ebfb9465e0d60b89b0fc42c67fbc2dc2", + "size_in_bytes": 16891645 + }, + "conclusion": "success", + "created_at": "2026-08-04T20:55:13Z", + "event": "push", + "failed_log_tail": "", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "jobs": [ + { + "completed_at": "2026-08-04T20:59:10Z", + "conclusion": "success", + "id": 92129581413, + "name": "training-corpus", + "started_at": "2026-08-04T20:55:15Z", + "status": "completed", + "steps": [ + { + "conclusion": "success", + "name": "Set up job", + "number": 1, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Run actions/checkout@v4", + "number": 2, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Run actions/setup-python@v5", + "number": 3, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Download immutable benchmark corpus", + "number": 4, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Download immutable V2.2 baseline report", + "number": 5, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Verify V2.2 and frozen V2 gates", + "number": 6, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Run all ARC training tasks with complete traces", + "number": 7, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Build checksum manifest after all writers close", + "number": 8, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Verify checksum manifest independently", + "number": 9, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Publish mechanistic training evidence", + "number": 10, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Write training summary", + "number": 11, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Post Run actions/setup-python@v5", + "number": 21, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Post Run actions/checkout@v4", + "number": 22, + "status": "completed" + }, + { + "conclusion": "success", + "name": "Complete job", + "number": 23, + "status": "completed" + } + ] + } + ], + "public_arc_evaluation_rerun": false, + "run_id": 30950052575, + "run_number": 36, + "schema_version": 1, + "status": "completed", + "target_commit": "29d64251ebfb9465e0d60b89b0fc42c67fbc2dc2", + "updated_at": "2026-08-04T20:59:11Z", + "workflow_name": "ARC V2.2 Mechanistic Training Corpus" +} diff --git a/docs/evidence/arc_v2_2/TRAINING_AXIS_COLLAPSE_0013.json b/docs/evidence/arc_v2_2/TRAINING_AXIS_COLLAPSE_0013.json new file mode 100644 index 00000000..f96ac760 --- /dev/null +++ b/docs/evidence/arc_v2_2/TRAINING_AXIS_COLLAPSE_0013.json @@ -0,0 +1,161 @@ +{ + "accepted_selected_kind_counts": { + "axis_duplicate_collapse": 6, + "bounding_box_complete": 4, + "color_map": 3, + "component_area_outlier_color": 1, + "component_area_select": 5, + "component_rank_color": 10, + "dihedral_quadrant_mosaic": 9, + "enclosed_region_fill": 3, + "marker_recolor": 1, + "mirror_concatenate": 5, + "palette_cardinality_repeat": 4, + "panel_boolean_combine": 20, + "reflect": 4, + "rotate": 3, + "scale": 3, + "tile": 1 + }, + "artifact": { + "digest": "sha256:638869a8bf111e6c20c7fca15b5c6495c63372ca95c3da4ffa5665543efc1eef", + "expired": false, + "id": 8918537540, + "name": "arc-v2-2-axis-collapse-f16af6c15f76d37e35ad4cb2b59c0ad79bab27dc", + "size_in_bytes": 35875933 + }, + "candidate_commit": "f16af6c15f76d37e35ad4cb2b59c0ad79bab27dc", + "diagnosis_digest": "5f07fde156a669706ef64832b6eafec8c6ac3f9df698833f53ed397aab64df78", + "frozen_core_byte_gate": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "hidden_failure_counts": { + "color_map": {"KeyError": 1}, + "component_area_outlier_color": {"ValueError": 1}, + "component_rank_color": {"KeyError": 3} + }, + "independent_verification": { + "checksum_manifest_entries": 1006, + "checksum_manifest_verified": true, + "compact_mechanistic_parity": 1000, + "evaluated_validator_commit": "f16af6c15f76d37e35ad4cb2b59c0ad79bab27dc", + "event_count_recomputed": 627020, + "parallel_validator_workers": 8, + "task_traces_validated": 1000 + }, + "mechanisms": { + "60b61512": { + "bbox_equivalence_class_count": 2, + "safety_action": "abstain_on_connectivity_disagreement", + "terminal_status": "AMBIGUOUS_PROGRAM", + "trace_digest": "c725eeb7bd783db3b405c3ce160dab7c017f4f1c5c199d0ff80a63084124ac1c" + }, + "axis_tasks": { + "2dee498d": { + "selected_candidate_id": "axis_collapse:0002", + "successful_axis_candidates": 2, + "trace_digest": "b5a58a3f5971871b0fa0669b8de73a266d602d6fe3b69f263001de46699480fa" + }, + "746b3537": { + "selected_candidate_id": "axis_collapse:0002", + "successful_axis_candidates": 1, + "trace_digest": "d6510a6c49fae9f133cce3c6aa4a7cae24871cbbd976d92ee8ae0d4cd134b9d3" + }, + "7b7f7511": { + "selected_candidate_id": "axis_collapse:0002", + "successful_axis_candidates": 1, + "trace_digest": "e71ef69befdd3b2569d17c1c8539cf0ced706f31fa641a599918ecf999cb27fa" + }, + "ce8d95cc": { + "selected_candidate_id": "axis_collapse:0002", + "successful_axis_candidates": 1, + "trace_digest": "10bd6c85f11df94d6b318730f3c69931e4cee2b7d8750812b2d83c89eb5fd619" + }, + "e1baa8a4": { + "selected_candidate_id": "axis_collapse:0002", + "successful_axis_candidates": 2, + "trace_digest": "e856f3f0f2d4919da879924fd61fc21f788bb0c19e13ae6d42d017c8a9bf54ac" + }, + "eb5a1d5d": { + "selected_candidate_id": "axis_collapse:0002", + "successful_axis_candidates": 1, + "trace_digest": "1d758b0d8af8839ea7e58cb6e1cde278920993dd8da802cc57f15281baa82d7f" + } + }, + "ownership_boundary": "NO_DUPLICATE_AXIS_REDUCTION", + "schema_version": 1 + }, + "mechanistic": { + "diagnosis_digest": "5f07fde156a669706ef64832b6eafec8c6ac3f9df698833f53ed397aab64df78", + "diagnosis_path": "diagnosis.json", + "parity_checked": true, + "parity_count": 1000, + "trace_count": 1000, + "trace_schema": "mechanistic-grid-trace-v1" + }, + "metrics": { + "abstained": 988, + "accepted": 88, + "attempted_accuracy": 1.0, + "correct": 88, + "coverage": 0.08178438661710037, + "failed": 0, + "families": 1000, + "incorrect_attempts": 0, + "raw_accuracy": 0.08178438661710037, + "rows": 1076 + }, + "public_arc_evaluation_rerun": false, + "report_digest": "b8d3295e06b25172b2541699fa7281f0f9ff0bf0056f8b0bfdf69ae8a2edde8e", + "schema_version": 1, + "status_counts": { + "ACCEPTED": 88, + "AMBIGUOUS_PROGRAM": 1, + "GRAMMAR_EXHAUSTED": 987 + }, + "total_event_count": 627020, + "transitions": { + "baseline_commit": "e50965009efb96457fd769d97b42027f8a4e98c0", + "candidate_commit": "f16af6c15f76d37e35ad4cb2b59c0ad79bab27dc", + "expected_gains_matched": true, + "gained_rows": { + "0c786b71": {"baseline_correct": 0, "current_correct": 1}, + "2dee498d": {"baseline_correct": 0, "current_correct": 1}, + "3aa6fb7a": {"baseline_correct": 0, "current_correct": 1}, + "3af2c5a8": {"baseline_correct": 0, "current_correct": 1}, + "46442a0e": {"baseline_correct": 0, "current_correct": 1}, + "4c4377d9": {"baseline_correct": 0, "current_correct": 1}, + "62c24649": {"baseline_correct": 0, "current_correct": 1}, + "67e8384a": {"baseline_correct": 0, "current_correct": 1}, + "6d0aefbc": {"baseline_correct": 0, "current_correct": 1}, + "6d75e8bb": {"baseline_correct": 0, "current_correct": 1}, + "6fa7a44f": {"baseline_correct": 0, "current_correct": 1}, + "746b3537": {"baseline_correct": 0, "current_correct": 1}, + "7953d61e": {"baseline_correct": 0, "current_correct": 1}, + "7b7f7511": {"baseline_correct": 0, "current_correct": 1}, + "7fe24cdd": {"baseline_correct": 0, "current_correct": 1}, + "833dafe3": {"baseline_correct": 0, "current_correct": 1}, + "8be77c9e": {"baseline_correct": 0, "current_correct": 1}, + "a59b95c0": {"baseline_correct": 0, "current_correct": 1}, + "aabf363d": {"baseline_correct": 0, "current_correct": 1}, + "ac0a08a4": {"baseline_correct": 0, "current_correct": 1}, + "b230c067": {"baseline_correct": 0, "current_correct": 1}, + "b91ae062": {"baseline_correct": 0, "current_correct": 1}, + "c9e6f938": {"baseline_correct": 0, "current_correct": 1}, + "ce8d95cc": {"baseline_correct": 0, "current_correct": 1}, + "d4b1c2b1": {"baseline_correct": 0, "current_correct": 1}, + "e1baa8a4": {"baseline_correct": 0, "current_correct": 1}, + "e7639916": {"baseline_correct": 0, "current_correct": 1}, + "eb5a1d5d": {"baseline_correct": 0, "current_correct": 1}, + "ed98d772": {"baseline_correct": 0, "current_correct": 1} + }, + "incorrect_attempts": 0, + "lost_rows": {}, + "schema_version": 1 + }, + "workflow": { + "conclusion": "success", + "job_id": 92210948345, + "name": "ARC Axis Collapse Mechanistic Corpus", + "run_id": 30976323506 + } +} diff --git a/docs/evidence/arc_v2_2/TRAINING_BLOCK_REDUCE_0014.json b/docs/evidence/arc_v2_2/TRAINING_BLOCK_REDUCE_0014.json new file mode 100644 index 00000000..674f57f3 --- /dev/null +++ b/docs/evidence/arc_v2_2/TRAINING_BLOCK_REDUCE_0014.json @@ -0,0 +1,68 @@ +{ + "artifact": { + "artifact_id": 9253566568, + "digest": "sha256:2aa21f60a23ae24d4059bf123b09d06ec21f0d588ce6c1e8e6c905fbd08eff08", + "name": "arc-v2-2-block-reduce-c3c4efaef66e2ff6b8f83c5381acdec13cb31f5e", + "run_id": 31910432846, + "sha256sums_digest": "9c9f892e31dea49088494c386f2fb019344edd444cf56c37e6cadfd44589150d" + }, + "candidate_commit": "c3c4efaef66e2ff6b8f83c5381acdec13cb31f5e", + "diagnosis_digest": "db2d98a3fe3b8530dccc532e8c942824143ea31f62d3d969bd31a4bcb2b0f818", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "independent_validation": { + "artifact_digest_matches_github": true, + "checksum_entries": 1006, + "checksum_failures": 0, + "compact_mechanistic_parity_tasks": 1000, + "lost_baseline_rows": {}, + "manifest_verified": true, + "trace_files": 1000 + }, + "mechanisms": { + "5614dbcf": { + "selected_candidate_id": "block_reduce:0000", + "successful_block_candidates": 2, + "trace_digest": "0ca45d8611d3f7fe7e7656edbbbc6b8afda72024a10b9736372a438f80e87246" + }, + "60b61512": { + "bbox_equivalence_class_count": 2, + "safety_action": "abstain_on_connectivity_disagreement", + "terminal_status": "AMBIGUOUS_PROGRAM", + "trace_digest": "4102acbc810aa07bc1fd1ef0c2398545ee57b0857d07fce580ae8ab540259c62" + }, + "68b67ca3": { + "selected_candidate_id": "block_reduce:0000", + "successful_block_candidates": 1, + "trace_digest": "ac7e0e5f201f352b848a0923fa0c891fafe08b024072f57b6f8149c324366742" + } + }, + "metrics": { + "abstained": 986, + "accepted": 90, + "attempted_accuracy": 1.0, + "correct": 90, + "coverage": 0.08364312267657993, + "failed": 0, + "families": 1000, + "incorrect_attempts": 0, + "raw_accuracy": 0.08364312267657993, + "rows": 1076 + }, + "public_arc_evaluation_rerun": false, + "report_digest": "fde61e7533bcc5bbce77fcc9d76a6a1b78a82b44a88ab52c9ad9a484161998dc", + "schema_version": 1, + "status_counts": { + "ACCEPTED": 90, + "AMBIGUOUS_PROGRAM": 1, + "GRAMMAR_EXHAUSTED": 985 + }, + "training_transition": { + "cumulative_gain_tasks": 31, + "expected_gains_matched": true, + "new_exact_tasks": [ + "5614dbcf", + "68b67ca3" + ], + "new_exact_rows": 2 + } +} diff --git a/docs/evidence/arc_v2_2/TRAINING_COMPONENT_OUTLIER_0008.json b/docs/evidence/arc_v2_2/TRAINING_COMPONENT_OUTLIER_0008.json new file mode 100644 index 00000000..608fa6c1 --- /dev/null +++ b/docs/evidence/arc_v2_2/TRAINING_COMPONENT_OUTLIER_0008.json @@ -0,0 +1,84 @@ +{ + "artifact": { + "digest": "sha256:96a6b3aabbeae4742e4df129fb76a1ad6fe3ce52a2580d9b1d35def4ee6e215d", + "id": 8909609720, + "name": "arc-v2-2-outlier-0439bd2e8611bc2ad0549249b417c01034ce546e", + "size_in_bytes": 18028856 + }, + "candidate_commit": "0439bd2e8611bc2ad0549249b417c01034ce546e", + "checksum_manifest": { + "entry_count": 1006, + "verified": true + }, + "diagnosis": { + "diagnosis_digest": "509ba463912ff7de7a58764f9f3e9cc7c98aadc590612215d5627c7a96d3dbcf", + "hidden_failure_counts": { + "color_map": { + "KeyError": 1 + }, + "component_area_outlier_color": { + "ValueError": 1 + }, + "component_rank_color": { + "KeyError": 3 + } + }, + "selected_component_outlier_tasks": 1, + "total_event_count": 328061, + "trace_count": 1000 + }, + "focused_green": { + "focused_tests": 15, + "frozen_byte_gate": true, + "frozen_tests": 5, + "run_id": 30951907670, + "v2_2_tests": 59 + }, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "mechanistic": { + "compact_parity_count": 1000, + "compact_parity_mismatches": 0, + "trace_count": 1000, + "trace_schema": "mechanistic-grid-trace-v1" + }, + "metrics": { + "accepted": 61, + "attempted_accuracy": 1.0, + "correct": 61, + "coverage": 0.056691449814126396, + "failed": 0, + "incorrect_attempts": 0, + "raw_accuracy": 0.056691449814126396, + "rows": 1076 + }, + "public_arc_evaluation_rerun": false, + "report_digest": "90b419c62ea9012cee5634420c8dd71f381267916f0a0d119793e1861ec84166", + "run_id": 30952042813, + "schema_version": 1, + "status_counts": { + "ACCEPTED": 61, + "GRAMMAR_EXHAUSTED": 1015 + }, + "transitions": { + "gained_rows": { + "aabf363d": { + "baseline_correct": 0, + "current_correct": 1 + }, + "b230c067": { + "baseline_correct": 0, + "current_correct": 1 + } + }, + "lost_rows": {} + }, + "b230c067": { + "eight_connected_success_count": 1, + "four_connected_rejection_count": 1, + "invariant": "one unique area extreme and uniform-area peers", + "preserved_rank_failure_count": 2, + "selected_candidate_id": "component_outlier:0001", + "terminal_status": "ACCEPTED", + "trace_digest": "71a3e46933f833002164cc3b4ed8d2cd2a4a94060a2bdfc330a7a1e0694ff5fd" + } +} diff --git a/docs/evidence/arc_v2_2/TRAINING_ENCLOSED_REGION_0004.json b/docs/evidence/arc_v2_2/TRAINING_ENCLOSED_REGION_0004.json new file mode 100644 index 00000000..e02a58a2 --- /dev/null +++ b/docs/evidence/arc_v2_2/TRAINING_ENCLOSED_REGION_0004.json @@ -0,0 +1,41 @@ +{ + "artifact": { + "digest": "sha256:52c9c2fe9e212ad8f52df3917e4db8b328874e679626d23f3171e2540a1d21a4", + "id": "8905158022", + "run_id": "30941101558" + }, + "candidate_commit": "73a1a45d0d77347f000a10998010573e17004c77", + "development_split": "ARC-AGI-2 training only", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "metrics": { + "accepted_rows": 55, + "correct_rows": 55, + "coverage": 0.05111524163568773, + "execution_failed_rows": 2, + "grammar_exhausted_rows": 1019, + "incorrect_attempts": 0, + "rows": 1076 + }, + "new_exact_tasks": [ + "00d62c1b", + "a5313dff", + "d5d6de2d" + ], + "panel_baseline": { + "accepted_rows": 51, + "candidate_commit": "0c17b540c6814a1ca807a73704abb785257b4121", + "correct_rows": 51, + "incorrect_attempts": 0, + "report_digest": "babfbf938fd02e1065ef5e5a7ae8c48cc5c8b6d71b9fa2d1dd6a7b6fbaa7306a" + }, + "report_digest": "7f0b587f2ae7d1a69f22faa56486becff736a254fa8ee7fb570938c31080d3ee", + "schema_version": 1, + "task_delta": { + "lost_exact_tasks": 0, + "new_exact_tasks": 3 + }, + "row_delta": { + "correct_rows": 4, + "incorrect_attempts": 0 + } +} diff --git a/docs/evidence/arc_v2_2/TRAINING_GRAVITY_COMPACT_0015.json b/docs/evidence/arc_v2_2/TRAINING_GRAVITY_COMPACT_0015.json new file mode 100644 index 00000000..38670e52 --- /dev/null +++ b/docs/evidence/arc_v2_2/TRAINING_GRAVITY_COMPACT_0015.json @@ -0,0 +1,68 @@ +{ + "artifact": { + "artifact_id": 9253746808, + "digest": "sha256:1105909b19e6f828c6463e13fc6aa6a024f2a3da972725ce0bc29a677a9c4e03", + "name": "arc-v2-2-gravity-compact-3c6510a5c69ee600a7fc8d3a0a6a3e15ee9a05b1", + "run_id": 31911074497, + "sha256sums_digest": "5c303b52a2f10658239ef0f4960d2480f6a23a2fe0469597dfa16e969609deea" + }, + "candidate_commit": "3c6510a5c69ee600a7fc8d3a0a6a3e15ee9a05b1", + "diagnosis_digest": "e69cf3ec84919878df2c923a1dadf50ff613211ce27ed27c039a0a711c15e471", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "independent_validation": { + "artifact_digest_matches_github": true, + "checksum_entries": 1006, + "checksum_failures": 0, + "compact_mechanistic_parity_tasks": 1000, + "lost_baseline_rows": {}, + "manifest_verified": true, + "trace_files": 1000 + }, + "mechanisms": { + "1e0a9b12": { + "selected_candidate_id": "gravity_compact:0000", + "successful_gravity_candidates": 1, + "trace_digest": "46d461c2eaf44e0f09a1b02627589ca274023343809cd12b8492fb3b8c92c08a" + }, + "3906de3d": { + "selected_candidate_id": "gravity_compact:0000", + "successful_gravity_candidates": 1, + "trace_digest": "459be859e27aadf6f36abf34b0de9a64e4587c887b8a1e5314887349bcc96dda" + }, + "60b61512": { + "bbox_equivalence_class_count": 2, + "safety_action": "abstain_on_connectivity_disagreement", + "terminal_status": "AMBIGUOUS_PROGRAM", + "trace_digest": "eaad20fe92bffa99618260cd11e00d7967a0fede2fd2a34442dd312826e81588" + } + }, + "metrics": { + "abstained": 984, + "accepted": 92, + "attempted_accuracy": 1.0, + "correct": 92, + "coverage": 0.08550185873605948, + "failed": 0, + "families": 1000, + "incorrect_attempts": 0, + "raw_accuracy": 0.08550185873605948, + "rows": 1076 + }, + "public_arc_evaluation_rerun": false, + "report_digest": "46fbf7e57c2722bcaab4bfd2f894599635cfa6e25db81775d2e7e962230930a0", + "schema_version": 1, + "status_counts": { + "ACCEPTED": 92, + "AMBIGUOUS_PROGRAM": 1, + "GRAMMAR_EXHAUSTED": 983 + }, + "training_transition": { + "cumulative_gain_tasks": 33, + "expected_gains_matched": true, + "new_exact_tasks": [ + "1e0a9b12", + "3906de3d" + ], + "new_exact_rows": 2 + } +} diff --git a/docs/evidence/arc_v2_2/TRAINING_LINE_CONNECT_0016.json b/docs/evidence/arc_v2_2/TRAINING_LINE_CONNECT_0016.json new file mode 100644 index 00000000..e3cf7531 --- /dev/null +++ b/docs/evidence/arc_v2_2/TRAINING_LINE_CONNECT_0016.json @@ -0,0 +1,80 @@ +{ + "artifact": { + "artifact_id": 9253943423, + "digest": "sha256:9f4c5e510536a2c713c736c291a15e91128779e206ce7b5a71766cf25483e921", + "name": "arc-v2-2-line-connect-df779fa892b9ce3bfddab4d03aa743a97d22d1b3", + "run_id": 31911780490, + "sha256sums_digest": "007c4e8b3a5f5eb6fffa794fb32b3618d0ae57d4a985b98a2deaac12affcfbc2" + }, + "candidate_commit": "df779fa892b9ce3bfddab4d03aa743a97d22d1b3", + "diagnosis_digest": "d0b35d1409645c441a03c449a90504956b5b66e255ae634742874837b5f7526f", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "independent_validation": { + "artifact_digest_matches_github": true, + "checksum_entries": 1006, + "checksum_failures": 0, + "compact_mechanistic_parity_tasks": 1000, + "lost_baseline_rows": {}, + "manifest_verified": true, + "trace_files": 1000 + }, + "mechanisms": { + "1f876c06": { + "selected_candidate_id": "line_connect:0000", + "successful_line_candidates": 2, + "trace_digest": "7ea489182e177a298d8ecb7462860a752e4209904e6b10a01e0edb37cd03a8ba" + }, + "22168020": { + "selected_candidate_id": "line_connect:0000", + "successful_line_candidates": 3, + "trace_digest": "a938df6ee8f258a5f600e41185251283518624d14891ef8aaad5eb1a3ff4fac8" + }, + "22eb0ac0": { + "selected_candidate_id": "line_connect:0000", + "successful_line_candidates": 3, + "trace_digest": "4ece7dbd896e15af93c643de4a81205af48a4a086ab7db5c81c3eeb14e7c150e" + }, + "60b61512": { + "bbox_equivalence_class_count": 2, + "safety_action": "abstain_on_connectivity_disagreement", + "terminal_status": "AMBIGUOUS_PROGRAM", + "trace_digest": "d1d80f60ea04c5c493e33f6a234a72ab123d8e0fedd640114ee35b0415b67a1e" + }, + "ded97339": { + "selected_candidate_id": "line_connect:0000", + "successful_line_candidates": 1, + "trace_digest": "3c79c8f79258b6c46f2e268fd798a40482e2245b9709e3dda2f23745c6da2e29" + } + }, + "metrics": { + "abstained": 980, + "accepted": 96, + "attempted_accuracy": 1.0, + "correct": 96, + "coverage": 0.08921933085501858, + "failed": 0, + "families": 1000, + "incorrect_attempts": 0, + "raw_accuracy": 0.08921933085501858, + "rows": 1076 + }, + "public_arc_evaluation_rerun": false, + "report_digest": "76113f54171aebab6806f4cafb4c6935cfcc0e52d588dcdd900d0c187849756b", + "schema_version": 1, + "status_counts": { + "ACCEPTED": 96, + "AMBIGUOUS_PROGRAM": 1, + "GRAMMAR_EXHAUSTED": 979 + }, + "training_transition": { + "cumulative_gain_tasks": 37, + "expected_gains_matched": true, + "new_exact_tasks": [ + "1f876c06", + "22168020", + "22eb0ac0", + "ded97339" + ], + "new_exact_rows": 4 + } +} diff --git a/docs/evidence/arc_v2_2/TRAINING_MARKER_RECOLOR_0007.json b/docs/evidence/arc_v2_2/TRAINING_MARKER_RECOLOR_0007.json new file mode 100644 index 00000000..d518930b --- /dev/null +++ b/docs/evidence/arc_v2_2/TRAINING_MARKER_RECOLOR_0007.json @@ -0,0 +1,74 @@ +{ + "artifact": { + "digest": "sha256:eb62c32eddeaf04817ee17d076469160ca9a653d6fd910b8d02844754c3f8edf", + "id": 8909206467, + "name": "arc-v2-2-mechanistic-bbb192edd1706b204428041015779c28ded0c4b7", + "size_in_bytes": 17989697 + }, + "candidate_commit": "bbb192edd1706b204428041015779c28ded0c4b7", + "checksum_manifest": { + "entry_count": 1006, + "verified": true + }, + "diagnosis": { + "diagnosis_digest": "a1d5c224724b77cd98d87589a78504960c9b490c86b2d0130a39a8fc704a3298", + "hidden_failure_counts": { + "color_map": { + "KeyError": 1 + }, + "component_rank_color": { + "KeyError": 3 + } + }, + "selected_marker_recolor_tasks": 1, + "total_event_count": 326046, + "trace_count": 1000 + }, + "focused_green": { + "focused_tests": 11, + "frozen_byte_gate": true, + "frozen_tests": 5, + "run_id": 30950841994, + "v2_2_tests": 55 + }, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "mechanistic": { + "compact_parity_count": 1000, + "compact_parity_mismatches": 0, + "trace_count": 1000, + "trace_schema": "mechanistic-grid-trace-v1" + }, + "metrics": { + "accepted": 60, + "attempted_accuracy": 1.0, + "correct": 60, + "coverage": 0.055762081784386616, + "incorrect_attempts": 0, + "raw_accuracy": 0.055762081784386616, + "rows": 1076 + }, + "public_arc_evaluation_rerun": false, + "report_digest": "4ba3f1e42d25d6d328891fb00cf79a2b26b218db7355fb0ad7fccef9f94ae7d3", + "run_id": 30951065593, + "schema_version": 1, + "status_counts": { + "ACCEPTED": 60, + "EXECUTION_FAILED": 1, + "GRAMMAR_EXHAUSTED": 1015 + }, + "transition": { + "gained_rows": { + "aabf363d": { + "baseline_correct": 0, + "current_correct": 1 + } + }, + "lost_rows": {}, + "preserved_color_map_failure_count": 1, + "repair_success_count": 2, + "selected_candidate_id": "marker_recolor:0000", + "task_id": "aabf363d", + "terminal_status": "ACCEPTED", + "trace_digest": "c961a60fee167994dfa1eea3efd25330fad6c59ba8c3d82e135a151ee50886bc" + } +} diff --git a/docs/evidence/arc_v2_2/TRAINING_MECHANISTIC_0006.json b/docs/evidence/arc_v2_2/TRAINING_MECHANISTIC_0006.json new file mode 100644 index 00000000..34497d0b --- /dev/null +++ b/docs/evidence/arc_v2_2/TRAINING_MECHANISTIC_0006.json @@ -0,0 +1,78 @@ +{ + "artifact": { + "digest": "sha256:83b835a5e58293a33d554875db1495061c1a3dd1dc004d1af79b11210269cbc5", + "id": 8908817384, + "name": "arc-v2-2-mechanistic-29d64251ebfb9465e0d60b89b0fc42c67fbc2dc2", + "size_in_bytes": 16891645 + }, + "candidate_commit": "29d64251ebfb9465e0d60b89b0fc42c67fbc2dc2", + "checksum_manifest": { + "entry_count": 1005, + "verified": true + }, + "diagnosis": { + "accepted_selected_kind_counts": { + "color_map": 3, + "component_area_select": 5, + "component_rank_color": 11, + "enclosed_region_fill": 3, + "panel_boolean_combine": 20, + "reflect": 4, + "rotate": 3, + "scale": 3, + "tile": 1 + }, + "diagnosis_digest": "f1c475df51ba62444513dafb705aba27773b9d1fe22c75034f38dbabdb21bcf2", + "hidden_failure_counts": { + "color_map": { + "KeyError": 1 + }, + "component_rank_color": { + "KeyError": 3 + } + }, + "total_event_count": 297205, + "trace_count": 1000 + }, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "mechanistic": { + "compact_parity_count": 1000, + "compact_parity_mismatches": 0, + "trace_count": 1000, + "trace_schema": "mechanistic-grid-trace-v1" + }, + "metrics": { + "accepted": 59, + "attempted_accuracy": 1.0, + "correct": 59, + "coverage": 0.05483271375464684, + "incorrect_attempts": 0, + "raw_accuracy": 0.05483271375464684, + "rows": 1076 + }, + "public_arc_evaluation_rerun": false, + "report_digest": "69880bd4fe7e295c6f61771b83385503f81f4e1b178f8472cd1f5fcdf8b78ede", + "root_causes": { + "aabf363d": { + "fitting_candidate_count": 1, + "matching_failure_count": 1, + "mechanism": "color_map hidden execution raises KeyError for an unseen source color", + "terminal_status": "EXECUTION_FAILED", + "trace_digest": "29544f72d9511885b6ea71838162ee86d83437198841c2e4815a5ffb824f54e4" + }, + "b230c067": { + "fitting_candidate_count": 2, + "matching_failure_count": 2, + "mechanism": "both component_rank_color candidates raise KeyError for an unseen hidden component rank", + "terminal_status": "EXECUTION_FAILED", + "trace_digest": "2f5ef2c8f1287e6ebfb186c5f1a3103eb943d6b5cbcb0ccd142d8bc56ab303e5" + } + }, + "run_id": 30950052575, + "schema_version": 1, + "status_counts": { + "ACCEPTED": 59, + "EXECUTION_FAILED": 2, + "GRAMMAR_EXHAUSTED": 1015 + } +} diff --git a/docs/evidence/arc_v2_2/TRAINING_MIRROR_CONCAT_0010.json b/docs/evidence/arc_v2_2/TRAINING_MIRROR_CONCAT_0010.json new file mode 100644 index 00000000..b427c01e --- /dev/null +++ b/docs/evidence/arc_v2_2/TRAINING_MIRROR_CONCAT_0010.json @@ -0,0 +1,165 @@ +{ + "accepted_selected_kind_counts": { + "bounding_box_complete": 4, + "color_map": 3, + "component_area_outlier_color": 1, + "component_area_select": 5, + "component_rank_color": 10, + "enclosed_region_fill": 3, + "marker_recolor": 1, + "mirror_concatenate": 5, + "panel_boolean_combine": 20, + "reflect": 4, + "rotate": 3, + "scale": 3, + "tile": 1 + }, + "artifact": { + "digest": "sha256:b5e395d8d3608331aae1bc360ef14d1d7431ef646dee3f66d41be7bcb77cf06c", + "expired": false, + "id": 8910434494, + "name": "arc-v2-2-mirror-ae96d567aeb65a951873ec186b3915510b99677c", + "size_in_bytes": 33654179 + }, + "candidate_commit": "ae96d567aeb65a951873ec186b3915510b99677c", + "diagnosis_digest": "c2bae85a09fd6939fb5463dc625a9d37e876a1bf57cf2227005dbf039c7203d7", + "frozen_core_byte_gate": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "hidden_failure_counts": { + "color_map": { + "KeyError": 1 + }, + "component_area_outlier_color": { + "ValueError": 1 + }, + "component_rank_color": { + "KeyError": 3 + } + }, + "independent_verification": { + "checksum_manifest_entries": 1006, + "checksum_manifest_verified": true, + "compact_mechanistic_parity": 1000, + "task_traces_validated": 1000 + }, + "mechanisms": { + "60b61512": { + "bbox_equivalence_class_count": 2, + "safety_action": "abstain_on_connectivity_disagreement", + "terminal_status": "AMBIGUOUS_PROGRAM", + "trace_digest": "f0a2fcf5ae41518b64119a7acabc4b6f3878081a48424ace494ef77e62ace1b5" + }, + "mirror_tasks": { + "4c4377d9": { + "selected_candidate_id": "mirror_concat:0001", + "successful_mirror_candidates": 1, + "trace_digest": "6624b3fbcd11d1b6ab03b33098f873a03b6ca50dec687308b16a21ba9725474e" + }, + "6d0aefbc": { + "selected_candidate_id": "mirror_concat:0002", + "successful_mirror_candidates": 1, + "trace_digest": "f23739e1cb0d0ca598471da7c64fe3711c9907c08cd38a5d94237844f869ca74" + }, + "6fa7a44f": { + "selected_candidate_id": "mirror_concat:0000", + "successful_mirror_candidates": 1, + "trace_digest": "c7dcf8ea4ab7583ebb20c4c7071e5d5e8a862f024f66044964bc669c50d7d59e" + }, + "8be77c9e": { + "selected_candidate_id": "mirror_concat:0000", + "successful_mirror_candidates": 1, + "trace_digest": "f82256fd01d3453b30f8eaf468b56b1354da5bb7093a75d6e5b8434eea58435a" + }, + "c9e6f938": { + "selected_candidate_id": "mirror_concat:0002", + "successful_mirror_candidates": 1, + "trace_digest": "6ccc3f301c9d46effbb7d44e9c745d518c652a625f09f211f7c9c40fcdc80f92" + } + } + }, + "mechanistic": { + "diagnosis_digest": "c2bae85a09fd6939fb5463dc625a9d37e876a1bf57cf2227005dbf039c7203d7", + "diagnosis_path": "diagnosis.json", + "parity_checked": true, + "parity_count": 1000, + "trace_count": 1000, + "trace_schema": "mechanistic-grid-trace-v1" + }, + "metrics": { + "abstained": 1007, + "accepted": 69, + "attempted_accuracy": 1.0, + "correct": 69, + "coverage": 0.06412639405204461, + "failed": 0, + "families": 1000, + "incorrect_attempts": 0, + "raw_accuracy": 0.06412639405204461, + "rows": 1076 + }, + "public_arc_evaluation_rerun": false, + "report_digest": "e15a1b0be8617fbb97b0cc8c64a03836eb60045a1b56268d9725f5f272a1021a", + "schema_version": 1, + "status_counts": { + "ACCEPTED": 69, + "AMBIGUOUS_PROGRAM": 1, + "GRAMMAR_EXHAUSTED": 1006 + }, + "total_event_count": 594961, + "transitions": { + "baseline_commit": "e50965009efb96457fd769d97b42027f8a4e98c0", + "candidate_commit": "ae96d567aeb65a951873ec186b3915510b99677c", + "expected_gains_matched": true, + "gained_rows": { + "3aa6fb7a": { + "baseline_correct": 0, + "current_correct": 1 + }, + "4c4377d9": { + "baseline_correct": 0, + "current_correct": 1 + }, + "6d0aefbc": { + "baseline_correct": 0, + "current_correct": 1 + }, + "6d75e8bb": { + "baseline_correct": 0, + "current_correct": 1 + }, + "6fa7a44f": { + "baseline_correct": 0, + "current_correct": 1 + }, + "8be77c9e": { + "baseline_correct": 0, + "current_correct": 1 + }, + "aabf363d": { + "baseline_correct": 0, + "current_correct": 1 + }, + "b230c067": { + "baseline_correct": 0, + "current_correct": 1 + }, + "c9e6f938": { + "baseline_correct": 0, + "current_correct": 1 + }, + "e7639916": { + "baseline_correct": 0, + "current_correct": 1 + } + }, + "incorrect_attempts": 0, + "lost_rows": {}, + "schema_version": 1 + }, + "workflow": { + "conclusion": "success", + "job_id": 92142138954, + "name": "ARC Mirror Concatenation Mechanistic Corpus", + "run_id": 30953847121 + } +} diff --git a/docs/evidence/arc_v2_2/TRAINING_PALETTE_REPEAT_0012.json b/docs/evidence/arc_v2_2/TRAINING_PALETTE_REPEAT_0012.json new file mode 100644 index 00000000..4402a44b --- /dev/null +++ b/docs/evidence/arc_v2_2/TRAINING_PALETTE_REPEAT_0012.json @@ -0,0 +1,143 @@ +{ + "accepted_selected_kind_counts": { + "bounding_box_complete": 4, + "color_map": 3, + "component_area_outlier_color": 1, + "component_area_select": 5, + "component_rank_color": 10, + "dihedral_quadrant_mosaic": 9, + "enclosed_region_fill": 3, + "marker_recolor": 1, + "mirror_concatenate": 5, + "palette_cardinality_repeat": 4, + "panel_boolean_combine": 20, + "reflect": 4, + "rotate": 3, + "scale": 3, + "tile": 1 + }, + "artifact": { + "digest": "sha256:6a7633388ef12cefda49951b8ae12fa39382c2d62b55fc8ff03818b3eb52343f", + "expired": false, + "id": 8918122407, + "name": "arc-v2-2-palette-repeat-1437a6b18a67fae0d275416207a4120432a58acd", + "size_in_bytes": 33788623 + }, + "candidate_commit": "1437a6b18a67fae0d275416207a4120432a58acd", + "diagnosis_digest": "92d51f982125188f4b2120810f385acd9c109f1fd02113f49972100ac3937def", + "frozen_core_byte_gate": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "hidden_failure_counts": { + "color_map": {"KeyError": 1}, + "component_area_outlier_color": {"ValueError": 1}, + "component_rank_color": {"KeyError": 3} + }, + "independent_verification": { + "checksum_manifest_entries": 1006, + "checksum_manifest_verified": true, + "compact_mechanistic_parity": 1000, + "evaluated_validator_commit": "1437a6b18a67fae0d275416207a4120432a58acd", + "event_count_recomputed": 599077, + "task_traces_validated": 1000 + }, + "mechanisms": { + "60b61512": { + "bbox_equivalence_class_count": 2, + "safety_action": "abstain_on_connectivity_disagreement", + "terminal_status": "AMBIGUOUS_PROGRAM", + "trace_digest": "12a35f3c2a6cf31b482a0f374349fd2386d6e1b0e909f729a4a23b16aa2f0d4b" + }, + "ownership_boundary": "NO_FACTOR_VARIATION", + "palette_tasks": { + "a59b95c0": { + "selected_candidate_id": "palette_repeat:0000", + "successful_palette_candidates": 1, + "trace_digest": "1fb47ebd848333815f78fbc6cbd786f6c3546ea440989a463c35985f051e0e28" + }, + "ac0a08a4": { + "selected_candidate_id": "palette_repeat:0000", + "successful_palette_candidates": 1, + "trace_digest": "1aa65271bc13f17bf3cf5221fdf9a9c02d32d361c322b8ffe98d5cf5fc8b5cad" + }, + "b91ae062": { + "selected_candidate_id": "palette_repeat:0000", + "successful_palette_candidates": 1, + "trace_digest": "5f10be98fa6835e7e103088205154ade25556281beb534bc77b6eb3fff6e2e2e" + }, + "d4b1c2b1": { + "selected_candidate_id": "palette_repeat:0000", + "successful_palette_candidates": 1, + "trace_digest": "f026b7a5251840231916708a47ae8bd928252a3147f228862d7f3a019bcc47bc" + } + }, + "schema_version": 1 + }, + "mechanistic": { + "diagnosis_digest": "92d51f982125188f4b2120810f385acd9c109f1fd02113f49972100ac3937def", + "diagnosis_path": "diagnosis.json", + "parity_checked": true, + "parity_count": 1000, + "trace_count": 1000, + "trace_schema": "mechanistic-grid-trace-v1" + }, + "metrics": { + "abstained": 994, + "accepted": 82, + "attempted_accuracy": 1.0, + "correct": 82, + "coverage": 0.0762081784386617, + "failed": 0, + "families": 1000, + "incorrect_attempts": 0, + "raw_accuracy": 0.0762081784386617, + "rows": 1076 + }, + "public_arc_evaluation_rerun": false, + "report_digest": "7c18caf0417f39c39ea291ff591eca9441978326f01123e4e7718834f6c25310", + "schema_version": 1, + "status_counts": { + "ACCEPTED": 82, + "AMBIGUOUS_PROGRAM": 1, + "GRAMMAR_EXHAUSTED": 993 + }, + "total_event_count": 599077, + "transitions": { + "baseline_commit": "e50965009efb96457fd769d97b42027f8a4e98c0", + "candidate_commit": "1437a6b18a67fae0d275416207a4120432a58acd", + "expected_gains_matched": true, + "gained_rows": { + "0c786b71": {"baseline_correct": 0, "current_correct": 1}, + "3aa6fb7a": {"baseline_correct": 0, "current_correct": 1}, + "3af2c5a8": {"baseline_correct": 0, "current_correct": 1}, + "46442a0e": {"baseline_correct": 0, "current_correct": 1}, + "4c4377d9": {"baseline_correct": 0, "current_correct": 1}, + "62c24649": {"baseline_correct": 0, "current_correct": 1}, + "67e8384a": {"baseline_correct": 0, "current_correct": 1}, + "6d0aefbc": {"baseline_correct": 0, "current_correct": 1}, + "6d75e8bb": {"baseline_correct": 0, "current_correct": 1}, + "6fa7a44f": {"baseline_correct": 0, "current_correct": 1}, + "7953d61e": {"baseline_correct": 0, "current_correct": 1}, + "7fe24cdd": {"baseline_correct": 0, "current_correct": 1}, + "833dafe3": {"baseline_correct": 0, "current_correct": 1}, + "8be77c9e": {"baseline_correct": 0, "current_correct": 1}, + "a59b95c0": {"baseline_correct": 0, "current_correct": 1}, + "aabf363d": {"baseline_correct": 0, "current_correct": 1}, + "ac0a08a4": {"baseline_correct": 0, "current_correct": 1}, + "b230c067": {"baseline_correct": 0, "current_correct": 1}, + "b91ae062": {"baseline_correct": 0, "current_correct": 1}, + "c9e6f938": {"baseline_correct": 0, "current_correct": 1}, + "d4b1c2b1": {"baseline_correct": 0, "current_correct": 1}, + "e7639916": {"baseline_correct": 0, "current_correct": 1}, + "ed98d772": {"baseline_correct": 0, "current_correct": 1} + }, + "incorrect_attempts": 0, + "lost_rows": {}, + "schema_version": 1 + }, + "workflow": { + "conclusion": "success", + "job_id": 92207716360, + "name": "ARC Palette Repeat Mechanistic Corpus", + "run_id": 30975220533 + } +} diff --git a/docs/evidence/arc_v2_2/TRAINING_PANEL_BOOLEAN_0003.json b/docs/evidence/arc_v2_2/TRAINING_PANEL_BOOLEAN_0003.json new file mode 100644 index 00000000..6468948c --- /dev/null +++ b/docs/evidence/arc_v2_2/TRAINING_PANEL_BOOLEAN_0003.json @@ -0,0 +1,58 @@ +{ + "artifact": { + "digest": "sha256:626bb39ef4ff12eae9e51d9a29003a9250677551a5bba87c5f7131922f737007", + "id": "8904691469", + "run_id": "30939960448" + }, + "candidate_commit": "0c17b540c6814a1ca807a73704abb785257b4121", + "development_split": "ARC-AGI-2 training only", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "metrics": { + "accepted_rows": 51, + "correct_rows": 51, + "coverage": 0.047397769516728624, + "execution_failed_rows": 2, + "grammar_exhausted_rows": 1023, + "incorrect_attempts": 0, + "rows": 1076 + }, + "new_exact_tasks": [ + "0520fde7", + "0c9aba6e", + "195ba7dc", + "1b2d62fb", + "31d5ba1a", + "3428a4f5", + "34b99a2b", + "506d28a5", + "5d2a5c43", + "6430c8c4", + "66f2d22f", + "94f9d214", + "99b1bc43", + "ce4f8723", + "d19f7514", + "dae9d2b5", + "e133d23d", + "e345f17b", + "f2829549", + "fafffa47" + ], + "rank_baseline": { + "accepted_rows": 26, + "candidate_commit": "95c8160dd9a88c4bcd66063ab88c4701c93cf502", + "correct_rows": 26, + "incorrect_attempts": 0, + "report_digest": "304dcd2f18f74a83af07adb49b0b7e2abc306014fb6ef3ff3a6bb9707301501e" + }, + "report_digest": "babfbf938fd02e1065ef5e5a7ae8c48cc5c8b6d71b9fa2d1dd6a7b6fbaa7306a", + "schema_version": 1, + "task_delta": { + "lost_exact_tasks": 0, + "new_exact_tasks": 20 + }, + "row_delta": { + "correct_rows": 25, + "incorrect_attempts": 0 + } +} diff --git a/docs/evidence/arc_v2_2/TRAINING_QUADRANT_MOSAIC_0011.json b/docs/evidence/arc_v2_2/TRAINING_QUADRANT_MOSAIC_0011.json new file mode 100644 index 00000000..c81b23e1 --- /dev/null +++ b/docs/evidence/arc_v2_2/TRAINING_QUADRANT_MOSAIC_0011.json @@ -0,0 +1,169 @@ +{ + "accepted_selected_kind_counts": { + "bounding_box_complete": 4, + "color_map": 3, + "component_area_outlier_color": 1, + "component_area_select": 5, + "component_rank_color": 10, + "dihedral_quadrant_mosaic": 9, + "enclosed_region_fill": 3, + "marker_recolor": 1, + "mirror_concatenate": 5, + "panel_boolean_combine": 20, + "reflect": 4, + "rotate": 3, + "scale": 3, + "tile": 1 + }, + "artifact": { + "digest": "sha256:c547f44f992a56db01d7407660dd5ec974206b0b39cf3ea7735ec2c84a5a203e", + "expired": false, + "id": 8917662164, + "name": "arc-v2-2-quadrant-mosaic-f8d71e457024191bc012cca1a164352d9be5af0d", + "size_in_bytes": 33728848 + }, + "candidate_commit": "f8d71e457024191bc012cca1a164352d9be5af0d", + "diagnosis_digest": "c63586aab378b9779fd1ebd16202f1bbfcae99359e1f626df78d5729423962f7", + "frozen_core_byte_gate": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "hidden_failure_counts": { + "color_map": { + "KeyError": 1 + }, + "component_area_outlier_color": { + "ValueError": 1 + }, + "component_rank_color": { + "KeyError": 3 + } + }, + "independent_verification": { + "checksum_manifest_entries": 1006, + "checksum_manifest_verified": true, + "compact_mechanistic_parity": 1000, + "evaluated_validator_commit": "f8d71e457024191bc012cca1a164352d9be5af0d", + "event_count_recomputed": 597037, + "task_traces_validated": 1000 + }, + "mechanisms": { + "60b61512": { + "bbox_equivalence_class_count": 2, + "safety_action": "abstain_on_connectivity_disagreement", + "terminal_status": "AMBIGUOUS_PROGRAM", + "trace_digest": "9705305da8fe622b82a22cd71fea7301cb625ce2c8f0c63690e9bf76ff538186" + }, + "quadrant_tasks": { + "0c786b71": { + "selected_candidate_id": "quadrant_mosaic:0000", + "successful_quadrant_candidates": 1, + "trace_digest": "b870b5bfafd0d46cb4907953fb99b582ec9fa0de77c8263e704ee5a675ac9d9b" + }, + "3af2c5a8": { + "selected_candidate_id": "quadrant_mosaic:0000", + "successful_quadrant_candidates": 1, + "trace_digest": "5e0ac1e8e2266d252f542c479b83a3ed52efdfa16f0685e1e8cb255fe5c215b6" + }, + "46442a0e": { + "selected_candidate_id": "quadrant_mosaic:0000", + "successful_quadrant_candidates": 1, + "trace_digest": "37c5cc99c6aeee011c9f28c65ed42aa9a8aa25cd004da31096904ad4a7a235c9" + }, + "62c24649": { + "selected_candidate_id": "quadrant_mosaic:0000", + "successful_quadrant_candidates": 1, + "trace_digest": "e90ddebadc8d65b408c00641d52fe738ec4a48398f1b9fa006573b118c931cf8" + }, + "67e8384a": { + "selected_candidate_id": "quadrant_mosaic:0000", + "successful_quadrant_candidates": 1, + "trace_digest": "d0ba1dbcd20e82d68582b0960d97986252b1b9141e6ae64dd4b7a427b0c6ef34" + }, + "7953d61e": { + "selected_candidate_id": "quadrant_mosaic:0000", + "successful_quadrant_candidates": 1, + "trace_digest": "225032a56410eaa06799042107262f9d7e5cdee380c1b5af5243306ad2121a98" + }, + "7fe24cdd": { + "selected_candidate_id": "quadrant_mosaic:0000", + "successful_quadrant_candidates": 1, + "trace_digest": "f4b35b71061ca3c3d82290bda32494eb6ee081729102a8f350802c819943ff18" + }, + "833dafe3": { + "selected_candidate_id": "quadrant_mosaic:0000", + "successful_quadrant_candidates": 1, + "trace_digest": "1b06fb719427b754704af58b6e2c67c441385ce05fcfb6b1365e64e087999333" + }, + "ed98d772": { + "selected_candidate_id": "quadrant_mosaic:0000", + "successful_quadrant_candidates": 1, + "trace_digest": "8128fd04afdd34d46287257d4344d03627a79b8049206595d1d7d0480b578d62" + } + }, + "role_boundary": "NO_DISTINCT_QUADRANT_ROLES", + "schema_version": 1 + }, + "mechanistic": { + "diagnosis_digest": "c63586aab378b9779fd1ebd16202f1bbfcae99359e1f626df78d5729423962f7", + "diagnosis_path": "diagnosis.json", + "parity_checked": true, + "parity_count": 1000, + "trace_count": 1000, + "trace_schema": "mechanistic-grid-trace-v1" + }, + "metrics": { + "abstained": 998, + "accepted": 78, + "attempted_accuracy": 1.0, + "correct": 78, + "coverage": 0.0724907063197026, + "failed": 0, + "families": 1000, + "incorrect_attempts": 0, + "raw_accuracy": 0.0724907063197026, + "rows": 1076 + }, + "public_arc_evaluation_rerun": false, + "report_digest": "df4a2a0ac21678353d38de916a9381a60f6f966d465bbfda5f8f55f8e02cce56", + "schema_version": 1, + "status_counts": { + "ACCEPTED": 78, + "AMBIGUOUS_PROGRAM": 1, + "GRAMMAR_EXHAUSTED": 997 + }, + "total_event_count": 597037, + "transitions": { + "baseline_commit": "e50965009efb96457fd769d97b42027f8a4e98c0", + "candidate_commit": "f8d71e457024191bc012cca1a164352d9be5af0d", + "expected_gains_matched": true, + "gained_rows": { + "0c786b71": {"baseline_correct": 0, "current_correct": 1}, + "3aa6fb7a": {"baseline_correct": 0, "current_correct": 1}, + "3af2c5a8": {"baseline_correct": 0, "current_correct": 1}, + "46442a0e": {"baseline_correct": 0, "current_correct": 1}, + "4c4377d9": {"baseline_correct": 0, "current_correct": 1}, + "62c24649": {"baseline_correct": 0, "current_correct": 1}, + "67e8384a": {"baseline_correct": 0, "current_correct": 1}, + "6d0aefbc": {"baseline_correct": 0, "current_correct": 1}, + "6d75e8bb": {"baseline_correct": 0, "current_correct": 1}, + "6fa7a44f": {"baseline_correct": 0, "current_correct": 1}, + "7953d61e": {"baseline_correct": 0, "current_correct": 1}, + "7fe24cdd": {"baseline_correct": 0, "current_correct": 1}, + "833dafe3": {"baseline_correct": 0, "current_correct": 1}, + "8be77c9e": {"baseline_correct": 0, "current_correct": 1}, + "aabf363d": {"baseline_correct": 0, "current_correct": 1}, + "b230c067": {"baseline_correct": 0, "current_correct": 1}, + "c9e6f938": {"baseline_correct": 0, "current_correct": 1}, + "e7639916": {"baseline_correct": 0, "current_correct": 1}, + "ed98d772": {"baseline_correct": 0, "current_correct": 1} + }, + "incorrect_attempts": 0, + "lost_rows": {}, + "schema_version": 1 + }, + "workflow": { + "conclusion": "success", + "job_id": 92203590865, + "name": "ARC Quadrant Mosaic Mechanistic Corpus", + "run_id": 30973820979 + } +} diff --git a/docs/evidence/arc_v2_2/TRAINING_SYMMETRY_COMPLETE_0017.json b/docs/evidence/arc_v2_2/TRAINING_SYMMETRY_COMPLETE_0017.json new file mode 100644 index 00000000..702d524c --- /dev/null +++ b/docs/evidence/arc_v2_2/TRAINING_SYMMETRY_COMPLETE_0017.json @@ -0,0 +1,83 @@ +{ + "artifact": { + "artifact_id": 9254110203, + "digest": "sha256:1dce9fa768b4ce80f354d420d2f689120f832390a39c3512281fefe9a4d9ca05", + "name": "arc-v2-2-symmetry-complete-18573507deb284e42d573085909bbf76be5abcb8", + "run_id": 31912407142, + "sha256sums_digest": "f67aed47fc3d4338cf34f9107b7218ee57cda1c23313fa1673cef551d599914e" + }, + "candidate_commit": "18573507deb284e42d573085909bbf76be5abcb8", + "diagnosis_digest": "971f597393e7c10244d767407f5bf99999821024b8ee804cd3d301f2466e2bec", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "independent_validation": { + "artifact_digest_matches_github": true, + "checksum_entries": 1006, + "checksum_failures": 0, + "compact_mechanistic_parity_tasks": 1000, + "lost_baseline_rows": {}, + "manifest_verified": true, + "trace_files": 1000 + }, + "mechanisms": { + "496994bd": { + "selected_candidate_id": "symmetry_complete:0001", + "successful_symmetry_candidates": 2, + "trace_digest": "70e05d79a59ce394b2889d7ae75535aff9ac55bec3dedd191d43a8b84467e947" + }, + "60b61512": { + "bbox_equivalence_class_count": 2, + "safety_action": "abstain_on_connectivity_disagreement", + "terminal_status": "AMBIGUOUS_PROGRAM", + "trace_digest": "759ac70b3dc31fd8d0e144fba6e661db5925c73009b2acaaf80bad4d99a21514" + }, + "b8825c91": { + "safety_action": "abstain_on_symmetry_disagreement", + "symmetry_equivalence_class_count": 2, + "terminal_status": "AMBIGUOUS_PROGRAM", + "trace_digest": "cce0a065323ff88200d5a3689438162ab7cc36f25a6274d04d96ecbaa09edb3a" + }, + "e729b7be": { + "selected_candidate_id": "symmetry_complete:0000", + "successful_symmetry_candidates": 1, + "trace_digest": "1bb3ff6a90c8344d46d8f450aba4378c6174fc5f3558d91b2fde2f16b92ba7c8" + }, + "f25ffba3": { + "selected_candidate_id": "symmetry_complete:0000", + "successful_symmetry_candidates": 1, + "trace_digest": "5d33b3caf970b06d8082e58e5d6449ca113d98a9d225c3d80afc6efc457191bb" + } + }, + "metrics": { + "abstained": 977, + "accepted": 99, + "attempted_accuracy": 1.0, + "correct": 99, + "coverage": 0.09200743494423792, + "failed": 0, + "families": 1000, + "incorrect_attempts": 0, + "raw_accuracy": 0.09200743494423792, + "rows": 1076 + }, + "public_arc_evaluation_rerun": false, + "report_digest": "da95d63d8a7a10199b58c33a028080d06882eb85cf405cd295913610b39619d4", + "schema_version": 1, + "status_counts": { + "ACCEPTED": 99, + "AMBIGUOUS_PROGRAM": 2, + "GRAMMAR_EXHAUSTED": 975 + }, + "training_transition": { + "cumulative_gain_tasks": 40, + "expected_gains_matched": true, + "new_ambiguous_tasks": [ + "b8825c91" + ], + "new_exact_tasks": [ + "496994bd", + "e729b7be", + "f25ffba3" + ], + "new_exact_rows": 3 + } +} diff --git a/docs/superpowers/plans/2026-08-04-arc-mechanistic-observability.md b/docs/superpowers/plans/2026-08-04-arc-mechanistic-observability.md new file mode 100644 index 00000000..086868f8 --- /dev/null +++ b/docs/superpowers/plans/2026-08-04-arc-mechanistic-observability.md @@ -0,0 +1,673 @@ +# ARC V2.2 Mechanistic Observability Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build deterministic, complete mechanistic traces for every ARC V2.2 grid-induction decision while proving compact-mode parity and reproducing the exact causes of the two current hidden-execution failures. + +**Architecture:** A passive content-addressed recorder stores canonical programs, grids, and output batches once and emits ordered events that reference those objects. Candidate generators expose structurally valid programs without internal fitting; one shared inference path owns demonstration fitting, hidden execution, equivalence grouping, and final decisions in both compact and mechanistic modes. Training runs execute compact and mechanistic passes, verify parity, write one sealed trace per task, and aggregate trace-only diagnosis. + +**Tech Stack:** Python 3.12, standard-library `dataclasses`, `hashlib`, and `json`; pytest; GitHub Actions; immutable ARC-AGI-2 training artifact. + +## Global Constraints + +- Work only on branch `universal-core-arc-training-v2-2`. +- Do not modify `research/universal_core/holonomy_v2`. +- Frozen V2.1 must remain byte-identical to `1d7c489bcdbff755d638b35ad47ce62bd4fe829d`. +- Use only the 1,000 ARC-AGI-2 training tasks for iterative development. +- Do not rerun ARC public evaluation. +- Do not add task-ID routing or target-informed model behavior. +- Mechanistic tracing must not alter statuses, predictions, program digests, freeze digests, or prediction digests. +- Every coherent unit uses focused RED, focused GREEN, affected-cluster GREEN, frozen-core gates, and an atomic commit. +- Full-corpus acceptance requires zero newly incorrect accepted rows and no lost exact tasks relative to the 59/1,076 baseline. + +## File structure + +- Create `research/universal_core/holonomy_v2_2/mechanistic_trace.py`: canonical object store, event recorder, trace sealing, trace validation, trace writing. +- Create `research/universal_core/holonomy_v2_2/trace_diagnosis.py`: deterministic aggregation of trace facts only. +- Create `research/universal_core/holonomy_v2_2/tests/test_mechanistic_trace.py`: recorder, sealing, determinism, validation, and task-ID invariance tests. +- Create `research/universal_core/holonomy_v2_2/tests/test_trace_diagnosis.py`: aggregate diagnosis tests. +- Modify `research/universal_core/holonomy_v2_2/panel_combine.py`: expose unfiltered structural enumeration. +- Modify `research/universal_core/holonomy_v2_2/region_fill.py`: expose unfiltered structural enumeration. +- Modify `research/universal_core/holonomy_v2_2/component_select.py`: expose unfiltered structural enumeration. +- Modify `research/universal_core/holonomy_v2_2/grid_induction.py`: central fitting, tracing, hidden execution, equivalence grouping, final decision, and dual-mode entry point. +- Modify `research/universal_core/holonomy_v2_2/training_eval.py`: optional trace directory, compact/mechanistic parity, trace references, diagnosis. +- Modify `research/universal_core/holonomy_v2_2/tests/test_grid_induction.py`: dual-mode parity, complete candidate lifecycle, deterministic replay, opaque delegate boundary. +- Modify `research/universal_core/holonomy_v2_2/tests/test_training_eval.py`: per-task traces, parity failure containment, references, diagnosis. +- Modify `.github/workflows/arc-training-corpus-v2-2.yml`: mandatory mechanistic corpus run and artifact publication. +- Create `research/universal_core/holonomy_v2_2/evidence/MECHANISTIC_TRACE_TDD.json`: immutable RED/GREEN and corpus evidence references after hosted verification. + +--- + +### Task 1: Deterministic recorder and completeness validator + +**Files:** +- Create: `research/universal_core/holonomy_v2_2/mechanistic_trace.py` +- Create: `research/universal_core/holonomy_v2_2/tests/test_mechanistic_trace.py` + +**Interfaces:** +- Consumes: JSON-safe Python mappings, task ID, terminal decision. +- Produces: + - `TraceRecorder(task_id: str)` + - `TraceRecorder.store(value: Any) -> str` + - `TraceRecorder.emit(event: str, **data: Any) -> None` + - `TraceRecorder.seal(terminal_status: str) -> dict[str, Any]` + - `validate_trace(trace: Mapping[str, Any]) -> None` + - `write_trace(trace: Mapping[str, Any], path: Path | str) -> Path` + +- [ ] **Step 1: Write failing recorder tests** + +Create tests with these exact contracts: + +```python +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from research.universal_core.holonomy_v2_2.mechanistic_trace import ( + TraceRecorder, + validate_trace, + write_trace, +) + + +def complete_trace(task_id: str = "alpha") -> dict[str, object]: + recorder = TraceRecorder(task_id) + program_ref = recorder.store({"kind": "identity"}) + grid_ref = recorder.store([[1]]) + recorder.emit( + "candidate_proposed", + candidate_id="fixed:0000", + generator="fixed", + program_digest=program_ref, + program_ref=program_ref, + ) + recorder.emit( + "demo_execution", + candidate_id="fixed:0000", + demo_index=0, + outcome="MATCH", + output_ref=grid_ref, + target_ref=grid_ref, + ) + recorder.emit( + "candidate_fit_decision", + candidate_id="fixed:0000", + decision="FITS_ALL_DEMOS", + ) + recorder.emit( + "hidden_execution", + candidate_id="fixed:0000", + hidden_index=0, + outcome="SUCCEEDED", + output_ref=grid_ref, + ) + recorder.emit( + "hidden_batch_decision", + candidate_id="fixed:0000", + decision="COMPLETED", + batch_ref=recorder.store([[[1]]]), + ) + recorder.emit( + "equivalence_class", + batch_ref=recorder.store([[[1]]]), + candidate_ids=["fixed:0000"], + ) + recorder.emit( + "final_decision", + status="ACCEPTED", + candidate_ids=["fixed:0000"], + ) + return recorder.seal("ACCEPTED") + + +def test_trace_is_deterministic_and_task_id_independent() -> None: + first = complete_trace("alpha") + second = complete_trace("beta") + assert first["trace_digest"] == second["trace_digest"] + assert first["envelope_digest"] != second["envelope_digest"] + assert json.dumps(first, sort_keys=True) == json.dumps(complete_trace("alpha"), sort_keys=True) + + +def test_validator_rejects_missing_candidate_terminal_event() -> None: + recorder = TraceRecorder("bad") + program_ref = recorder.store({"kind": "identity"}) + recorder.emit( + "candidate_proposed", + candidate_id="fixed:0000", + generator="fixed", + program_digest=program_ref, + program_ref=program_ref, + ) + recorder.emit("final_decision", status="GRAMMAR_EXHAUSTED", candidate_ids=[]) + with pytest.raises(ValueError, match="terminal demonstration decision"): + recorder.seal("GRAMMAR_EXHAUSTED") + + +def test_write_trace_emits_canonical_json(tmp_path: Path) -> None: + trace = complete_trace() + path = write_trace(trace, tmp_path / "trace.json") + assert path.read_text(encoding="utf-8").endswith("\n") + validate_trace(json.loads(path.read_text(encoding="utf-8"))) +``` + +- [ ] **Step 2: Run the focused test and verify RED** + +Run: + +```bash +python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_mechanistic_trace.py +``` + +Expected: collection failure because `holonomy_v2_2.mechanistic_trace` does not exist. + +- [ ] **Step 3: Implement the minimal recorder** + +Implement these rules: + +```python +_TRACE_SCHEMA = "mechanistic-grid-trace-v1" + + +def _canonical(value: Any) -> bytes: + return (json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n").encode("utf-8") + + +def _digest(value: Any) -> str: + return hashlib.sha256(_canonical(value)).hexdigest() +``` + +`TraceRecorder.store` must canonicalize the value, use its SHA-256 as the key, and reject a hash collision with unequal canonical bytes. `emit` must append `{"ordinal": n, "event": event, ...}` with contiguous zero-based ordinals. `seal` must build: + +```python +{ + "schema": _TRACE_SCHEMA, + "task_id": task_id, + "objects": {digest: value, ...}, + "events": [...], + "event_count": len(events), + "terminal_status": terminal_status, + "trace_digest": digest({"schema": schema, "objects": objects, "events": events, "terminal_status": terminal_status}), + "envelope_digest": digest({"task_id": task_id, "trace_digest": trace_digest}), +} +``` + +`validate_trace` must verify digests, contiguous ordinals, object references, candidate lifecycle completeness, hidden lifecycle completeness for fitting non-duplicate candidates, equivalence-class membership, and exactly one final decision matching `terminal_status`. + +- [ ] **Step 4: Run focused GREEN** + +Run: + +```bash +python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_mechanistic_trace.py +``` + +Expected: all recorder tests pass. + +- [ ] **Step 5: Run the existing V2.2 cluster** + +Run: + +```bash +python -m pytest -q research/universal_core/holonomy_v2_2/tests +``` + +Expected: existing tests plus recorder tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add research/universal_core/holonomy_v2_2/mechanistic_trace.py research/universal_core/holonomy_v2_2/tests/test_mechanistic_trace.py +git commit -m "arc: add deterministic mechanistic trace ledger" +``` + +--- + +### Task 2: Expose every structural candidate before fitting + +**Files:** +- Modify: `research/universal_core/holonomy_v2_2/panel_combine.py:panel_candidates` +- Modify: `research/universal_core/holonomy_v2_2/region_fill.py:region_candidates` +- Modify: `research/universal_core/holonomy_v2_2/component_select.py:component_select_candidates` +- Modify: `research/universal_core/holonomy_v2_2/tests/test_panel_combine.py` +- Modify: `research/universal_core/holonomy_v2_2/tests/test_region_fill.py` +- Modify: `research/universal_core/holonomy_v2_2/tests/test_component_select.py` + +**Interfaces:** +- Consumes: normalized demonstration `(source_grid, target_grid)` pairs. +- Produces: + - `panel_programs(demos) -> tuple[tuple[Program, ...], str | None]` + - `region_programs(demos) -> tuple[tuple[Program, ...], str | None]` + - `component_select_programs(demos) -> tuple[tuple[Program, ...], str | None]` +- Compatibility wrappers `*_candidates` remain available and retain their currently filtered behavior until Task 3 switches the grid path. + +- [ ] **Step 1: Write failing enumeration tests** + +Add one test per module proving an intentionally non-fitting structural program is returned. Example panel contract: + +```python +def test_panel_programs_expose_structural_hypotheses_before_fit() -> None: + demos = (([[1, 0, 9, 2, 0]], [[4, 0]]),) + programs, reason = panel_combine.panel_programs(demos) + assert reason is None + assert len(programs) > len(panel_combine.panel_candidates(demos)) + assert any(item["predicate"] == "xor" for item in programs) +``` + +Region and component-selection tests must similarly assert structural enumeration is larger than the fitting subset. Add skip-reason tests for invalid target-color cardinality or unavailable background evidence. + +- [ ] **Step 2: Run the three focused files and verify RED** + +Run: + +```bash +python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_panel_combine.py \ + research/universal_core/holonomy_v2_2/tests/test_region_fill.py \ + research/universal_core/holonomy_v2_2/tests/test_component_select.py +``` + +Expected: failures because `panel_programs`, `region_programs`, and `component_select_programs` do not exist. + +- [ ] **Step 3: Implement unfiltered enumerators** + +Move only structural loops into the new functions. They must not call `_fits`. Return an exact reason when no structural family can be formed, using stable values such as: + +- `TARGET_COLOR_CARDINALITY_NOT_TWO` +- `NO_BACKGROUND_CANDIDATE` +- `NO_STRUCTURAL_PROGRAM` + +Keep current `*_candidates` wrappers as: + +```python +def panel_candidates(demos: Sequence[tuple[Grid, Grid]]) -> tuple[Program, ...]: + programs, _ = panel_programs(demos) + return tuple(program for program in programs if _fits(program, demos)) +``` + +Apply the same pattern to region and component-selection modules. + +- [ ] **Step 4: Run focused GREEN and the V2.2 cluster** + +Run: + +```bash +python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_panel_combine.py \ + research/universal_core/holonomy_v2_2/tests/test_region_fill.py \ + research/universal_core/holonomy_v2_2/tests/test_component_select.py +python -m pytest -q research/universal_core/holonomy_v2_2/tests +``` + +Expected: all tests pass with unchanged public predictions. + +- [ ] **Step 5: Commit** + +```bash +git add \ + research/universal_core/holonomy_v2_2/panel_combine.py \ + research/universal_core/holonomy_v2_2/region_fill.py \ + research/universal_core/holonomy_v2_2/component_select.py \ + research/universal_core/holonomy_v2_2/tests/test_panel_combine.py \ + research/universal_core/holonomy_v2_2/tests/test_region_fill.py \ + research/universal_core/holonomy_v2_2/tests/test_component_select.py +git commit -m "arc: expose structural grid hypotheses" +``` + +--- + +### Task 3: Integrate complete tracing into the shared inference path + +**Files:** +- Modify: `research/universal_core/holonomy_v2_2/grid_induction.py:_candidate_programs,_fits,run_public_task_v2_2` +- Modify: `research/universal_core/holonomy_v2_2/tests/test_grid_induction.py` +- Modify: `research/universal_core/holonomy_v2_2/tests/test_component_rank.py` + +**Interfaces:** +- Consumes: + - `TraceRecorder | None` + - structural program batches from Task 2. +- Produces: + - `run_public_task_v2_2(task, engine, *, mechanistic: bool = False) -> dict[str, Any]` + - mechanistic result field `mechanistic_trace` only when requested. + +- [ ] **Step 1: Write failing dual-mode and lifecycle tests** + +Add contracts for: + +```python +def without_trace(result: dict[str, object]) -> dict[str, object]: + copied = dict(result) + copied.pop("mechanistic_trace", None) + return copied + + +def test_mechanistic_mode_has_exact_compact_parity() -> None: + value = task( + [{"input": [[0, 1]], "output": [[2, 3]]}], + [[[1, 0]]], + "parity", + ) + compact = grid_induction.run_public_task_v2_2(value, engine=None) + traced = grid_induction.run_public_task_v2_2(value, engine=None, mechanistic=True) + assert without_trace(traced) == compact + validate_trace(traced["mechanistic_trace"]) + + +def test_trace_records_rejected_and_fitting_candidates() -> None: + value = task( + [{"input": [[1, 2]], "output": [[2, 1]]}], + [[[3, 4]]], + "candidate-ledger", + ) + trace = grid_induction.run_public_task_v2_2(value, None, mechanistic=True)["mechanistic_trace"] + events = trace["events"] + proposed = [item for item in events if item["event"] == "candidate_proposed"] + terminal = [item for item in events if item["event"] == "candidate_fit_decision"] + assert len(proposed) == len(terminal) + assert {item["decision"] for item in terminal} >= {"FITS_ALL_DEMOS", "DEMO_MISMATCH"} + + +def test_hidden_key_error_is_recorded_with_type_and_message() -> None: + value = make_component_rank_task_that_introduces_an_unseen_hidden_rank() + result = grid_induction.run_public_task_v2_2(value, None, mechanistic=True) + failures = [ + item for item in result["mechanistic_trace"]["events"] + if item["event"] == "hidden_execution" and item["outcome"] == "FAILED" + ] + assert failures + assert all(item["exception_type"] == "KeyError" for item in failures) + assert all("unmapped component rank" in item["exception_message"] for item in failures) +``` + +Also add deterministic replay, task-ID-independent `trace_digest`, accepted-equivalence membership, ambiguity membership, grammar exhaustion, and non-grid `delegate_boundary` tests. + +- [ ] **Step 2: Run focused tests and verify RED** + +Run: + +```bash +python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_grid_induction.py \ + research/universal_core/holonomy_v2_2/tests/test_component_rank.py \ + research/universal_core/holonomy_v2_2/tests/test_mechanistic_trace.py +``` + +Expected: `TypeError` for the missing `mechanistic` keyword and missing trace events. + +- [ ] **Step 3: Replace opaque fitting with one traced evaluator** + +Introduce internal records with deterministic candidate IDs. The central evaluator must: + +1. emit `generator_start`; +2. emit `generator_skip` or `candidate_proposed` for every program; +3. execute every demonstration for every proposal; +4. store source, target, produced grids, and batches in the trace object store; +5. emit exact mismatch coordinates or typed failures; +6. emit one `candidate_fit_decision` per proposal; +7. deduplicate fitting programs by program digest and emit `candidate_duplicate`; +8. execute every hidden row for every fitting representative; +9. emit one `hidden_batch_decision` per representative; +10. emit one `equivalence_class` per successful batch; +11. emit one `final_decision` and seal the trace. + +The public result must be constructed before trace attachment. Attach the sealed trace only after public digests are final. + +Do not catch exceptions outside the existing `KeyError`, `TypeError`, and `ValueError` boundary. + +- [ ] **Step 4: Run focused GREEN** + +Run: + +```bash +python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_grid_induction.py \ + research/universal_core/holonomy_v2_2/tests/test_component_rank.py \ + research/universal_core/holonomy_v2_2/tests/test_mechanistic_trace.py +``` + +Expected: all focused tests pass. + +- [ ] **Step 5: Run complete V2.2 and frozen integration gates** + +Run: + +```bash +python -m pytest -q research/universal_core/holonomy_v2_2/tests +PYTHONPATH=research python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py +git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- research/universal_core/holonomy_v2 +``` + +Expected: all tests pass and the frozen diff command exits zero. + +- [ ] **Step 6: Commit** + +```bash +git add \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/tests/test_grid_induction.py \ + research/universal_core/holonomy_v2_2/tests/test_component_rank.py +git commit -m "arc: trace complete grid inference path" +``` + +--- + +### Task 4: Add corpus trace output, parity, and diagnosis + +**Files:** +- Create: `research/universal_core/holonomy_v2_2/trace_diagnosis.py` +- Create: `research/universal_core/holonomy_v2_2/tests/test_trace_diagnosis.py` +- Modify: `research/universal_core/holonomy_v2_2/training_eval.py:run_training_corpus` +- Modify: `research/universal_core/holonomy_v2_2/tests/test_training_eval.py` + +**Interfaces:** +- Consumes: + - `run_training_corpus(..., mechanistic_dir: Path | str | None = None, verify_parity: bool = False)` + - sealed per-task traces. +- Produces: + - per-report `mechanistic_trace` reference containing `path`, `trace_digest`, `envelope_digest`, `event_count`, and `terminal_status`; + - `build_trace_diagnosis(traces: Sequence[Mapping[str, Any]]) -> dict[str, Any]`; + - canonical `diagnosis.json` when tracing is enabled. + +- [ ] **Step 1: Write failing evaluator and diagnosis tests** + +Add tests proving: + +- two fixture tasks write `a.json` and `b.json` traces in sorted order; +- the aggregate report excludes full event arrays; +- each trace reference matches the file digest and envelope digest; +- compact/mechanistic mismatch raises `RuntimeError("mechanistic parity failure")`; +- diagnosis counts hidden `KeyError` failures by candidate kind and accepted decisions by selected kind; +- scoring targets are not present in any trace object unless they also occur as demonstration targets. + +Use a fake runner accepting `mechanistic=` and returning a minimal valid trace from Task 1. + +- [ ] **Step 2: Run focused tests and verify RED** + +Run: + +```bash +python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_training_eval.py \ + research/universal_core/holonomy_v2_2/tests/test_trace_diagnosis.py +``` + +Expected: import failure for `trace_diagnosis` and unexpected keyword errors for `mechanistic_dir`. + +- [ ] **Step 3: Implement traced corpus execution** + +When `mechanistic_dir` is set: + +1. run compact mode; +2. run mechanistic mode; +3. remove only `mechanistic_trace` from the mechanistic result and compare exact equality when `verify_parity=True`; +4. validate and write the trace to `/.json`; +5. use the compact result for scoring; +6. append a digest-only trace reference to the task report; +7. build and write `diagnosis.json` after all tasks. + +Seal the aggregate report only after trace references and diagnosis digest are present. + +`trace_diagnosis.py` must derive counts exclusively from events and object references. It must not inspect ARC test outputs or scoring metrics. + +- [ ] **Step 4: Run focused GREEN and all V2.2 tests** + +Run: + +```bash +python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_training_eval.py \ + research/universal_core/holonomy_v2_2/tests/test_trace_diagnosis.py +python -m pytest -q research/universal_core/holonomy_v2_2/tests +``` + +Expected: all tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add \ + research/universal_core/holonomy_v2_2/trace_diagnosis.py \ + research/universal_core/holonomy_v2_2/training_eval.py \ + research/universal_core/holonomy_v2_2/tests/test_trace_diagnosis.py \ + research/universal_core/holonomy_v2_2/tests/test_training_eval.py +git commit -m "arc: publish mechanistic corpus diagnosis" +``` + +--- + +### Task 5: Publish full mechanistic training evidence and confirm root causes + +**Files:** +- Modify: `.github/workflows/arc-training-corpus-v2-2.yml` +- Create: `research/universal_core/holonomy_v2_2/evidence/MECHANISTIC_TRACE_TDD.json` +- Create after successful run: `docs/evidence/arc_v2_2/TRAINING_MECHANISTIC_0006.json` + +**Interfaces:** +- Consumes: pinned corpus artifact from workflow run `30929169392` and the Task 4 evaluator. +- Produces: one artifact containing `report.json`, `console-summary.json`, `traces/*.json`, `traces/diagnosis.json`, `parity.json`, and `SHA256SUMS`. + +- [ ] **Step 1: Add a hosted RED evidence commit before implementation is wired** + +Commit the focused tests from Tasks 1–4 before production implementation and allow `.github/workflows/arc-training-v2-2.yml` to record the expected failure. Capture run ID, failing test names, and exact reason in `MECHANISTIC_TRACE_TDD.json`. + +- [ ] **Step 2: Update the corpus workflow** + +Change the Python invocation to: + +```python +trace_dir = Path('/tmp/arc-v2-2-training/traces') +report = run_training_corpus( + vendor, + HolonomyEngine(SearchConfig()), + run_label='arc-v2-2-training-' + os.environ['GITHUB_SHA'][:12], + candidate_commit=os.environ['GITHUB_SHA'], + mechanistic_dir=trace_dir, + verify_parity=True, +) +``` + +After writing the report, create `parity.json` with task count and zero mismatches, then generate a sorted SHA-256 manifest over every artifact file except the manifest itself. Fail unless exactly 1,000 task trace files exist and every trace validates. + +- [ ] **Step 3: Run hosted focused GREEN** + +Push a normal branch commit so `.github/workflows/arc-training-v2-2.yml` runs. Require: + +- all V2.2 tests pass; +- frozen V2 integration tests pass; +- frozen V2 byte gate passes. + +Record run ID and commit SHA in `MECHANISTIC_TRACE_TDD.json`. + +- [ ] **Step 4: Run the full 1,000-task mechanistic corpus** + +Require the corpus workflow to finish successfully and publish its artifact. Download and independently verify: + +```bash +sha256sum -c SHA256SUMS +python - <<'PY' +import json +from pathlib import Path +from research.universal_core.holonomy_v2_2.mechanistic_trace import validate_trace + +root = Path('traces') +paths = sorted(path for path in root.glob('*.json') if path.name != 'diagnosis.json') +assert len(paths) == 1000 +for path in paths: + validate_trace(json.loads(path.read_text(encoding='utf-8'))) +PY +``` + +- [ ] **Step 5: Compare against the 59/1,076 baseline** + +Require: + +- 1,076 rows evaluated; +- at least 59 correct accepted rows; +- zero incorrect accepted rows; +- no previously exact task lost; +- status changes explained by trace events rather than instrumentation side effects. + +Tracing itself is rejected if any compact prediction or digest changes. + +- [ ] **Step 6: Confirm the two existing execution failures from trace evidence** + +For task `aabf363d`, require at least one hidden failure event with: + +```json +{ + "exception_type": "KeyError", + "exception_message": "'unmapped grid cell'", + "candidate_kind": "color_map" +} +``` + +For task `b230c067`, require hidden failure events for both fitting representatives with: + +```json +{ + "exception_type": "KeyError", + "exception_message": "'unmapped component rank'", + "candidate_kind": "component_rank_color" +} +``` + +Do not implement either capability fix in this task. Seal the confirmed mechanisms first. + +- [ ] **Step 7: Write immutable evidence** + +Create `TRAINING_MECHANISTIC_0006.json` containing: + +- candidate commit; +- focused RED and GREEN run IDs; +- full corpus run ID; +- artifact ID and digest; +- report digest; +- diagnosis digest; +- trace count and total event count; +- compact/mechanistic parity count; +- frozen-core commit; +- unchanged training metrics; +- exact trace references for `aabf363d` and `b230c067`; +- explicit statement that public ARC evaluation was not rerun. + +- [ ] **Step 8: Commit the workflow and evidence** + +```bash +git add \ + .github/workflows/arc-training-corpus-v2-2.yml \ + research/universal_core/holonomy_v2_2/evidence/MECHANISTIC_TRACE_TDD.json \ + docs/evidence/arc_v2_2/TRAINING_MECHANISTIC_0006.json +git commit -m "arc: seal mechanistic training evidence" +``` + +## Plan self-review + +- Spec coverage: recorder, complete candidate lifecycle, hidden execution, equivalence classes, final decisions, parity, deterministic replay, task-ID invariance, training traces, diagnosis, workflow artifact, frozen boundary, and both root-cause confirmations are assigned to explicit tasks. +- Scope: capability upgrades are intentionally excluded from this plan and begin only after Task 5 seals root-cause evidence. +- Placeholder scan: no `TBD`, `TODO`, deferred implementation instruction, or unnamed error-handling step remains. +- Type consistency: `TraceRecorder`, `validate_trace`, `write_trace`, `mechanistic`, `mechanistic_dir`, `verify_parity`, and `build_trace_diagnosis` use the same names and signatures throughout. diff --git a/docs/superpowers/plans/2026-08-04-dihedral-quadrant-mosaic.md b/docs/superpowers/plans/2026-08-04-dihedral-quadrant-mosaic.md new file mode 100644 index 00000000..863efe50 --- /dev/null +++ b/docs/superpowers/plans/2026-08-04-dihedral-quadrant-mosaic.md @@ -0,0 +1,376 @@ +# Dihedral Quadrant Mosaic Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a generic, fully traced `dihedral_quadrant_mosaic` ARC primitive that converts nine training-only grammar gaps into exact acceptances without changing any prior correct row or the frozen V2.1 core. + +**Architecture:** A focused `quadrant_mosaic.py` module derives transform combinations by independently matching each source-sized output quadrant against shape-preserving dihedral transforms across all demonstrations. `grid_induction.py` executes the selected program, while `grid_mechanistic_runtime.py` exposes every structural hypothesis through the existing candidate lifecycle and hidden-output consensus. + +**Tech Stack:** Python 3.12, pytest, GitHub Actions, canonical JSON evidence, SHA-256 artifact manifests. + +## Global Constraints + +- Use only the 1,000 ARC training tasks for development and acceptance. +- Do not rerun the 120-task public ARC evaluation set. +- Do not modify `research/universal_core/holonomy_v2`; require byte identity against `1d7c489bcdbff755d638b35ad47ce62bd4fe829d`. +- Do not route by task ID or prefer one fitting transform without output consensus. +- Preserve compact/mechanistic result parity for all 1,000 training tasks. +- Expected cumulative corpus result: 78 correct and accepted rows, zero incorrect attempts, zero failures, one `AMBIGUOUS_PROGRAM`, and 997 `GRAMMAR_EXHAUSTED` rows. + +--- + +### Task 1: Immutable fixtures and RED contract + +**Files:** +- Create: `research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/0c786b71.json` +- Create: `research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/3af2c5a8.json` +- Create: `research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/46442a0e.json` +- Create: `research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/62c24649.json` +- Create: `research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/67e8384a.json` +- Create: `research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/7953d61e.json` +- Create: `research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/7fe24cdd.json` +- Create: `research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/833dafe3.json` +- Create: `research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/ed98d772.json` +- Create: `research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py` +- Create: `.github/workflows/arc-quadrant-mosaic-red.yml` +- Evidence: `research/universal_core/holonomy_v2_2/evidence/QUADRANT_MOSAIC_RED.json` + +**Interfaces:** +- Consumes: pinned ARC training JSON payloads. +- Produces: a focused test contract importing `apply_quadrant_mosaic` and `quadrant_mosaic_programs` from the intentionally absent module. + +- [ ] **Step 1: Import the nine exact training payloads** + +Extract the files without alteration from: + +```text +research/universal_core/public_benchmarks/vendor/arc_agi_2/data/training/.json +``` + +Verify each fixture retains both `train` and `test` arrays and contains exactly one hidden test row. + +- [ ] **Step 2: Write direct execution tests** + +Create tests covering the canonical reflected mosaic: + +```python +program = { + "kind": "dihedral_quadrant_mosaic", + "top_left": "identity", + "top_right": "reflect_columns", + "bottom_left": "reflect_rows", + "bottom_right": "rotate_180", +} +assert apply_quadrant_mosaic(program, [[1, 2], [3, 4]]) == [ + [1, 2, 2, 1], + [3, 4, 4, 3], + [3, 4, 4, 3], + [1, 2, 2, 1], +] +``` + +Also require `ValueError` when a 16-by-16 source would produce a 32-by-32 output. + +- [ ] **Step 3: Write deterministic derivation tests** + +Use demonstrations whose four quadrants uniquely identify: + +```python +( + "identity", + "reflect_columns", + "reflect_rows", + "rotate_180", +) +``` + +Assert `quadrant_mosaic_programs(demos)` returns one canonical program and `reason is None`. Add a non-double-dimension demonstration and require `programs == ()` with `reason == "OUTPUT_NOT_DOUBLE_SOURCE_SHAPE"`. + +- [ ] **Step 4: Write exact real-task tests** + +For each of the nine fixture IDs, build the public task using `train` as demonstrations and only `test[*].input` as hidden inputs. Require: + +```python +result["predictions"] == [ + {"status": "ACCEPTED", "prediction": item["output"]} + for item in payload["test"] +] +``` + +In mechanistic mode, require terminal status `ACCEPTED`, at least one successful hidden execution with `candidate_kind == "dihedral_quadrant_mosaic"`, and a selected candidate ID beginning with `quadrant_mosaic:`. + +- [ ] **Step 5: Write ambiguity regression** + +Use a demonstration source that is invariant under multiple transforms and an output composed from that symmetric source. Use an asymmetric hidden source. Require multiple fitting quadrant-mosaic candidates to produce at least two output classes and the final prediction status to be `AMBIGUOUS_PROGRAM`. + +- [ ] **Step 6: Run focused RED** + +Run: + +```bash +python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py +``` + +Expected: collection fails only with: + +```text +ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.quadrant_mosaic' +``` + +- [ ] **Step 7: Commit RED evidence** + +The workflow must reject any other failure reason, then commit `QUADRANT_MOSAIC_RED.json` containing the source commit, run ID, exact command, expected missing module, nine expected gains, frozen-core SHA, and `public_arc_evaluation_rerun: false`. + +--- + +### Task 2: Isolated dihedral transform and candidate engine + +**Files:** +- Create: `research/universal_core/holonomy_v2_2/quadrant_mosaic.py` +- Test: `research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py` + +**Interfaces:** +- Produces: `apply_quadrant_mosaic(program: Mapping[str, Any], grid: Grid) -> Grid`. +- Produces: `quadrant_mosaic_programs(demos: Sequence[tuple[Grid, Grid]]) -> tuple[tuple[Program, ...], str | None]`. + +- [ ] **Step 1: Define canonical transform order** + +```python +_TRANSFORM_ORDER = ( + "identity", + "rotate_90", + "rotate_180", + "rotate_270", + "reflect_columns", + "reflect_rows", + "reflect_main_diagonal", + "reflect_anti_diagonal", +) +``` + +- [ ] **Step 2: Implement transform application** + +Implement `_apply_transform(name, grid)` using copied lists. Reject unknown names. Quarter-turn and diagonal transforms may return swapped dimensions; caller validation determines whether they are shape-preserving. + +- [ ] **Step 3: Implement quadrant splitting and execution** + +`apply_quadrant_mosaic` must: + +1. reject source height or width greater than 15; +2. apply the four named transforms in top-left, top-right, bottom-left, bottom-right order; +3. require every transformed grid to equal the source shape; +4. concatenate top quadrants row-wise, then bottom quadrants row-wise; +5. return a fresh grid. + +- [ ] **Step 4: Implement demonstration-derived candidates** + +For each demonstration: + +```python +expected_shape = (2 * source_height, 2 * source_width) +``` + +Return `OUTPUT_NOT_DOUBLE_SOURCE_SHAPE` if any target differs. For each quadrant, intersect the transform names whose transformed source exactly matches the target quadrant across every demonstration. Return `NO_SUPPORTED_QUADRANT_TRANSFORM` if any intersection is empty. Otherwise return the Cartesian product in `_TRANSFORM_ORDER`, encoded as canonical program dictionaries. + +- [ ] **Step 5: Run module tests GREEN** + +Run: + +```bash +python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py +``` + +Expected: direct module tests pass; full integration tests may still fail because dispatch is not wired. + +- [ ] **Step 6: Commit isolated implementation** + +```bash +git add research/universal_core/holonomy_v2_2/quadrant_mosaic.py +git commit -m "arc: add dihedral quadrant mosaic primitive" +``` + +--- + +### Task 3: Interpreter and mechanistic integration + +**Files:** +- Modify: `research/universal_core/holonomy_v2_2/grid_induction.py` +- Modify: `research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py` +- Create: `.github/workflows/arc-quadrant-mosaic-wire.yml` +- Evidence: `research/universal_core/holonomy_v2_2/evidence/QUADRANT_MOSAIC_GREEN.json` + +**Interfaces:** +- Consumes: `apply_quadrant_mosaic` and `quadrant_mosaic_programs`. +- Produces: grammar version `grid-v2.2-10` and generator prefix `quadrant_mosaic:`. + +- [ ] **Step 1: Add interpreter dispatch** + +In `grid_induction.py`, import `apply_quadrant_mosaic`, advance `_VERSION` from `grid-v2.2-9` to `grid-v2.2-10`, and add: + +```python +if kind == "dihedral_quadrant_mosaic": + return apply_quadrant_mosaic(program, grid) +``` + +- [ ] **Step 2: Add structural generator batch** + +In `grid_mechanistic_runtime.py`, import `quadrant_mosaic_programs`, derive: + +```python +quadrant_mosaic, quadrant_mosaic_reason = quadrant_mosaic_programs(demos) +``` + +and append: + +```python +("quadrant_mosaic", quadrant_mosaic, quadrant_mosaic_reason) +``` + +to `_generator_batches` after `mirror_concat`. + +- [ ] **Step 3: Syntax and diff gates** + +Run: + +```bash +python -m py_compile \ + research/universal_core/holonomy_v2_2/quadrant_mosaic.py \ + research/universal_core/holonomy_v2_2/grid_induction.py \ + research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py +git diff --check +``` + +- [ ] **Step 4: Run focused and cluster GREEN** + +Run: + +```bash +python -m pytest -q \ + research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py \ + research/universal_core/holonomy_v2_2/tests/test_mirror_concat.py \ + research/universal_core/holonomy_v2_2/tests/test_bbox_complete.py \ + research/universal_core/holonomy_v2_2/tests/test_component_outlier.py \ + research/universal_core/holonomy_v2_2/tests/test_marker_recolor.py \ + research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py +python -m pytest -q research/universal_core/holonomy_v2_2/tests +``` + +- [ ] **Step 5: Run frozen gates** + +```bash +PYTHONPATH=research python -m pytest -q \ + research/universal_core/holonomy_v2/tests/test_blind_adapter.py \ + research/universal_core/holonomy_v2/tests/test_engine.py +git diff --quiet 1d7c489bcdbff755d638b35ad47ce62bd4fe829d -- \ + research/universal_core/holonomy_v2 +``` + +- [ ] **Step 6: Commit only on complete GREEN** + +The guarded workflow restores both integration files on any failed gate. On success it commits the two integration files and `QUADRANT_MOSAIC_GREEN.json`, recording focused/full/frozen counts, run ID, expected nine-task delta, and byte-gate result. + +--- + +### Task 4: Strict 1,000-task mechanistic corpus gate + +**Files:** +- Create: `.github/workflows/arc-quadrant-mosaic-corpus.yml` +- Artifact: `arc-v2-2-quadrant-mosaic-${GITHUB_SHA}` + +**Interfaces:** +- Consumes: immutable benchmark artifact from run `30929169392` and immutable V2.2 baseline report from run `30941837872`. +- Produces: `report.json`, `traces/*.json`, `traces/diagnosis.json`, `transitions.json`, `mechanisms.json`, `parity.json`, and `SHA256SUMS`. + +- [ ] **Step 1: Run compact and mechanistic modes for all training tasks** + +Call `run_training_corpus(..., mechanistic_dir=trace_dir, verify_parity=True)` and require 1,000 trace files, 1,000 validated traces, and parity count 1,000. + +- [ ] **Step 2: Enforce exact cumulative metrics** + +Require: + +```python +report["metrics"]["rows"] == 1076 +report["metrics"]["correct"] == 78 +report["metrics"]["accepted"] == 78 +report["metrics"]["incorrect_attempts"] == 0 +report["metrics"]["failed"] == 0 +report["status_counts"] == { + "ACCEPTED": 78, + "AMBIGUOUS_PROGRAM": 1, + "GRAMMAR_EXHAUSTED": 997, +} +``` + +- [ ] **Step 3: Enforce exact cumulative gain set** + +Require no lost baseline row and exactly these 19 gains over `e509650...`: + +```text +0c786b71, 3aa6fb7a, 3af2c5a8, 46442a0e, 4c4377d9, +62c24649, 67e8384a, 6d0aefbc, 6d75e8bb, 6fa7a44f, +7953d61e, 7fe24cdd, 833dafe3, 8be77c9e, aabf363d, +b230c067, c9e6f938, e7639916, ed98d772 +``` + +- [ ] **Step 4: Verify mechanism selection** + +For each of the nine new tasks require terminal status `ACCEPTED`, at least one successful `dihedral_quadrant_mosaic` hidden execution, and selected candidate ID prefix `quadrant_mosaic:`. Recheck the prior marker, component-outlier, bounding-box, and mirror task prefixes. Require `60b61512` to remain `AMBIGUOUS_PROGRAM` with at least two bounding-box output classes. + +- [ ] **Step 5: Build and verify artifact manifest** + +Generate `SHA256SUMS` only after every writer closes, then run: + +```bash +cd /tmp/arc-v2-2-quadrant-mosaic +sha256sum -c SHA256SUMS +``` + +- [ ] **Step 6: Publish the artifact** + +Upload every report, trace, and manifest with 90-day retention. The workflow fails before publication on any metric, transition, trace, parity, or checksum mismatch. + +--- + +### Task 5: Independent artifact validation and evidence sealing + +**Files:** +- Create: `docs/evidence/arc_v2_2/TRAINING_QUADRANT_MOSAIC_0011.json` +- Optionally update: PR #40 body with the current primitive set and latest verified training result. + +**Interfaces:** +- Consumes: exact successful corpus run and artifact ID. +- Produces: immutable repository evidence referencing the evaluated commit, run, artifact digest, report digest, diagnosis digest, counts, transitions, and mechanism trace digests. + +- [ ] **Step 1: Download the exact artifact by ID** + +Do not resolve by “latest.” Match workflow name, evaluated head SHA, artifact name, and artifact ID. + +- [ ] **Step 2: Verify every checksum independently** + +Run `sha256sum -c SHA256SUMS`, require 1,006 entries, parse all 1,000 task traces, and call `validate_trace` on each. + +- [ ] **Step 3: Recompute acceptance assertions** + +Independently read `report.json`, `transitions.json`, and `mechanisms.json`; require 78 correct/accepted, zero incorrect attempts, the exact 19-task gain set, nine quadrant-mosaic selected candidates, and preserved `60b61512` ambiguity. + +- [ ] **Step 4: Commit sealed evidence** + +Write `TRAINING_QUADRANT_MOSAIC_0011.json` with: + +- candidate commit; +- workflow run and job IDs; +- artifact ID, name, size, and SHA-256 digest; +- checksum count and validation result; +- report and diagnosis digests; +- metrics and status counts; +- total trace event count; +- selected-kind and hidden-failure counts; +- exact gained/lost rows; +- per-task selected candidate and trace digest; +- frozen-core SHA and byte-gate result; +- `public_arc_evaluation_rerun: false`. + +- [ ] **Step 5: Verify branch and PR state** + +Confirm PR #40 remains open, draft, unmerged, and based on `universal-core-public-benchmark-adapters-v1`. Do not merge it. diff --git a/docs/superpowers/plans/2026-08-04-palette-cardinality-repeat.md b/docs/superpowers/plans/2026-08-04-palette-cardinality-repeat.md new file mode 100644 index 00000000..4432af0b --- /dev/null +++ b/docs/superpowers/plans/2026-08-04-palette-cardinality-repeat.md @@ -0,0 +1,91 @@ +# Palette-Cardinality Repeat Implementation Plan + +> **Execution:** Use checkpointed strict TDD on `universal-core-arc-training-v2-2`. Do not touch the frozen V2.1 core or public ARC evaluation data. + +**Goal:** Add `palette_cardinality_repeat` and convert exactly four training-only grammar gaps into correct acceptances. + +## Constraints + +- Training corpus only: 1,000 ARC tasks / 1,076 rows. +- Frozen path `research/universal_core/holonomy_v2` must remain byte-identical to `1d7c489bcdbff755d638b35ad47ce62bd4fe829d`. +- Public ARC evaluation rerun: prohibited. +- Expected cumulative metrics: 82 correct/accepted, zero incorrect attempts, zero failures. +- Expected statuses: ACCEPTED 82, AMBIGUOUS_PROGRAM 1, GRAMMAR_EXHAUSTED 993. +- Exact new gains: `a59b95c0`, `ac0a08a4`, `b91ae062`, `d4b1c2b1`. + +### Task 1: Pinned fixtures and RED contract + +**Create:** +- `research/universal_core/holonomy_v2_2/tests/fixtures/palette_repeat/{a59b95c0,ac0a08a4,b91ae062,d4b1c2b1}.json` +- `research/universal_core/holonomy_v2_2/tests/test_palette_repeat_fixtures.py` +- `research/universal_core/holonomy_v2_2/tests/test_palette_repeat.py` +- `.github/workflows/arc-palette-repeat-red.yml` + +- Import each fixture byte-for-byte from the pinned ARC training corpus. +- Pin each fixture SHA-256 in a focused integrity test. +- Add direct execution tests for: + - scale by `distinct_all`; + - tile by `distinct_all`; + - scale by `distinct_non_background`. +- Add candidate derivation tests requiring factor variation across demonstrations. +- Add a constant-factor test requiring `NO_FACTOR_VARIATION`. +- Add exact mechanistic acceptance tests for all four real tasks. +- Run focused RED and require only missing `palette_repeat` module. +- Commit RED evidence with exact run ID and task list. + +### Task 2: Isolated implementation + +**Create:** `research/universal_core/holonomy_v2_2/palette_repeat.py` + +Implement: + +```python +apply_palette_repeat(program, grid) -> Grid +palette_repeat_programs(demos) -> tuple[tuple[Program, ...], str | None] +``` + +- Supported operations: `scale`, `tile`. +- Supported statistics: `distinct_all`, `distinct_non_background`. +- Background candidates for the second statistic must appear in every demonstration input. +- Retain candidates only when all demonstrations execute exactly. +- Require at least two distinct demonstrated factors. +- Enforce factor >= 1 and ARC 30-by-30 bounds. +- Run direct module tests GREEN and commit the isolated module. + +### Task 3: Guarded integration + +**Modify:** +- `research/universal_core/holonomy_v2_2/grid_induction.py` +- `research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py` + +**Create:** `.github/workflows/arc-palette-repeat-wire.yml` + +- Import and dispatch `apply_palette_repeat` for kind `palette_cardinality_repeat`. +- Advance grammar version from `grid-v2.2-10` to `grid-v2.2-11`. +- Register generator batch prefix `palette_repeat:`. +- Run fixture, palette, quadrant, mirror, bbox, component-outlier, marker, mechanistic, and fixed scale/tile overlap tests. +- Run all V2.2 tests, frozen integration tests, and frozen byte gate. +- Commit shared files only on complete GREEN; otherwise restore them and commit diagnostics. + +### Task 4: Strict 1,000-task corpus acceptance + +**Create:** `.github/workflows/arc-palette-repeat-corpus.yml` + +- Run all training tasks compact + mechanistic with parity verification. +- Validate all 1,000 traces. +- Require exact cumulative gain set of 23 tasks: prior 19 plus the four new IDs. +- Require metrics 82/1,076, zero wrong attempts/failures. +- Require status counts ACCEPTED 82, AMBIGUOUS_PROGRAM 1, GRAMMAR_EXHAUSTED 993. +- Require each new task select prefix `palette_repeat:` and retain prior primitive-selection guards. +- Require `60b61512` remain ambiguity-safe. +- Generate `SHA256SUMS` after writers close, verify it, and publish the exact artifact. + +### Task 5: Independent validation and evidence sealing + +**Create:** `docs/evidence/arc_v2_2/TRAINING_PALETTE_REPEAT_0012.json` + +- Resolve artifact by exact evaluated SHA and workflow run. +- Independently verify all checksums and 1,000 traces. +- Recompute metrics, transitions, selected kinds, and per-task mechanisms. +- Commit run/job/artifact IDs and digests, report/diagnosis digests, event count, exact gained/lost rows, frozen byte gate, and `public_arc_evaluation_rerun: false`. +- Confirm PR #40 remains draft, open, and unmerged. diff --git a/docs/superpowers/plans/2026-08-05-axis-duplicate-collapse.md b/docs/superpowers/plans/2026-08-05-axis-duplicate-collapse.md new file mode 100644 index 00000000..c3f40fbe --- /dev/null +++ b/docs/superpowers/plans/2026-08-05-axis-duplicate-collapse.md @@ -0,0 +1,68 @@ +# Axis Duplicate Collapse Implementation Plan + +**Goal:** Add `axis_duplicate_collapse` and convert exactly six ARC training grammar gaps into correct acceptances. + +## Constraints + +- Use only the 1,000 ARC training tasks / 1,076 rows. +- Do not rerun public ARC evaluation. +- Keep `research/universal_core/holonomy_v2` byte-identical to `1d7c489bcdbff755d638b35ad47ce62bd4fe829d`. +- Expected cumulative result: 88 correct/accepted, zero incorrect attempts/failures. +- Expected statuses: ACCEPTED 88, AMBIGUOUS_PROGRAM 1, GRAMMAR_EXHAUSTED 987. +- Exact new gains: `2dee498d`, `746b3537`, `7b7f7511`, `ce8d95cc`, `e1baa8a4`, `eb5a1d5d`. + +### Task 1: Immutable fixtures and RED + +Create: + +- `research/universal_core/holonomy_v2_2/tests/fixtures/axis_collapse/*.json` +- `research/universal_core/holonomy_v2_2/tests/test_axis_collapse_fixtures.py` +- `research/universal_core/holonomy_v2_2/tests/test_axis_collapse.py` +- `.github/workflows/arc-axis-collapse-red.yml` + +Restore the six JSON files directly from immutable corpus artifact run `30929169392`. Verify exact SHA-256 values. Test adjacent-run collapse, global-first collapse, stable first-occurrence ordering, row-only/column-only/both modes, row/column commutativity, no-effect generator skip, and exact mechanistic acceptance for all six tasks. Require missing-module RED and commit evidence. + +### Task 2: Isolated module + +Create `research/universal_core/holonomy_v2_2/axis_collapse.py` with: + +```python +apply_axis_collapse(program, grid) -> Grid +axis_collapse_programs(demos) -> tuple[tuple[Program, ...], str | None] +``` + +Support scopes `adjacent_runs` and `global_first`; axes `rows`, `columns`, `both`. Use one canonical rows-then-columns execution for `both`; the operations commute. Expose candidates only when they change at least one demonstration. Return `NO_DUPLICATE_AXIS_REDUCTION` when none changes. Verify direct module tests GREEN. + +### Task 3: Guarded integration + +Modify `grid_induction.py` and `grid_mechanistic_runtime.py` under a fail-closed workflow: + +- dispatch kind `axis_duplicate_collapse`; +- advance version `grid-v2.2-11` to `grid-v2.2-12`; +- add generator prefix `axis_collapse:`. + +Run axis tests, palette, quadrant, mirror, bbox, component-outlier, marker, mechanistic tests, all V2.2 tests, frozen integration tests, and frozen-byte gate. Commit shared files only on complete GREEN. + +### Task 4: Strict corpus gate + +Create `.github/workflows/arc-axis-collapse-corpus.yml`. + +Run all training tasks compact and mechanistic. Require: + +- 1,000 validated traces and 1,000 parity matches; +- exact cumulative 29-task gain set: prior 23 plus the six new task IDs; +- metrics 88 correct/accepted, zero wrong attempts/failures; +- statuses ACCEPTED 88, AMBIGUOUS_PROGRAM 1, GRAMMAR_EXHAUSTED 987; +- each new task selects `axis_collapse:`; +- all prior primitive-selection guards remain stable; +- `60b61512` remains ambiguity-safe. + +Generate and verify `SHA256SUMS`, then publish the exact artifact. + +### Task 5: Independent validation and evidence + +Independently verify all artifact checksums and traces, recompute metrics/transitions/mechanisms/event count, and commit: + +- `docs/evidence/arc_v2_2/TRAINING_AXIS_COLLAPSE_0013.json` + +Record exact run/job/artifact IDs and digests, frozen byte gate, and `public_arc_evaluation_rerun: false`. Confirm PR #40 remains draft and unmerged. diff --git a/docs/superpowers/plans/2026-08-05-fixed-block-reduction.md b/docs/superpowers/plans/2026-08-05-fixed-block-reduction.md new file mode 100644 index 00000000..4426e763 --- /dev/null +++ b/docs/superpowers/plans/2026-08-05-fixed-block-reduction.md @@ -0,0 +1,45 @@ +# Fixed-Block Reduction Implementation Plan + +**Goal:** Add `fixed_block_reduce` and convert exactly two ARC training grammar gaps into correct acceptances. + +## Constraints + +- Training only: 1,000 tasks / 1,076 rows. +- Public ARC evaluation rerun prohibited. +- Frozen V2.1 bytes unchanged against `1d7c489bcdbff755d638b35ad47ce62bd4fe829d`. +- Expected cumulative metrics: 90 correct/accepted, zero incorrect attempts/failures. +- Expected statuses: ACCEPTED 90, AMBIGUOUS_PROGRAM 1, GRAMMAR_EXHAUSTED 985. +- Exact new gains: `5614dbcf`, `68b67ca3`. + +### Task 1: Fixtures and RED + +Create fixture integrity and behavior tests under `tests/fixtures/block_reduce`, `test_block_reduce_fixtures.py`, and `test_block_reduce.py`. Restore source JSON directly from immutable artifact run `30929169392`. Test strict mode, tie failure, unique non-background, conflicting non-background failure, stable factor derivation, unstable-factor skip, and exact mechanistic acceptance for both tasks. Verify missing-module RED and commit evidence. + +### Task 2: Isolated module + +Create `block_reduce.py` exporting: + +```python +apply_block_reduce(program, grid) -> Grid +block_reduce_programs(demos) -> tuple[tuple[Program, ...], str | None] +``` + +Implement stable block factors, `strict_mode`, and `unique_non_background`. Enforce divisibility, nonempty output, and ARC bounds. Verify direct module GREEN. + +### Task 3: Guarded integration + +Modify `grid_induction.py` and `grid_mechanistic_runtime.py` only through a fail-closed workflow: + +- dispatch `fixed_block_reduce`; +- advance `grid-v2.2-12` to `grid-v2.2-13`; +- register generator prefix `block_reduce:`. + +Run block, axis, palette, quadrant, mirror, bbox, component-outlier, marker, mechanistic, full V2.2, frozen integration, and byte gates. Commit shared files only on complete GREEN. + +### Task 4: Strict corpus gate + +Run all 1,000 training tasks compact and mechanistic. Require exactly 31 cumulative gains, including `5614dbcf` and `68b67ca3`; 90 correct/accepted; zero incorrect attempts/failures; statuses 90/1/985; 1,000 traces and parity matches; prior primitive guards unchanged; `60b61512` ambiguity preserved. Build and verify `SHA256SUMS`, then publish the exact artifact. + +### Task 5: Independent validation + +Independently verify all checksums and traces, recompute metrics/transitions/mechanisms/event count, and seal `docs/evidence/arc_v2_2/TRAINING_BLOCK_REDUCE_0014.json`. Record exact run/job/artifact IDs and digests, frozen byte gate, and `public_arc_evaluation_rerun: false`. Keep PR #40 draft and unmerged. diff --git a/docs/superpowers/plans/2026-08-15-anchor-ray-extension.md b/docs/superpowers/plans/2026-08-15-anchor-ray-extension.md new file mode 100644 index 00000000..9ee331d2 --- /dev/null +++ b/docs/superpowers/plans/2026-08-15-anchor-ray-extension.md @@ -0,0 +1,78 @@ +# Anchor-to-Border Ray Extension Implementation Plan + +**Goal:** Add `anchor_ray_extend` as a generic V2.2 primitive that converts exactly `623ea044` and `d037b0a7` from grammar exhaustion to exact acceptance without changing any prior result. + +## Global gates + +- Training-only development on 1,000 ARC tasks / 1,076 hidden rows. +- No public ARC evaluation rerun. +- Frozen V2.1 byte identity against `1d7c489bcdbff755d638b35ad47ce62bd4fe829d`. +- No task-ID routing or preferred fitting candidate. +- Final exact metrics: 101 correct, 101 accepted, 0 incorrect attempts, 0 failures. +- Final statuses: 101 `ACCEPTED`, 2 `AMBIGUOUS_PROGRAM`, 973 `GRAMMAR_EXHAUSTED`. +- Cumulative correct gain set: prior 40 gains plus `623ea044`, `d037b0a7`. +- Preserve ambiguity for both `60b61512` and `b8825c91`. + +## Task 1 — Exact fixtures and RED contract + +Pin exact payloads from immutable benchmark artifact run `30929169392`: + +```text +623ea044 578ab7e47a2d67489f1103715615f719ef81e9faf773be85586803d6da44cbdd +d037b0a7 9e39a4acdb3e3bff7dd54bed5f4ab77973f4027d48beb62a9c740d8a7a0b0aee +``` + +Create fixture integrity and behavior tests before production code. + +Direct tests cover cardinal rays, diagonal rays, combined directions, original-anchor-only behavior, cross-color claim conflict, unique demonstration derivation, shape mismatch, no-op skip, and two exact real-task mechanistic assertions. + +Hosted RED accepts only missing `ray_extend` module after fixture and frozen byte checks pass. + +## Task 2 — Isolated module GREEN + +Create `ray_extend.py` with: + +```python +apply_ray_extend(program, grid) +ray_extend_programs(demos) +``` + +Use original non-background anchors only. Precompute claims to the border. Reject different-color original overwrite and cross-color claim collisions. Candidate order: background ascending, then `up`, `down`, `left`, `right`, `vertical`, `horizontal`, `orthogonal`, `diagonal`, `all`. Require exact demo fit and at least one changed demo. + +Run fixture plus direct tests excluding real integration assertions. Require frozen byte gate and seal module GREEN evidence. + +## Task 3 — Guarded integration + +Fail-closed workflow: + +- `grid_induction.py`: import executor, `grid-v2.2-16` -> `grid-v2.2-17`, dispatch `anchor_ray_extend`. +- `grid_mechanistic_runtime.py`: import generator, derive after symmetry completion, append `ray_extend` after `symmetry_complete`. + +Run ray, symmetry, line, gravity, block, axis, palette, quadrant, mirror, bbox, outlier, marker, mechanistic, all V2.2, frozen integration, and byte gates. Restore shared files on any failure. + +## Task 4 — Strict corpus gate + +Run all 1,000 tasks with 1,000 validated traces and compact/mechanistic parity. + +Require: + +```text +rows 1076 +correct 101 +accepted 101 +incorrect_attempts 0 +failed 0 +ACCEPTED 101 +AMBIGUOUS_PROGRAM 2 +GRAMMAR_EXHAUSTED 973 +``` + +Require exact cumulative correct gain set of 42 tasks. New tasks must select prefix `ray_extend:` with successful `anchor_ray_extend` executions. Preserve all prior mechanism selectors, `b8825c91` symmetry ambiguity, and `60b61512` bbox ambiguity. + +Publish complete checksummed artifact only on full GREEN. + +## Task 5 — Independent validation and sealing + +Verify artifact digest, manifest, 1,000 traces, 1,000 parity checks, exact metrics/statuses/gains, two ray mechanisms, all prior selectors, both ambiguity cases, and zero wrong accepts. + +Seal as `docs/evidence/arc_v2_2/TRAINING_RAY_EXTEND_0018.json`. Keep PR #40 draft and unmerged. diff --git a/docs/superpowers/plans/2026-08-15-axis-gravity-compaction.md b/docs/superpowers/plans/2026-08-15-axis-gravity-compaction.md new file mode 100644 index 00000000..7aae65e6 --- /dev/null +++ b/docs/superpowers/plans/2026-08-15-axis-gravity-compaction.md @@ -0,0 +1,162 @@ +# Axis Gravity Compaction Implementation Plan + +**Goal:** Add `axis_gravity_compact` as a generic V2.2 training-only primitive that converts exactly `1e0a9b12` and `3906de3d` from grammar exhaustion to exact acceptance without changing any prior result. + +**Architecture:** A standalone `gravity_compact.py` executor compacts non-background cells independently within rows or columns while preserving order. Candidate derivation enumerates common demonstrated backgrounds and four canonical directions, retains only exact non-no-op demonstration fits, and delegates hidden disagreement to the existing consensus machinery. + +## Global gates + +- Develop only against the 1,000 ARC training tasks. +- Never rerun the 120-task public ARC evaluation split during this cycle. +- Never modify `research/universal_core/holonomy_v2`; byte gate against `1d7c489bcdbff755d638b35ad47ce62bd4fe829d`. +- No task-ID routing or preferred candidate heuristic. +- Preserve all prior selected mechanism prefixes. +- Preserve `60b61512` as `AMBIGUOUS_PROGRAM` with at least two bbox output classes. +- Expected final training metrics: 1,076 rows, 92 correct, 92 accepted, 0 incorrect attempts, 0 failures. +- Expected final statuses: 92 `ACCEPTED`, 1 `AMBIGUOUS_PROGRAM`, 983 `GRAMMAR_EXHAUSTED`. + +## Task 1 — Pin fixtures and RED contract + +Create: + +- `research/universal_core/holonomy_v2_2/tests/fixtures/gravity_compact/1e0a9b12.json` +- `research/universal_core/holonomy_v2_2/tests/fixtures/gravity_compact/3906de3d.json` +- `research/universal_core/holonomy_v2_2/tests/test_gravity_compact_fixtures.py` +- `research/universal_core/holonomy_v2_2/tests/test_gravity_compact.py` +- `.github/workflows/arc-gravity-compact-red.yml` + +Fixture SHA-256 values must be exactly: + +```text +1e0a9b12.json 6a71dbedc8403bd28fa76bd8b472597a8ecaab3af31151a2a3b72baeadd5e617 +3906de3d.json b4c20d433380e00840a75c6976ad0d0fabd80e052ab77b47422d64b1cae5f6dc +``` + +Direct tests must cover: + +1. vertical `up` and `down` compaction; +2. horizontal `left` and `right` compaction; +3. preservation of non-background lane order and color counts; +4. unknown direction rejection; +5. candidate derivation returns the unique demonstrated candidate; +6. shape mismatch returns `NO_SHAPE_PRESERVING_GRAVITY`; +7. no-op demonstrations return no candidate with `NO_GRAVITY_COMPACTION`; +8. both pinned real tasks become trace-explained exact acceptances after integration. + +Run RED and accept only: + +```text +ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.gravity_compact' +``` + +Record `GRAVITY_COMPACT_RED.json` with run ID, source SHA, fixture integrity, two expected gains, frozen SHA, and `public_arc_evaluation_rerun: false`. + +## Task 2 — Isolated module GREEN + +Create `research/universal_core/holonomy_v2_2/gravity_compact.py`. + +Interfaces: + +```python +apply_gravity_compact(program: Mapping[str, Any], grid: Grid) -> Grid +gravity_compact_programs(demos: Sequence[tuple[Grid, Grid]]) -> tuple[tuple[Program, ...], str | None] +``` + +Implementation constraints: + +- copy the input; never mutate it; +- canonical direction order: `up`, `down`, `left`, `right`; +- vertical directions compact columns; horizontal directions compact rows; +- preserve non-background order exactly; +- candidate background set is the intersection of source colors across demonstrations; +- candidate must fit every demonstration exactly; +- candidate must change at least one source; +- deterministic program order: background ascending, then canonical direction order; +- return `NO_DEMONSTRATION`, `NO_SHAPE_PRESERVING_GRAVITY`, or `NO_GRAVITY_COMPACTION` as applicable. + +Run fixture plus direct tests excluding real integration assertions. Require frozen byte gate. Record `GRAVITY_COMPACT_MODULE_GREEN.json`. + +## Task 3 — Guarded integration + +Modify only through a fail-closed workflow: + +- `grid_induction.py` + - import `apply_gravity_compact`; + - advance `_VERSION` from `grid-v2.2-13` to `grid-v2.2-14`; + - dispatch kind `axis_gravity_compact`. +- `grid_mechanistic_runtime.py` + - import `gravity_compact_programs`; + - derive candidates after `block_reduce`; + - register generator batch `gravity_compact` after `block_reduce`. + +Run: + +- all gravity tests; +- block-reduction tests; +- axis-collapse tests; +- palette-repeat tests; +- quadrant-mosaic tests; +- mirror/bbox/component-outlier/marker/mechanistic regression cluster; +- all V2.2 tests; +- frozen V2 integration tests; +- frozen byte gate. + +On any failure, restore both shared runtime files and commit only diagnostics. On complete GREEN, commit shared files and `GRAVITY_COMPACT_GREEN.json`. + +## Task 4 — Strict 1,000-task corpus gate + +Use the immutable benchmark artifact from run `30929169392` and original V2.2 baseline report from run `30941837872`. + +Require: + +```text +rows 1076 +correct 92 +accepted 92 +incorrect_attempts 0 +failed 0 +ACCEPTED 92 +AMBIGUOUS_PROGRAM 1 +GRAMMAR_EXHAUSTED 983 +trace_count 1000 +parity_count 1000 +``` + +Require the cumulative gain set to equal the prior 31 fixed-block gains plus exactly: + +```text +1e0a9b12 +3906de3d +``` + +For both new tasks require: + +- terminal status `ACCEPTED`; +- a successful hidden execution with candidate kind `axis_gravity_compact`; +- selected candidate ID beginning `gravity_compact:`. + +Recheck every prior primitive mechanism guard and `60b61512` ambiguity. + +Generate `SHA256SUMS` after all writers close, verify it independently, then publish the complete artifact. + +## Task 5 — Independent validation and sealing + +Download the exact artifact by ID. Independently require: + +- ZIP SHA equals GitHub artifact digest; +- 1,006 manifest entries and zero checksum failures; +- 1,000 task trace files; +- 1,000 compact/mechanistic parity checks and zero mismatches; +- exact 92/1/983 status boundary; +- zero lost baseline rows; +- 33 cumulative gain tasks; +- exact gravity task mechanism traces; +- preserved `60b61512` ambiguity. + +Seal as: + +```text +docs/evidence/arc_v2_2/TRAINING_GRAVITY_COMPACT_0015.json +``` + +Keep PR #40 draft and unmerged. diff --git a/docs/superpowers/plans/2026-08-15-reflective-symmetry-completion.md b/docs/superpowers/plans/2026-08-15-reflective-symmetry-completion.md new file mode 100644 index 00000000..399aae43 --- /dev/null +++ b/docs/superpowers/plans/2026-08-15-reflective-symmetry-completion.md @@ -0,0 +1,80 @@ +# Reflective Symmetry Completion Implementation Plan + +**Goal:** Add `reflective_symmetry_complete` as a generic V2.2 primitive that converts three exact training gaps to acceptance and one underdetermined training gap to explicit ambiguity without changing any prior result. + +## Global gates + +- Training-only development on 1,000 ARC tasks / 1,076 hidden rows. +- No public ARC evaluation rerun. +- Frozen V2.1 byte identity against `1d7c489bcdbff755d638b35ad47ce62bd4fe829d`. +- No task-ID routing or transform preference. +- Final exact metrics: 99 correct, 99 accepted, 0 incorrect attempts, 0 failures. +- Final statuses: 99 `ACCEPTED`, 2 `AMBIGUOUS_PROGRAM`, 975 `GRAMMAR_EXHAUSTED`. +- Cumulative correct gain set: prior 37 gains plus `496994bd`, `e729b7be`, `f25ffba3`. +- `b8825c91` must transition to ambiguity, not acceptance. + +## Task 1 — Exact fixtures and RED contract + +Pin from immutable benchmark artifact run `30929169392`: + +```text +496994bd ecc40bb943911e33cdfc1e668a5bb49b22ad55a662aa37d68ba57e884897dc25 +b8825c91 ccf892a7ba8797ab4c69b241d80dd389cde0b9c7b8c0667a8b3a273873c4d49d +e729b7be d2565e3c8e6482a1a3cec2ccf604032c223ee6d6f579dfee98a72b4cbdc12c99 +f25ffba3 98bd5da81220ac72cf0a2d8ae093ed602066e14cb28ffce6db3110d91382067c +``` + +Create fixture integrity and behavioral tests before production code. + +Direct tests cover vertical, horizontal, 180-degree, diagonal completion, original-anchor-only behavior, cross-color conflict, non-square diagonal rejection, exact derivation, shape mismatch skip, no-op skip, three exact real tasks, and `b8825c91` hidden disagreement ambiguity. + +Hosted RED accepts only missing `symmetry_complete` module after fixture and frozen-byte gates pass. + +## Task 2 — Isolated module GREEN + +Create `symmetry_complete.py` with: + +```python +apply_symmetry_complete(program, grid) +symmetry_complete_programs(demos) +``` + +Use original source anchors only. Precompute reflection claims. Reject different-color collisions/overwrites. Diagonal transforms require square grids. Candidate order is background ascending then transform order `vertical`, `horizontal`, `rotate_180`, `main_diagonal`, `anti_diagonal`. Require exact demo fit and at least one changed demo. + +Run fixture plus direct tests excluding real integration assertions. Require frozen byte gate and seal module GREEN evidence. + +## Task 3 — Guarded integration + +Fail-closed workflow: + +- `grid_induction.py`: import executor, `grid-v2.2-15` -> `grid-v2.2-16`, dispatch `reflective_symmetry_complete`. +- `grid_mechanistic_runtime.py`: import generator, derive after line connection, append `symmetry_complete` after `line_connect`. + +Run symmetry, line, gravity, block, axis, palette, quadrant, mirror, bbox, outlier, marker, mechanistic, all V2.2, frozen integration, and byte gates. Restore shared files on any failure. + +## Task 4 — Strict corpus gate + +Run all 1,000 tasks with 1,000 validated traces and compact/mechanistic parity. + +Require: + +```text +rows 1076 +correct 99 +accepted 99 +incorrect_attempts 0 +failed 0 +ACCEPTED 99 +AMBIGUOUS_PROGRAM 2 +GRAMMAR_EXHAUSTED 975 +``` + +Require exact cumulative correct gain set of 40 tasks. New accepted mechanisms use prefix `symmetry_complete:` for `496994bd`, `e729b7be`, `f25ffba3`. Require `b8825c91` terminal ambiguity with at least two symmetry-completion equivalence classes. Preserve all prior mechanism guards and `60b61512` bbox ambiguity. + +Publish complete checksummed artifact only on full GREEN. + +## Task 5 — Independent validation and sealing + +Verify artifact digest, all manifest entries, 1,000 traces, parity, exact metrics/statuses/gains, three accepted symmetry traces, both ambiguity cases, all prior guards, and zero wrong accepts. + +Seal as `docs/evidence/arc_v2_2/TRAINING_SYMMETRY_COMPLETE_0017.json`. Keep PR #40 draft and unmerged. diff --git a/docs/superpowers/plans/2026-08-15-same-color-line-connect.md b/docs/superpowers/plans/2026-08-15-same-color-line-connect.md new file mode 100644 index 00000000..10116dd7 --- /dev/null +++ b/docs/superpowers/plans/2026-08-15-same-color-line-connect.md @@ -0,0 +1,73 @@ +# Same-Color Straight-Line Connection Implementation Plan + +**Goal:** Add `same_color_line_connect` as a generic V2.2 primitive that converts exactly four training gaps to exact acceptance while preserving every prior result and safety abstention. + +## Global gates + +- Training-only development on 1,000 ARC tasks / 1,076 hidden rows. +- No public ARC evaluation rerun. +- Frozen V2.1 byte identity at `1d7c489bcdbff755d638b35ad47ce62bd4fe829d`. +- No task-ID routing or preferred fitting candidate. +- Final exact metrics: 96 correct, 96 accepted, 0 incorrect attempts, 0 failures. +- Final statuses: 96 `ACCEPTED`, 1 `AMBIGUOUS_PROGRAM`, 979 `GRAMMAR_EXHAUSTED`. +- Cumulative gain set: prior 33 gains plus `1f876c06`, `22168020`, `22eb0ac0`, `ded97339`. + +## Task 1 — Exact fixtures and RED contract + +Pin exact vendor payloads under `tests/fixtures/line_connect/` and enforce known SHA-256 digests. + +Create `test_line_connect_fixtures.py` and `test_line_connect.py` before production code. + +Direct tests cover: + +- horizontal connection; +- vertical connection; +- diagonal connection; +- original-anchor-only behavior; +- cross-color conflict rejection; +- unique demonstration derivation; +- shape mismatch skip; +- no-op skip; +- four real tasks as exact trace-explained acceptances after integration. + +Hosted RED must fail only because `research.universal_core.holonomy_v2_2.line_connect` is missing, after fixture digests pass. + +## Task 2 — Isolated module GREEN + +Create `line_connect.py` with: + +```python +apply_line_connect(program, grid) +line_connect_programs(demos) +``` + +Use original anchors only. Precompute segment claims. Reject different-color claim collisions and overwrites of existing different non-background cells. Canonical modes: horizontal, vertical, orthogonal, diagonal, all. + +Candidate order: background ascending, then canonical mode order. Require exact demo fit and at least one changed demo. + +Run fixture plus direct tests excluding real integration assertions. Require frozen byte gate and seal module GREEN evidence. + +## Task 3 — Guarded integration + +Fail-closed workflow modifications: + +- `grid_induction.py`: import executor, `grid-v2.2-14` -> `grid-v2.2-15`, dispatch `same_color_line_connect`. +- `grid_mechanistic_runtime.py`: import generator, derive after gravity, append `line_connect` after `gravity_compact`. + +Run line, gravity, block, axis, palette, quadrant, mirror, bbox, outlier, marker, mechanistic, full V2.2, frozen tests, and byte gate. Restore shared files on any failure. + +## Task 4 — Strict corpus gate + +Run all 1,000 training tasks with 1,000 validated traces and compact/mechanistic parity. + +Require exact 37-task gain set, 96/1/979 status boundary, zero wrong accepts, no lost prior row, prior mechanism guard preservation, and `60b61512` ambiguity preservation. + +For each new task require successful `same_color_line_connect` hidden execution and selected candidate prefix `line_connect:`. + +Publish complete checksummed artifact only after every gate passes. + +## Task 5 — Independent validation and sealing + +Verify artifact ZIP digest, 1,006 manifest entries, zero checksum failures, 1,000 traces, 1,000 parity checks, exact metrics/statuses/gains, line mechanism traces, prior selector guards, and preserved ambiguity. + +Seal as `docs/evidence/arc_v2_2/TRAINING_LINE_CONNECT_0016.json`. Keep PR #40 draft and unmerged. diff --git a/docs/superpowers/specs/2026-08-04-arc-mechanistic-observability-design.md b/docs/superpowers/specs/2026-08-04-arc-mechanistic-observability-design.md new file mode 100644 index 00000000..bbad1b8d --- /dev/null +++ b/docs/superpowers/specs/2026-08-04-arc-mechanistic-observability-design.md @@ -0,0 +1,252 @@ +# ARC V2.2 Mechanistic Observability Design + +## Status + +Approved for implementation on `universal-core-arc-training-v2-2` under the standing instruction to use the recommended engineering choice without pausing for routine decisions. + +## Purpose + +Add deterministic, event-level observability to the complete ARC V2.2 grid-induction path so every hypothesis, demonstration check, rejection, hidden execution, equivalence class, abstention, and accepted prediction can be reconstructed exactly. + +The trace system is diagnostic only. It must not change candidate generation, candidate ordering, predictions, status values, program digests, freeze digests, or prediction digests. + +## Honesty boundary + +“Complete mechanistic visibility” means complete visibility into every explicit decision and transformation performed by `research/universal_core/holonomy_v2_2` for grid tasks. + +It does not claim visibility into: + +- Python interpreter internals; +- operating-system or hardware execution; +- the immutable V2.1 implementation when a non-grid task is delegated to it. + +Non-grid delegation is recorded as an explicit opaque boundary. The frozen V2.1 tree at `1d7c489bcdbff755d638b35ad47ce62bd4fe829d` remains byte-identical. + +## Selected approach + +Use dual execution modes: + +1. **Compact mode** preserves the existing public result shape and low artifact cost. +2. **Mechanistic mode** runs the same inference path while collecting a complete deterministic event ledger. + +All ARC remediation and training-corpus development runs use mechanistic mode. Compact and mechanistic outputs must pass a parity gate after removing trace-only fields. + +This approach is preferred over always-on tracing because it preserves a small production-facing result while making full evidence mandatory during development. + +## Architecture + +### 1. Deterministic trace recorder + +Add `research/universal_core/holonomy_v2_2/mechanistic_trace.py` with a recorder that: + +- emits monotonically numbered events; +- canonicalizes every event using sorted-key JSON; +- rejects non-JSON-safe values; +- seals the ledger with SHA-256; +- records the trace schema as `mechanistic-grid-trace-v1`; +- excludes `task_id` from the trace-content digest so task-name invariance remains testable; +- includes a separate task-bound envelope digest for artifact integrity. + +The recorder is passive. It cannot decide whether a candidate fits or alter any program. + +### 2. Single inference path + +Extend the entry point to: + +```python +run_public_task_v2_2(task, engine, *, mechanistic=False) +``` + +Both modes execute one shared inference path. Mechanistic mode provides a recorder to that path; compact mode provides no recorder. No duplicate inference implementation is allowed. + +### 3. Candidate provenance + +Every candidate receives: + +- a canonical program representation; +- a program digest; +- a generator name; +- a generator-local ordinal; +- the exact structural parameters used to construct it. + +Candidate generators must expose all structurally valid candidates rather than silently filtering them internally. Demonstration fitting is centralized so every proposal receives one visible fit decision. + +Existing generator families covered by this requirement are: + +- fixed geometric transforms; +- color mapping; +- scale and tiling; +- crop; +- component-rank recoloring; +- panel Boolean combination; +- enclosed-region operations; +- component-area selection; +- every later primitive. + +For derived generators that cannot construct a candidate, the trace records a generator skip with an exact reason. + +### 4. Demonstration execution ledger + +For every proposed candidate and every demonstration, record exactly one of: + +- successful execution with the complete output grid and output digest; +- typed execution failure with exception class and message. + +For successful outputs, record the comparison result against the target. A mismatch records: + +- source and target dimensions; +- produced dimensions; +- all differing coordinates when dimensions match; +- complete produced and target grids. + +Each candidate then receives exactly one terminal fit decision: + +- `FITS_ALL_DEMOS`; +- `DEMO_MISMATCH`; +- `DEMO_EXECUTION_FAILED`. + +Duplicate canonical programs are recorded before one representative is retained. + +### 5. Hidden execution ledger + +Every fitting, deduplicated candidate receives exactly one execution result for every hidden input: + +- complete output and digest; or +- typed execution failure. + +Successful complete output batches are grouped by canonical batch digest. The trace records every equivalence-class member and its program digest. + +### 6. Final decision ledger + +The final event records one exact outcome: + +- `GRAMMAR_EXHAUSTED` when no candidate fits all demonstrations; +- `EXECUTION_FAILED` when fitting candidates exist but none complete all hidden inputs; +- `AMBIGUOUS_PROGRAM` when completed candidates form multiple output classes; +- `ACCEPTED` when all completed candidates form one output class. + +For an accepted result, the trace records: + +- the complete agreeing program-digest set; +- the deterministic minimum-digest selected program; +- output-batch digest; +- public program, freeze, and prediction digests. + +For an abstention, it records the exact candidate and failure counts that caused it. + +## Completeness invariants + +Mechanistic mode fails closed if any invariant is violated: + +1. Every proposed candidate has one terminal demonstration-fit decision. +2. Every fitting deduplicated candidate has one terminal hidden-batch result. +3. Every successful hidden batch belongs to exactly one output equivalence class. +4. Every final-decision reference points to an event or digest already present in the ledger. +5. Event ordinals are contiguous and deterministic. +6. Replaying the same task produces a byte-identical trace. +7. Changing only `task_id` preserves the trace-content digest. +8. Compact and mechanistic modes produce identical non-trace results. + +## Training-corpus integration + +Extend `run_training_corpus` with optional mechanistic output parameters. In mechanistic mode it writes one canonical trace file per task and stores only trace references in the sealed aggregate report: + +- trace schema; +- trace-content digest; +- task-bound envelope digest; +- relative artifact path; +- event count; +- terminal decision. + +The full grids remain in per-task traces rather than inflating the aggregate report. + +Add a deterministic corpus diagnosis that aggregates only trace facts, including: + +- rejection counts by generator and cause; +- hidden execution failures by exception type and candidate kind; +- ambiguity counts and competing candidate kinds; +- grammar-exhaustion counts by generator coverage; +- accepted counts by selected primitive. + +Targets used for scoring remain outside the model trace and are never fed back into induction. A separate post-score comparison may classify correct and incorrect predictions but must not modify the mechanistic ledger. + +## Workflow and evidence + +Update the training workflow so every development corpus run publishes: + +- compact aggregate report; +- per-task mechanistic traces; +- corpus diagnosis; +- parity report; +- checksum manifest. + +Required gates are: + +- focused trace tests; +- all V2.2 tests; +- frozen V2 integration tests; +- frozen V2 byte-diff gate; +- deterministic replay gate; +- compact/mechanistic parity gate; +- full 1,000-task ARC training run; +- zero newly incorrect accepted rows; +- no public ARC evaluation rerun. + +## Root-cause evidence already identified + +The current two `EXECUTION_FAILED` training tasks are understood mechanistically and will become the first trace-driven capability upgrades after observability is green. + +### `aabf363d` + +The only fitting candidate is a literal color map. The demonstrations use an isolated marker cell to specify a new object color, but the hidden object uses an unseen source color. Hidden execution raises `KeyError("unmapped grid cell")`. + +The generic missing mechanism is marker-directed recoloring with marker removal, not another literal mapping entry. + +### `b230c067` + +Two component-rank programs fit the demonstrations. The hidden input introduces a component-area rank not seen in the demonstrations. Both candidates raise `KeyError("unmapped component rank")`. + +The generic missing mechanism is binary extreme-versus-other component recoloring, not exact rank enumeration. + +These facts are provisional local analysis until reproduced by the new event ledger. Implementation begins only after the trace confirms them. + +## Subsequent trace-driven primitives + +After the observability layer proves parity and completeness, capability work proceeds one primitive at a time under strict TDD: + +1. marker-directed object recoloring; +2. binary component-extreme recoloring; +3. bounding-box completion using preserve-existing interior or outline modes. + +Each primitive must show: + +- a focused RED fixture; +- trace-confirmed root cause; +- smallest generic implementation; +- focused GREEN; +- complete V2.2 and frozen-core gates; +- full training-corpus comparison; +- no lost exact tasks and no incorrect accepted rows. + +## Non-goals + +This unit does not: + +- modify frozen V2.1; +- add task-ID routing; +- use ARC public-evaluation targets for development; +- claim neural activation interpretability for a symbolic program enumerator; +- change scoring policy; +- improve coverage before trace parity is proven. + +## Acceptance criteria + +The design is complete when: + +- all V2.2 grid candidates and decisions are reconstructible from a sealed trace; +- a completeness checker proves there are no unrecorded candidate or execution paths; +- trace replay is deterministic; +- compact and mechanistic outputs are identical apart from trace references; +- the full ARC training artifact contains one valid trace per task; +- the frozen V2.1 byte gate passes; +- the two current execution failures are reproduced with their exact exception mechanisms in the new traces. diff --git a/docs/superpowers/specs/2026-08-04-dihedral-quadrant-mosaic-design.md b/docs/superpowers/specs/2026-08-04-dihedral-quadrant-mosaic-design.md new file mode 100644 index 00000000..2e554094 --- /dev/null +++ b/docs/superpowers/specs/2026-08-04-dihedral-quadrant-mosaic-design.md @@ -0,0 +1,114 @@ +# Dihedral Quadrant Mosaic Design + +## Objective + +Extend ARC grid induction with a generic primitive for outputs formed as a 2-by-2 mosaic of dihedral transforms of the source grid. Development remains restricted to the 1,000 ARC training tasks. The frozen V2.1 core remains byte-identical, and the public ARC evaluation set is not rerun. + +## Evidence and expected gain + +An exhaustive training-only scan found nine currently grammar-exhausted tasks whose demonstrations and hidden targets are exactly represented by this family: + +- `0c786b71` +- `3af2c5a8` +- `46442a0e` +- `62c24649` +- `67e8384a` +- `7953d61e` +- `7fe24cdd` +- `833dafe3` +- `ed98d772` + +For all nine tasks, the fitting programs produce one hidden-output equivalence class and the class matches the training target. The scan found no wrong completed hidden output and no ambiguity for this family. + +Expected cumulative corpus result after integration: + +- 78 correct and accepted rows out of 1,076 +- zero incorrect accepted rows +- zero runner failures +- one preserved `AMBIGUOUS_PROGRAM` task (`60b61512`) +- 997 `GRAMMAR_EXHAUSTED` rows + +## Program model + +The new program kind is `dihedral_quadrant_mosaic` with four ordered transform fields: + +```json +{ + "kind": "dihedral_quadrant_mosaic", + "top_left": "identity", + "top_right": "reflect_columns", + "bottom_left": "reflect_rows", + "bottom_right": "rotate_180" +} +``` + +Each field is one of the eight square-grid dihedral actions: + +- `identity` +- `rotate_90` +- `rotate_180` +- `rotate_270` +- `reflect_columns` +- `reflect_rows` +- `reflect_main_diagonal` +- `reflect_anti_diagonal` + +For rectangular grids, only transforms that preserve the source shape may occupy a quadrant. Quarter-turn and diagonal transforms are therefore excluded unless source height equals source width. + +## Candidate derivation + +The generator activates only when every demonstration output has dimensions exactly twice the source height and twice the source width. + +For each of the four output quadrants independently: + +1. split every demonstration output into source-sized quadrants; +2. evaluate every shape-preserving dihedral transform of the corresponding source; +3. retain transform names that reproduce that quadrant in every demonstration. + +If any quadrant has no supported transform, the generator emits no programs and records `NO_SUPPORTED_QUADRANT_TRANSFORM`. + +Otherwise, it emits the deterministic Cartesian product of the four retained transform sets. Transform order is fixed by the list above, making program order and trace output deterministic. + +No transform is preferred merely because it is simpler or appears first. Symmetric demonstrations may support multiple named transforms. Those hypotheses remain explicit so an asymmetric hidden input can distinguish them. Existing output-equivalence consensus accepts only when all surviving hidden predictions agree and abstains when they diverge. + +## Execution + +Execution applies the four named transforms to the source grid, verifies that every transformed grid has the source shape, and concatenates them into two quadrant rows. It rejects outputs exceeding ARC's 30-by-30 bounds. + +The implementation lives in a focused module: + +- `research/universal_core/holonomy_v2_2/quadrant_mosaic.py` + +The shared interpreter adds one dispatch case, and the mechanistic runtime adds one generator batch. The grid grammar version advances from `grid-v2.2-9` to `grid-v2.2-10`. + +## Observability + +The existing mechanistic runtime records this family without special-case trace logic: + +- generator start/end or skip reason; +- every proposed transform tuple; +- every demonstration execution and fit decision; +- hidden execution outcomes; +- output-equivalence classes; +- final acceptance or ambiguity. + +Compact and mechanistic modes must remain exactly identical after removal of the trace envelope. + +## Testing + +Strict TDD sequence: + +1. add direct unit tests for representative quadrant arrangements and bounds; +2. add the nine immutable training fixtures and exact acceptance tests; +3. add a synthetic symmetry case where demonstrations support multiple transforms but the hidden input causes divergent outputs, requiring `AMBIGUOUS_PROGRAM`; +4. verify RED because `quadrant_mosaic` is absent; +5. implement the isolated module; +6. wire the interpreter and mechanistic generator; +7. run focused tests, the complete V2.2 cluster, frozen V2 integration tests, and the frozen-byte gate; +8. run all 1,000 ARC training tasks in compact and mechanistic modes; +9. require exactly the nine expected gains, no lost correct rows, no wrong accepts, one preserved ambiguity, 1,000 validated traces, and 1,000 parity matches; +10. publish and independently checksum the corpus artifact. + +## Scope exclusions + +This unit does not add arbitrary quadrant coloring, separators, overlays, repeated tiling, learned transform preference, task-ID routing, or public-evaluation tuning. Those are separate grammar families and require independent evidence and TDD cycles. diff --git a/docs/superpowers/specs/2026-08-04-palette-cardinality-repeat-design.md b/docs/superpowers/specs/2026-08-04-palette-cardinality-repeat-design.md new file mode 100644 index 00000000..3c7c2129 --- /dev/null +++ b/docs/superpowers/specs/2026-08-04-palette-cardinality-repeat-design.md @@ -0,0 +1,106 @@ +# Palette-Cardinality Repeat Design + +## Objective + +Add a generic ARC primitive whose scale or tile factor is computed from the input grid's palette cardinality. The primitive targets four training-only grammar gaps while preserving all existing fixed scale and tile behavior. + +Development uses only the 1,000 ARC training tasks. The public ARC evaluation set is not rerun, and the frozen V2.1 core remains byte-identical. + +## Training-only evidence + +An exhaustive scan of all 1,000 training tasks, including already accepted tasks, found exactly four compatible tasks: + +- `a59b95c0`: whole-grid tiling by the number of distinct input colors; +- `ac0a08a4`: per-cell scaling by the number of distinct non-background colors; +- `b91ae062`: per-cell scaling by the number of distinct non-background colors; +- `d4b1c2b1`: per-cell scaling by the number of distinct input colors. + +Each task has one fitting program, one hidden-output equivalence class, and an exact hidden target. The scan found no compatible already accepted task, no wrong completed hidden output, and no disagreement. + +Expected cumulative training result after integration: + +- 82 correct and accepted rows out of 1,076; +- zero incorrect accepted rows; +- zero runner failures; +- one preserved `AMBIGUOUS_PROGRAM` task (`60b61512`); +- 993 `GRAMMAR_EXHAUSTED` rows. + +## Program model + +The new kind is `palette_cardinality_repeat`: + +```json +{ + "kind": "palette_cardinality_repeat", + "operation": "scale", + "statistic": "distinct_non_background", + "background": 0 +} +``` + +`operation` is one of: + +- `scale`: replace each cell by a factor-by-factor constant block; +- `tile`: repeat the complete grid factor times vertically and horizontally. + +`statistic` is one of: + +- `distinct_all`: factor equals the number of distinct colors in the input; +- `distinct_non_background`: factor equals the number of distinct colors other than the demonstrated background. + +The `background` field is present only for `distinct_non_background`. + +## Candidate derivation + +For each operation and statistic: + +1. compute the statistic-derived factor independently for every demonstration input; +2. require factor >= 1 and a result no larger than ARC's 30-by-30 bounds; +3. execute the scale or tile operation; +4. retain the candidate only if every demonstration output is reproduced exactly; +5. require at least two distinct demonstrated factors. + +The factor-variation requirement is the grammar-ownership boundary. A constant demonstrated factor provides no evidence that palette cardinality, rather than a fixed scale or tile program, controls the output. Such cases emit no dynamic candidate and remain owned by the existing grammar. + +For `distinct_non_background`, background candidates are colors present in every demonstration input. Exact demonstration execution determines whether a candidate survives; no corner or modal heuristic is required. + +## Execution and safety + +At hidden execution time, the factor is recomputed from the hidden input. The operation rejects factor < 1 and outputs beyond 30-by-30. Multiple fitting candidates remain explicit and are handled by the existing hidden-output consensus. No statistic or operation is preferred without agreement. + +The isolated implementation lives at: + +- `research/universal_core/holonomy_v2_2/palette_repeat.py` + +The shared grid grammar advances from `grid-v2.2-10` to `grid-v2.2-11`. + +## Observability + +The existing mechanistic runtime records this family through its standard lifecycle: + +- generator start/end or skip reason; +- proposed operation/statistic/background programs; +- every demonstration execution and fit decision; +- hidden factor execution, failure, and output class; +- final selected candidate or abstention. + +Compact and mechanistic results must remain exactly identical after removal of the trace envelope. + +## TDD and acceptance + +Strict sequence: + +1. add immutable pinned fixtures for the four task IDs with SHA-256 integrity tests; +2. add direct scale/tile execution and factor-variation tests; +3. verify missing-module RED; +4. implement the isolated module; +5. verify module GREEN; +6. wire interpreter dispatch and mechanistic generator under a fail-closed workflow; +7. run focused tests, all V2.2 tests, frozen V2 integration tests, and frozen-byte identity; +8. run all 1,000 training tasks in compact and mechanistic modes; +9. require exactly the four new gains, no lost rows, zero incorrect attempts, 1,000 validated traces, 1,000 parity matches, and preserved `60b61512` ambiguity; +10. checksum, publish, independently validate, and seal the artifact evidence. + +## Scope exclusions + +This unit does not infer factors from cell counts, component counts, dimensions, task IDs, or target metadata. It does not add arbitrary block substitution or self-stamping. Those are separate families requiring independent evidence and TDD. diff --git a/docs/superpowers/specs/2026-08-05-axis-duplicate-collapse-design.md b/docs/superpowers/specs/2026-08-05-axis-duplicate-collapse-design.md new file mode 100644 index 00000000..d97fa742 --- /dev/null +++ b/docs/superpowers/specs/2026-08-05-axis-duplicate-collapse-design.md @@ -0,0 +1,123 @@ +# Axis Duplicate Collapse Design + +## Objective + +Add a generic ARC primitive that compresses repeated rows and columns while preserving their first representative. The primitive targets six training-only grammar gaps and leaves all existing accepted mechanisms unchanged. + +Development uses only the 1,000 ARC training tasks. The public ARC evaluation set is not rerun, and the frozen V2.1 core remains byte-identical. + +## Training-only evidence + +Exhaustive enumeration across all 1,000 training tasks found exactly six compatible grammar-exhausted tasks: + +- `2dee498d` +- `746b3537` +- `7b7f7511` +- `ce8d95cc` +- `e1baa8a4` +- `eb5a1d5d` + +Every fitting candidate produces the exact hidden target. All fitting scope/axis candidates agree on the six hidden outputs. The scan found: + +- zero wrong completed hidden outputs; +- zero hidden-output disagreements; +- zero compatible already accepted tasks. + +Expected cumulative training result after integration: + +- 88 correct and accepted rows out of 1,076; +- zero incorrect accepted rows; +- zero runner failures; +- one preserved `AMBIGUOUS_PROGRAM` task (`60b61512`); +- 987 `GRAMMAR_EXHAUSTED` rows. + +## Program model + +The new kind is `axis_duplicate_collapse`: + +```json +{ + "kind": "axis_duplicate_collapse", + "scope": "adjacent_runs", + "axes": "both" +} +``` + +`scope` is one of: + +- `adjacent_runs`: replace each maximal consecutive run of identical rows or columns with its first member; +- `global_first`: retain the first occurrence of every distinct row or column and remove later duplicates, even when separated. + +`axes` is one of: + +- `rows` +- `columns` +- `both` + +Rows and columns are compared as exact value tuples. Retained order is stable. + +Row and column collapse commute: removing duplicate column vectors keeps one representative of every column-equivalence class and therefore preserves row equality; the symmetric argument applies to column equality after row collapse. `both` consequently has one canonical execution order. + +## Candidate derivation + +The generator considers all six scope/axis combinations. A candidate is exposed only when: + +1. executing it changes at least one demonstration input; +2. its output dimensions do not exceed the corresponding input dimensions; +3. the candidate remains within ARC bounds. + +The mechanistic runtime performs normal demonstration fitting and records every mismatch or acceptance. No scope or axis set is preferred. + +This boundary avoids identity overlap. Training enumeration found no exact overlap with an existing accepted task. + +## Hidden execution and ambiguity + +The selected collapse operation is recomputed directly on each hidden input. When multiple candidates fit demonstrations: + +- identical hidden outputs are deduplicated into one output class; +- divergent hidden outputs trigger the existing `AMBIGUOUS_PROGRAM` abstention. + +## Implementation + +The isolated module is: + +- `research/universal_core/holonomy_v2_2/axis_collapse.py` + +It exports: + +```python +apply_axis_collapse(program, grid) -> Grid +axis_collapse_programs(demos) -> tuple[tuple[Program, ...], str | None] +``` + +The shared grid grammar advances from `grid-v2.2-11` to `grid-v2.2-12`. + +## Observability + +The existing mechanistic runtime records: + +- generator start/end or skip reason; +- every scope/axis proposal; +- every demonstration execution and fit decision; +- every hidden execution; +- candidate deduplication and output-equivalence classes; +- final acceptance or ambiguity. + +Compact and mechanistic results must remain exactly identical after removing the trace envelope. + +## TDD and acceptance + +1. Restore six fixtures byte-for-byte from immutable corpus artifact run `30929169392`. +2. Pin fixture SHA-256 values in a regression test. +3. Add direct tests for adjacent-run and global-first collapse, stable ordering, row-only/column-only/both modes, commutativity, and no-effect skip behavior. +4. Verify missing-module RED. +5. Implement the isolated module and verify module GREEN. +6. Wire interpreter dispatch and mechanistic generator through a fail-closed workflow. +7. Run focused tests, all V2.2 tests, frozen V2 integration tests, and frozen-byte identity. +8. Run all 1,000 training tasks compact and mechanistic. +9. Require exactly the six expected gains, no lost rows, zero incorrect attempts, 1,000 validated traces, 1,000 parity matches, and preserved `60b61512` ambiguity. +10. Checksum, publish, independently validate, and seal the artifact. + +## Scope exclusions + +This unit does not remove blank rows or columns merely because they contain a background color; it removes rows or columns only because they duplicate another row or column under the selected scope. It does not sort rows/columns, infer semantic separators, or use task identifiers. diff --git a/docs/superpowers/specs/2026-08-05-fixed-block-reduction-design.md b/docs/superpowers/specs/2026-08-05-fixed-block-reduction-design.md new file mode 100644 index 00000000..c212cf47 --- /dev/null +++ b/docs/superpowers/specs/2026-08-05-fixed-block-reduction-design.md @@ -0,0 +1,113 @@ +# Fixed-Block Reduction Design + +## Objective + +Add a generic ARC primitive that reduces a grid partitioned into fixed-size blocks to one output cell per block. The primitive targets two training-only grammar gaps while preserving all existing accepted mechanisms. + +Development uses only the 1,000 ARC training tasks. The public ARC evaluation set is not rerun, and the frozen V2.1 core remains byte-identical. + +## Training-only evidence + +Exhaustive enumeration across all 1,000 training tasks found exactly two compatible grammar-exhausted tasks: + +- `5614dbcf`: reduce 3-by-3 blocks by strict mode; an equivalent candidate ignores demonstrated noise color 5 and requires one remaining color per block; +- `68b67ca3`: reduce 2-by-2 blocks by the unique non-background color, using background 0. + +For both tasks: + +- every fitting candidate predicts the exact hidden target; +- fitting candidates form one hidden-output equivalence class; +- no wrong hidden output is completed; +- no already accepted task is compatible. + +Expected cumulative result after integration: + +- 90 correct and accepted rows out of 1,076; +- zero incorrect accepted rows; +- zero runner failures; +- one preserved `AMBIGUOUS_PROGRAM` task (`60b61512`); +- 985 `GRAMMAR_EXHAUSTED` rows. + +## Program model + +The new kind is `fixed_block_reduce`: + +```json +{ + "kind": "fixed_block_reduce", + "row_factor": 3, + "col_factor": 3, + "reducer": "strict_mode" +} +``` + +or: + +```json +{ + "kind": "fixed_block_reduce", + "row_factor": 2, + "col_factor": 2, + "reducer": "unique_non_background", + "background": 0 +} +``` + +Reducers: + +- `strict_mode`: choose the unique most frequent color in the block; ties fail execution; +- `unique_non_background`: remove the configured background color, require at most one distinct remaining color, and return that color when present or the background when the block is otherwise empty. + +## Candidate derivation + +For every demonstration, output height and width must divide input height and width. The resulting row and column factors must be identical across all demonstrations and at least one factor must exceed one. + +The generator then considers: + +1. one `strict_mode` candidate; +2. one `unique_non_background` candidate for every color appearing in every demonstration input. + +The generator exposes the structural candidates. The existing mechanistic fitting stage records all block executions and retains only candidates reproducing every demonstration target exactly. + +When source/output dimensions do not define one stable block shape, the generator emits no candidates with `NO_STABLE_BLOCK_FACTORS`. + +## Execution and safety + +Execution partitions the source into non-overlapping blocks of `row_factor` by `col_factor`. Factors must divide the hidden input dimensions exactly. The output must remain within ARC bounds and cannot be empty. + +Multiple fitting reducers remain explicit. Hidden-output consensus accepts only agreement and abstains on divergence. No reducer or background is preferred. + +## Implementation + +The isolated module is: + +- `research/universal_core/holonomy_v2_2/block_reduce.py` + +It exports: + +```python +apply_block_reduce(program, grid) -> Grid +block_reduce_programs(demos) -> tuple[tuple[Program, ...], str | None] +``` + +The shared grammar advances from `grid-v2.2-12` to `grid-v2.2-13`. + +## Observability + +The existing mechanistic runtime records every reducer/background proposal, demonstration execution, fit decision, hidden execution, output class, and final decision. Compact and mechanistic results must remain exactly identical after removing the trace envelope. + +## TDD and acceptance + +1. Restore both fixtures byte-for-byte from immutable corpus artifact run `30929169392` and pin SHA-256 values. +2. Add direct tests for strict-mode success/tie failure, unique-non-background success/conflict failure, stable factor derivation, and exact acceptance of both real tasks. +3. Verify missing-module RED. +4. Implement and verify the isolated module. +5. Wire interpreter dispatch and mechanistic generator through a fail-closed workflow. +6. Run focused tests, all V2.2 tests, frozen V2 integration tests, and frozen-byte identity. +7. Run all 1,000 training tasks compact and mechanistic. +8. Require exactly the two new gains, no lost rows, zero incorrect attempts, 1,000 validated traces, 1,000 parity matches, and preserved `60b61512` ambiguity. +9. Checksum, publish, independently validate, and seal the artifact. + +## Scope exclusions + +This unit does not infer overlapping windows, variable block sizes within one task, weighted voting, arbitrary representative pixels, or task-ID rules. Those require separate evidence and TDD cycles. diff --git a/docs/superpowers/specs/2026-08-15-anchor-ray-extension-design.md b/docs/superpowers/specs/2026-08-15-anchor-ray-extension-design.md new file mode 100644 index 00000000..9fa6f465 --- /dev/null +++ b/docs/superpowers/specs/2026-08-15-anchor-ray-extension-design.md @@ -0,0 +1,131 @@ +# Anchor-to-Border Ray Extension Design + +## Goal + +Add one generic ARC primitive that extends every original non-background anchor toward selected grid borders along canonical straight directions, preserving original cells and rejecting color conflicts. + +An exhaustive training-only scan over all 1,000 ARC training tasks found exactly two compatible tasks: + +- `623ea044`: background `0`, diagonal rays. +- `d037b0a7`: background `0`, downward rays. + +Each task has exactly one fitting program/output class, the hidden prediction is exact, there are zero wrong completed hidden outputs, and no currently accepted task fits this family. + +## Baseline and Acceptance Boundary + +The immutable pre-ray checkpoint is reflective symmetry V2.2 at candidate `18573507deb284e42d573085909bbf76be5abcb8`: + +- 1,000 training tasks / 1,076 hidden rows. +- 99 accepted and correct rows. +- 0 incorrect attempts. +- 0 failures. +- 2 `AMBIGUOUS_PROGRAM` rows. +- 975 `GRAMMAR_EXHAUSTED` rows. +- 1,000 compact/mechanistic parity checks. +- frozen V2.1 byte identity against `1d7c489bcdbff755d638b35ad47ce62bd4fe829d`. + +Acceptance requires exactly: + +- 101 accepted and correct rows. +- 0 incorrect attempts. +- 0 failures. +- 2 `AMBIGUOUS_PROGRAM` rows. +- 973 `GRAMMAR_EXHAUSTED` rows. +- 42 cumulative correct gain tasks over original `e509650...`. +- no lost prior-correct row, no changed prior mechanism guard, and both existing ambiguity cases preserved. + +## Program Schema + +```python +{ + "kind": "anchor_ray_extend", + "background": 0, + "mode": "up" | "down" | "left" | "right" | "vertical" | "horizontal" | "orthogonal" | "diagonal" | "all", +} +``` + +Canonical mode order is exactly the order shown above. + +## Execution Semantics + +Anchors are all original source cells whose color differs from `background`. Newly written cells never become anchors. + +Mode directions: + +```text +up -> N +down -> S +left -> W +right -> E +vertical -> N + S +horizontal -> W + E +orthogonal -> N + S + W + E +diagonal -> NW + NE + SW + SE +all -> all eight directions +``` + +For each original anchor and each selected direction, claim every in-bounds cell from the adjacent position through the border with the anchor color. + +A claim fails if it would overwrite an original different non-background color or if two different anchor colors claim the same cell. Same-color overlaps are allowed. Claims are computed from original anchors only and applied after validation. + +Execution preserves grid shape and every original cell value. + +## Candidate Derivation + +`ray_extend_programs(demos)`: + +1. Require at least one demonstration. +2. Require source and target shape equality in every demo; otherwise return `NO_SHAPE_PRESERVING_RAY_EXTENSION`. +3. Intersect source colors across demos to obtain background candidates. +4. Enumerate backgrounds ascending and modes in canonical order. +5. Execute each candidate on every demonstration. +6. Retain only exact demo fits. +7. Require at least one demo source to change; no-op candidates are omitted. +8. Return all fitting candidates without preference. +9. If none fit, return `NO_RAY_EXTENSION`. + +Existing hidden-output consensus decides acceptance or ambiguity. + +## Training-Only Discovery Evidence + +Full 1,000-task family scan: + +```text +623ea044 -> background=0, mode=diagonal -> exact hidden output +d037b0a7 -> background=0, mode=down -> exact hidden output +``` + +No other task fits. No current acceptance overlaps. + +Pinned fixture SHA-256 digests: + +```text +623ea044.json 578ab7e47a2d67489f1103715615f719ef81e9faf773be85586803d6da44cbdd +d037b0a7.json 9e39a4acdb3e3bff7dd54bed5f4ab77973f4027d48beb62a9c740d8a7a0b0aee +``` + +## Integration + +Create `research/universal_core/holonomy_v2_2/ray_extend.py` exposing: + +```python +apply_ray_extend(program, grid) +ray_extend_programs(demos) +``` + +Then: + +- `grid_induction.py`: dispatch `anchor_ray_extend` and advance `grid-v2.2-16` to `grid-v2.2-17`. +- `grid_mechanistic_runtime.py`: register generator prefix `ray_extend:` immediately after `symmetry_complete`. + +## TDD and Evidence Requirements + +1. Pin both exact fixtures from immutable benchmark run `30929169392` and verify digests before commit. +2. Commit cardinal/diagonal/multi-direction execution, original-anchor-only, collision, derivation, shape/no-op, and two real-task tests before production code. +3. Confirm missing-module RED. +4. Implement isolated module and pass module-only GREEN. +5. Wire via fail-closed integration workflow. +6. Run all 1,000 training tasks with complete traces and parity. +7. Require exactly two new accepted gains, exact 101/2/973 statuses, zero wrong accepts, and all prior mechanism/ambiguity guards. +8. Independently verify the artifact and seal immutable evidence. +9. Do not rerun public ARC evaluation. diff --git a/docs/superpowers/specs/2026-08-15-axis-gravity-compaction-design.md b/docs/superpowers/specs/2026-08-15-axis-gravity-compaction-design.md new file mode 100644 index 00000000..4259ddee --- /dev/null +++ b/docs/superpowers/specs/2026-08-15-axis-gravity-compaction-design.md @@ -0,0 +1,145 @@ +# Axis Gravity Compaction Design + +## Goal + +Add one generic ARC grid primitive that compacts all non-background cells within independent rows or columns toward a selected edge while preserving lane order, color identity, grid shape, and per-lane non-background counts. + +The training-only discovery scan found exactly two compatible grammar gaps across all 1,000 ARC training tasks: + +- `1e0a9b12`: compact each column downward through background `0`. +- `3906de3d`: compact each column upward through background `0`. + +No other training task fits any candidate in this family. Both compatible tasks have exactly one fitting candidate, exact hidden outputs, zero wrong completed hidden outputs, and zero hidden-output ambiguity. + +## Baseline and Acceptance Boundary + +The immutable pre-gravity checkpoint is fixed-block V2.2 at candidate commit `c3c4efaef66e2ff6b8f83c5381acdec13cb31f5e`: + +- 1,000 ARC training tasks / 1,076 hidden rows. +- 90 accepted and correct rows. +- 0 incorrect attempts. +- 0 failures. +- 1 `AMBIGUOUS_PROGRAM` row. +- 985 `GRAMMAR_EXHAUSTED` rows. +- 1,000 compact/mechanistic parity checks. +- frozen V2.1 byte identity against `1d7c489bcdbff755d638b35ad47ce62bd4fe829d`. + +The gravity primitive is accepted only if the strict corpus becomes exactly: + +- 92 accepted and correct rows. +- 0 incorrect attempts. +- 0 failures. +- 1 `AMBIGUOUS_PROGRAM` row. +- 983 `GRAMMAR_EXHAUSTED` rows. +- 33 cumulative gain tasks over the original `e509650...` training baseline. +- no lost prior-correct row and no changed prior mechanism guard. + +## Program Schema + +```python +{ + "kind": "axis_gravity_compact", + "background": 0, + "direction": "up" | "down" | "left" | "right", +} +``` + +No task IDs, coordinates, task-specific colors, or corpus-specific routing are part of the program. + +## Execution Semantics + +### Vertical directions + +For every column independently: + +1. Read cells from top to bottom. +2. Remove cells equal to `background`. +3. Preserve the exact order of the remaining cells. +4. For `up`, write the sequence from the top and fill the remainder with `background`. +5. For `down`, fill the prefix with `background` and write the sequence ending at the bottom. + +### Horizontal directions + +For every row independently, apply the same process from left to right: + +- `left` writes the preserved sequence at the left edge. +- `right` writes the preserved sequence at the right edge. + +### Invariants + +Execution must preserve: + +- input height and width; +- every lane's multiset of cell colors; +- every lane's relative order of non-background cells; +- the total count of each color; +- rectangularity and ARC cell values. + +An unknown direction is an error. + +## Candidate Derivation + +`gravity_compact_programs(demos)` enumerates only demonstration-supported candidates: + +1. Require at least one demonstration. +2. Require every target to have the same shape as its source; otherwise return `NO_SHAPE_PRESERVING_GRAVITY`. +3. Compute colors common to every demonstration source as background candidates. +4. For each common background in sorted order and each direction in canonical order `up`, `down`, `left`, `right`, execute the candidate on every demonstration. +5. Retain a candidate only when it reproduces every demonstration target exactly. +6. Require the candidate to change at least one demonstration source; identity-equivalent candidates are not emitted. +7. If none remain, return `NO_GRAVITY_COMPACTION`. +8. Return all retained candidates without preference. Existing hidden-output equivalence/consensus logic decides acceptance or ambiguity. + +This design deliberately does not infer one direction from heuristic geometry or choose among multiple fitting candidates. + +## Training-Only Discovery Evidence + +An exhaustive prototype enumerated the full candidate family over all 1,000 training tasks: + +```text +1e0a9b12 -> background=0, direction=down -> exact hidden output +3906de3d -> background=0, direction=up -> exact hidden output +``` + +Family-wide scan result: + +- compatible tasks: 2; +- fitting programs: 2 total; +- compatible tasks with multiple hidden output classes: 0; +- wrong completed hidden outputs: 0; +- compatible already-accepted tasks: 0. + +The two pinned fixture SHA-256 digests from the immutable benchmark ZIP are: + +```text +1e0a9b12.json 6a71dbedc8403bd28fa76bd8b472597a8ecaab3af31151a2a3b72baeadd5e617 +3906de3d.json b4c20d433380e00840a75c6976ad0d0fabd80e052ab77b47422d64b1cae5f6dc +``` + +## Integration + +Create `research/universal_core/holonomy_v2_2/gravity_compact.py` exposing: + +```python +apply_gravity_compact(program, grid) +gravity_compact_programs(demos) +``` + +Then integrate through: + +- `grid_induction.py`: import executor, dispatch `axis_gravity_compact`, advance `grid-v2.2-13` to `grid-v2.2-14`. +- `grid_mechanistic_runtime.py`: register generator prefix `gravity_compact:` after `block_reduce`. + +The generator remains structurally independent from fixed-block reduction, axis duplicate collapse, and all earlier primitives. + +## TDD and Evidence Requirements + +1. Pin exact fixtures and byte digests. +2. Commit direct execution, derivation, exact real-task, and no-op-boundary tests before production code. +3. Confirm RED from missing `gravity_compact` module. +4. Implement the isolated module and pass module-only GREEN. +5. Wire shared runtime only through a fail-closed workflow that restores both shared files on any regression. +6. Run the complete 1,000-task training corpus with 1,000 traces and parity verification. +7. Require exactly two new gain tasks and the 92/1/983 status boundary. +8. Verify the published artifact independently and seal immutable evidence under `docs/evidence/arc_v2_2/`. +9. Do not rerun the public ARC evaluation split. diff --git a/docs/superpowers/specs/2026-08-15-reflective-symmetry-completion-design.md b/docs/superpowers/specs/2026-08-15-reflective-symmetry-completion-design.md new file mode 100644 index 00000000..3f06d1a5 --- /dev/null +++ b/docs/superpowers/specs/2026-08-15-reflective-symmetry-completion-design.md @@ -0,0 +1,131 @@ +# Reflective Symmetry Completion Design + +## Goal + +Add one generic ARC primitive that completes an input by reflecting its original non-background cells through one demonstrated symmetry while preserving the source cells in place. + +The exhaustive training-only scan over all 1,000 ARC training tasks found exactly four compatible residual tasks under individual shape-preserving transforms: + +- `496994bd`: horizontal reflection and 180-degree rotation both fit the demonstrations and agree on the hidden output; exact acceptance is safe by hidden-output consensus. +- `e729b7be`: 180-degree rotational completion; unique fitting program and exact hidden output. +- `f25ffba3`: horizontal reflective completion; unique fitting program and exact hidden output. +- `b8825c91`: vertical reflection and 180-degree rotation both fit demonstrations but produce two hidden output classes. The correct action is `AMBIGUOUS_PROGRAM`. + +No currently accepted training task fits this family. There are zero wrong accepted hidden outputs when existing consensus is applied. + +## Baseline and Acceptance Boundary + +The immutable pre-symmetry checkpoint is same-color line connection V2.2 at candidate `df779fa892b9ce3bfddab4d03aa743a97d22d1b3`: + +- 1,000 training tasks / 1,076 hidden rows. +- 96 accepted and correct rows. +- 0 incorrect attempts. +- 0 failures. +- 1 `AMBIGUOUS_PROGRAM` row. +- 979 `GRAMMAR_EXHAUSTED` rows. +- 1,000 compact/mechanistic parity checks. +- frozen V2.1 byte identity against `1d7c489bcdbff755d638b35ad47ce62bd4fe829d`. + +Acceptance requires exactly: + +- 99 accepted and correct rows. +- 0 incorrect attempts. +- 0 failures. +- 2 `AMBIGUOUS_PROGRAM` rows. +- 975 `GRAMMAR_EXHAUSTED` rows. +- 40 cumulative gain tasks over the original `e509650...` training baseline. +- no lost prior-correct row and no changed prior mechanism guard. + +`b8825c91` must specifically transition from `GRAMMAR_EXHAUSTED` to `AMBIGUOUS_PROGRAM` with at least two symmetry-completion output classes. + +## Program Schema + +```python +{ + "kind": "reflective_symmetry_complete", + "background": 0, + "transform": "vertical" | "horizontal" | "rotate_180" | "main_diagonal" | "anti_diagonal", +} +``` + +Canonical transform order is exactly the order shown above. + +## Execution Semantics + +The input grid is copied unchanged first. Reflection claims are derived only from original non-background cells; newly written cells never become new anchors. + +For every original non-background cell `(row, col, color)`: + +- `vertical`: map to `(row, width - 1 - col)`. +- `horizontal`: map to `(height - 1 - row, col)`. +- `rotate_180`: map to `(height - 1 - row, width - 1 - col)`. +- `main_diagonal`: square grids only; map to `(col, row)`. +- `anti_diagonal`: square grids only; map to `(width - 1 - col, height - 1 - row)`. + +The reflected cell is written only when the original target cell is background or already the same color. A reflected claim that would overwrite a different original non-background color fails. Different-color reflected claims on the same cell also fail. Same-color overlap is allowed. + +Execution preserves input shape and all original cell values. + +## Candidate Derivation + +`symmetry_complete_programs(demos)`: + +1. Require at least one demonstration. +2. Require source and target shape equality in every demonstration; otherwise return `NO_SHAPE_PRESERVING_SYMMETRY`. +3. Intersect source colors across demonstrations to obtain background candidates. +4. Enumerate backgrounds ascending and transforms in canonical order. +5. Execute each candidate on every demonstration. +6. Retain only candidates reproducing every target exactly. +7. Require at least one demonstration to change; no-op candidates are omitted. +8. Return all fitting candidates without preference. +9. If none fit, return `NO_REFLECTIVE_SYMMETRY_COMPLETION`. + +Existing hidden-output equivalence and consensus logic decides acceptance or ambiguity. No transform is preferred merely because it appears earlier. + +## Training-Only Discovery Evidence + +Canonical family scan results: + +```text +496994bd -> background 0 -> horizontal, rotate_180 -> one exact hidden output class +b8825c91 -> background 4 -> vertical, rotate_180 -> two hidden output classes -> abstain +e729b7be -> background 7 -> rotate_180 -> exact +f25ffba3 -> background 0 -> horizontal -> exact +``` + +No accepted task overlaps this family. + +Pinned fixture SHA-256 digests: + +```text +496994bd.json ecc40bb943911e33cdfc1e668a5bb49b22ad55a662aa37d68ba57e884897dc25 +b8825c91.json ccf892a7ba8797ab4c69b241d80dd389cde0b9c7b8c0667a8b3a273873c4d49d +e729b7be.json d2565e3c8e6482a1a3cec2ccf604032c223ee6d6f579dfee98a72b4cbdc12c99 +f25ffba3.json 98bd5da81220ac72cf0a2d8ae093ed602066e14cb28ffce6db3110d91382067c +``` + +## Integration + +Create `research/universal_core/holonomy_v2_2/symmetry_complete.py` exposing: + +```python +apply_symmetry_complete(program, grid) +symmetry_complete_programs(demos) +``` + +Then: + +- `grid_induction.py`: dispatch `reflective_symmetry_complete` and advance `grid-v2.2-15` to `grid-v2.2-16`. +- `grid_mechanistic_runtime.py`: register generator prefix `symmetry_complete:` immediately after `line_connect`. + +## TDD and Evidence Requirements + +1. Pin all four exact fixtures from immutable benchmark run `30929169392`; verify digests before commit. +2. Commit execution, diagonal-shape, conflict, derivation, no-op, three exact-real-task, and one ambiguity-real-task tests before production code. +3. Confirm missing-module RED. +4. Implement isolated module and pass module-only GREEN. +5. Wire only through a fail-closed integration workflow. +6. Run all 1,000 training tasks with complete traces and parity. +7. Require exactly three new correct gains, `b8825c91` ambiguity, exact 99/2/975 statuses, zero wrong accepts, and all prior mechanism guards. +8. Independently verify the artifact and seal immutable evidence. +9. Do not rerun public ARC evaluation. diff --git a/docs/superpowers/specs/2026-08-15-same-color-line-connect-design.md b/docs/superpowers/specs/2026-08-15-same-color-line-connect-design.md new file mode 100644 index 00000000..46f04da6 --- /dev/null +++ b/docs/superpowers/specs/2026-08-15-same-color-line-connect-design.md @@ -0,0 +1,140 @@ +# Same-Color Straight-Line Connection Design + +## Goal + +Add one generic ARC primitive that connects same-color source anchors along selected straight-line families without moving or recoloring existing cells. + +The exhaustive training-only scan over all 1,000 ARC training tasks found exactly four compatible grammar gaps: + +- `1f876c06`: diagonal connections. +- `22168020`: horizontal connections. +- `22eb0ac0`: horizontal connections. +- `ded97339`: orthogonal horizontal/vertical connections. + +All fitting demonstration programs predict the exact hidden output. Demo-equivalent modes collapse to one hidden output class for every compatible task. There are zero wrong completed hidden outputs and zero overlap with already accepted tasks. + +## Baseline and Acceptance Boundary + +The immutable pre-line checkpoint is gravity V2.2 at candidate `3c6510a5c69ee600a7fc8d3a0a6a3e15ee9a05b1`: + +- 1,000 training tasks / 1,076 hidden rows. +- 92 accepted and correct rows. +- 0 incorrect attempts. +- 0 failures. +- 1 `AMBIGUOUS_PROGRAM` row. +- 983 `GRAMMAR_EXHAUSTED` rows. +- 1,000 compact/mechanistic parity checks. +- frozen V2.1 byte identity at `1d7c489bcdbff755d638b35ad47ce62bd4fe829d`. + +Acceptance requires exactly: + +- 96 accepted and correct rows. +- 0 incorrect attempts. +- 0 failures. +- 1 `AMBIGUOUS_PROGRAM` row. +- 979 `GRAMMAR_EXHAUSTED` rows. +- 37 cumulative gain tasks over original `e509650...` baseline. +- no lost prior-correct row and no changed prior mechanism guard. + +## Program Schema + +```python +{ + "kind": "same_color_line_connect", + "background": 0, + "mode": "horizontal" | "vertical" | "orthogonal" | "diagonal" | "all", +} +``` + +Canonical mode order is exactly the order shown above. + +## Execution Semantics + +Anchors are determined once from the original source grid. Newly filled cells never become anchors and therefore cannot create cascading segments. + +For every non-background color independently, selected line families are evaluated: + +- horizontal: same row; +- vertical: same column; +- main diagonal: equal `row - col`; +- anti-diagonal: equal `row + col`. + +For each color and line containing at least two original anchors, connect the outermost anchors by filling every cell on the straight segment with that color. + +Modes select line families as follows: + +```text +horizontal -> horizontal +vertical -> vertical +orthogonal -> horizontal + vertical +diagonal -> main diagonal + anti-diagonal +all -> horizontal + vertical + both diagonals +``` + +All segment claims are derived from original anchors before writing output. If two different colors claim the same cell, or a segment would overwrite an existing different non-background color, execution fails. Same-color overlapping claims are allowed. + +Execution preserves source shape and every existing cell value. + +## Candidate Derivation + +`line_connect_programs(demos)`: + +1. Require at least one demonstration. +2. Require source and target shapes to match for every demonstration; otherwise return `NO_SHAPE_PRESERVING_LINE_CONNECT`. +3. Intersect source colors across demonstrations to obtain background candidates. +4. Enumerate backgrounds ascending and modes in canonical order. +5. Execute each candidate on every demonstration. +6. Retain only candidates reproducing every target exactly. +7. Require at least one demonstration to change; no-op candidates are omitted. +8. Return all fitting candidates without preference. +9. If none fit, return `NO_LINE_CONNECTION`. + +Existing hidden-output equivalence and consensus logic decides acceptance or ambiguity. + +## Training-Only Discovery Evidence + +Canonical family scan results: + +```text +1f876c06 -> background 0 -> diagonal, all +22168020 -> background 0 -> horizontal, orthogonal, all +22eb0ac0 -> background 0 -> horizontal, orthogonal, all +ded97339 -> background 0 -> orthogonal +``` + +For every task, all fitting candidates produce one hidden output class and that class exactly equals the hidden target. + +Pinned fixture SHA-256 digests: + +```text +1f876c06.json b53a1a25685895400323e813a373f0a66fc9512f7dbbb326d52a93695fca8143 +22168020.json d067b4f83f84120300310199e3a95c65258edcc53d8b0f425288d7ddb71a962e +22eb0ac0.json cf37cc34e7b0aa123c2806eca77a7d9916b50afea4e6e2db71d5149fe1f1af0d +ded97339.json fd14b4ee37bb1539c26a79abed743626b37f5935794ce82cbd0b3332c207906e +``` + +## Integration + +Create `research/universal_core/holonomy_v2_2/line_connect.py` exposing: + +```python +apply_line_connect(program, grid) +line_connect_programs(demos) +``` + +Then: + +- `grid_induction.py`: dispatch `same_color_line_connect` and advance `grid-v2.2-14` to `grid-v2.2-15`. +- `grid_mechanistic_runtime.py`: register `line_connect:` immediately after `gravity_compact`. + +## TDD and Evidence Requirements + +1. Pin all four exact fixtures and digest tests. +2. Commit direct horizontal, vertical, diagonal, conflict, derivation, no-op, and real-task tests before production code. +3. Confirm missing-module RED. +4. Implement isolated module and pass module-only GREEN. +5. Wire through a fail-closed integration workflow. +6. Run all 1,000 training tasks with complete traces and parity. +7. Require exactly four new gain tasks, exact 96/1/979 statuses, zero wrong accepts, and all prior mechanism guards. +8. Independently verify the artifact and seal immutable evidence. +9. Do not rerun public ARC evaluation. diff --git a/research/universal_core/holonomy_v2_2/__init__.py b/research/universal_core/holonomy_v2_2/__init__.py new file mode 100644 index 00000000..af57a309 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/__init__.py @@ -0,0 +1 @@ +"""Training-only V2.2 extensions over the frozen V2.1 core.""" diff --git a/research/universal_core/holonomy_v2_2/axis_collapse.py b/research/universal_core/holonomy_v2_2/axis_collapse.py new file mode 100644 index 00000000..e55f33c4 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/axis_collapse.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from typing import Any, Mapping, Sequence + + +Grid = list[list[int]] +Program = dict[str, Any] + + +def _collapse_rows(grid: Grid, scope: str) -> Grid: + output: Grid = [] + seen: set[tuple[int, ...]] = set() + previous: tuple[int, ...] | None = None + for row in grid: + key = tuple(row) + if scope == "adjacent_runs": + if key == previous: + continue + previous = key + elif scope == "global_first": + if key in seen: + continue + seen.add(key) + else: + raise ValueError("unknown axis collapse scope") + output.append(list(row)) + if not output: + raise ValueError("axis collapse removed every row") + return output + + +def _collapse_columns(grid: Grid, scope: str) -> Grid: + columns = [tuple(row[index] for row in grid) for index in range(len(grid[0]))] + keep: list[int] = [] + seen: set[tuple[int, ...]] = set() + previous: tuple[int, ...] | None = None + for index, column in enumerate(columns): + if scope == "adjacent_runs": + if column == previous: + continue + previous = column + elif scope == "global_first": + if column in seen: + continue + seen.add(column) + else: + raise ValueError("unknown axis collapse scope") + keep.append(index) + if not keep: + raise ValueError("axis collapse removed every column") + return [[row[index] for index in keep] for row in grid] + + +def apply_axis_collapse(program: Mapping[str, Any], grid: Grid) -> Grid: + scope = str(program["scope"]) + axes = str(program["axes"]) + output = [list(row) for row in grid] + if axes == "rows": + return _collapse_rows(output, scope) + if axes == "columns": + return _collapse_columns(output, scope) + if axes == "both": + return _collapse_columns(_collapse_rows(output, scope), scope) + raise ValueError("unknown axis collapse axes") + + +def axis_collapse_programs( + demos: Sequence[tuple[Grid, Grid]], +) -> tuple[tuple[Program, ...], str | None]: + if not demos: + return (), "NO_DEMONSTRATION" + + programs: list[Program] = [] + for scope in ("adjacent_runs", "global_first"): + for axes in ("rows", "columns", "both"): + program = { + "kind": "axis_duplicate_collapse", + "scope": scope, + "axes": axes, + } + if any(apply_axis_collapse(program, source) != source for source, _ in demos): + programs.append(program) + + if not programs: + return (), "NO_DUPLICATE_AXIS_REDUCTION" + return tuple(programs), None diff --git a/research/universal_core/holonomy_v2_2/bbox_complete.py b/research/universal_core/holonomy_v2_2/bbox_complete.py new file mode 100644 index 00000000..24f2ff4a --- /dev/null +++ b/research/universal_core/holonomy_v2_2/bbox_complete.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +from copy import deepcopy +from typing import Any, Mapping, Sequence + + +Grid = list[list[int]] +Program = dict[str, Any] +Point = tuple[int, int] +Component = tuple[Point, ...] + + +def _steps(connectivity: int) -> tuple[Point, ...]: + orthogonal: tuple[Point, ...] = ((1, 0), (-1, 0), (0, 1), (0, -1)) + if connectivity == 4: + return orthogonal + if connectivity == 8: + return orthogonal + ((1, 1), (1, -1), (-1, 1), (-1, -1)) + raise ValueError("bounding-box connectivity must be four or eight") + + +def _components(grid: Grid, background: int, connectivity: int) -> tuple[Component, ...]: + height = len(grid) + width = len(grid[0]) + seen: set[Point] = set() + found: list[Component] = [] + for row in range(height): + for col in range(width): + point = (row, col) + if point in seen or grid[row][col] == background: + continue + color = grid[row][col] + pending = [point] + seen.add(point) + cells: list[Point] = [] + while pending: + current = pending.pop() + cells.append(current) + for row_step, col_step in _steps(connectivity): + next_row = current[0] + row_step + next_col = current[1] + col_step + next_point = (next_row, next_col) + if not (0 <= next_row < height and 0 <= next_col < width): + continue + if next_point in seen or grid[next_row][next_col] != color: + continue + seen.add(next_point) + pending.append(next_point) + found.append(tuple(sorted(cells))) + return tuple(found) + + +def _bounds(cells: Sequence[Point]) -> tuple[int, int, int, int]: + if not cells: + raise ValueError("bounding-box completion requires a foreground cell") + return ( + min(row for row, _ in cells), + max(row for row, _ in cells), + min(col for _, col in cells), + max(col for _, col in cells), + ) + + +def _groups(program: Mapping[str, Any], grid: Grid) -> tuple[Component, ...]: + background = int(program["background"]) + scope = str(program["scope"]) + if scope == "global": + cells = tuple( + (row, col) + for row, values in enumerate(grid) + for col, value in enumerate(values) + if value != background + ) + if not cells: + raise ValueError("global bounding box requires foreground") + return (cells,) + if scope == "components": + components = _components( + grid, + background, + int(program["connectivity"]), + ) + if not components: + raise ValueError("component bounding boxes require foreground") + return components + raise ValueError("unknown bounding-box scope") + + +def apply_bbox_complete(program: Mapping[str, Any], grid: Grid) -> Grid: + background = int(program["background"]) + fill_color = int(program["fill_color"]) + mode = str(program["mode"]) + if fill_color == background: + raise ValueError("bounding-box fill color must differ from background") + if mode not in {"fill", "outline"}: + raise ValueError("unknown bounding-box completion mode") + + output = deepcopy(grid) + for cells in _groups(program, grid): + top, bottom, left, right = _bounds(cells) + for row in range(top, bottom + 1): + for col in range(left, right + 1): + if grid[row][col] != background: + continue + on_border = row in (top, bottom) or col in (left, right) + if mode == "fill" or on_border: + output[row][col] = fill_color + return output + + +def _background_candidates( + demos: Sequence[tuple[Grid, Grid]], +) -> tuple[int, ...]: + common = set(cell for row in demos[0][0] for cell in row) + for source, _ in demos[1:]: + common.intersection_update(cell for row in source for cell in row) + return tuple(sorted(common)) + + +def bbox_complete_programs( + demos: Sequence[tuple[Grid, Grid]], +) -> tuple[tuple[Program, ...], str | None]: + if any( + len(source) != len(target) or len(source[0]) != len(target[0]) + for source, target in demos + ): + return (), "GRID_SHAPE_CHANGED" + backgrounds = _background_candidates(demos) + if not backgrounds: + return (), "NO_BACKGROUND_CANDIDATE" + target_colors = sorted({ + cell + for _, target in demos + for row in target + for cell in row + }) + programs: list[Program] = [] + for background in backgrounds: + for fill_color in target_colors: + if fill_color == background: + continue + for mode in ("fill", "outline"): + programs.append({ + "kind": "bounding_box_complete", + "background": background, + "fill_color": fill_color, + "mode": mode, + "scope": "global", + }) + for connectivity in (4, 8): + programs.append({ + "kind": "bounding_box_complete", + "background": background, + "fill_color": fill_color, + "mode": mode, + "scope": "components", + "connectivity": connectivity, + }) + if not programs: + return (), "NO_STRUCTURAL_PROGRAM" + return tuple(programs), None diff --git a/research/universal_core/holonomy_v2_2/block_reduce.py b/research/universal_core/holonomy_v2_2/block_reduce.py new file mode 100644 index 00000000..1d253fee --- /dev/null +++ b/research/universal_core/holonomy_v2_2/block_reduce.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +from collections import Counter +from typing import Any, Mapping, Sequence + + +Grid = list[list[int]] +Program = dict[str, Any] + + +def _shape(grid: Grid) -> tuple[int, int]: + if not grid or not grid[0]: + raise ValueError("block reduction requires a nonempty grid") + width = len(grid[0]) + if any(len(row) != width for row in grid): + raise ValueError("block reduction requires a rectangular grid") + return len(grid), width + + +def _block_value(block: list[int], reducer: str, background: int | None) -> int: + if reducer == "strict_mode": + counts = Counter(block) + highest = max(counts.values()) + modes = [color for color, count in counts.items() if count == highest] + if len(modes) != 1: + raise ValueError("strict mode requires a unique mode") + return modes[0] + + if reducer == "unique_non_background": + if background is None: + raise ValueError("unique non-background reduction requires a background") + colors = {color for color in block if color != background} + if not colors: + return background + if len(colors) != 1: + raise ValueError("block must contain at most one non-background color") + return next(iter(colors)) + + raise ValueError("unknown block reducer") + + +def apply_block_reduce(program: Mapping[str, Any], grid: Grid) -> Grid: + height, width = _shape(grid) + row_factor = int(program["row_factor"]) + col_factor = int(program["col_factor"]) + if row_factor < 1 or col_factor < 1: + raise ValueError("block factors must be positive") + if height % row_factor or width % col_factor: + raise ValueError("grid dimensions must be divisible by block factors") + + output_height = height // row_factor + output_width = width // col_factor + if not 1 <= output_height <= 30 or not 1 <= output_width <= 30: + raise ValueError("block reduction output exceeds ARC bounds") + + reducer = str(program["reducer"]) + background = int(program["background"]) if "background" in program else None + output: Grid = [] + for out_row in range(output_height): + row: list[int] = [] + row_start = out_row * row_factor + for out_col in range(output_width): + col_start = out_col * col_factor + block = [ + grid[source_row][source_col] + for source_row in range(row_start, row_start + row_factor) + for source_col in range(col_start, col_start + col_factor) + ] + row.append(_block_value(block, reducer, background)) + output.append(row) + return output + + +def _fits(program: Program, demos: Sequence[tuple[Grid, Grid]]) -> bool: + try: + return all(apply_block_reduce(program, source) == target for source, target in demos) + except (KeyError, TypeError, ValueError): + return False + + +def _stable_factors(demos: Sequence[tuple[Grid, Grid]]) -> tuple[int, int] | None: + factors: tuple[int, int] | None = None + for source, target in demos: + source_height, source_width = _shape(source) + target_height, target_width = _shape(target) + if source_height % target_height or source_width % target_width: + return None + current = (source_height // target_height, source_width // target_width) + if current[0] < 1 or current[1] < 1 or current == (1, 1): + return None + if factors is None: + factors = current + elif factors != current: + return None + return factors + + +def block_reduce_programs( + demos: Sequence[tuple[Grid, Grid]], +) -> tuple[tuple[Program, ...], str | None]: + if not demos: + return (), "NO_DEMONSTRATION" + + factors = _stable_factors(demos) + if factors is None: + return (), "NO_STABLE_BLOCK_FACTORS" + row_factor, col_factor = factors + + programs: list[Program] = [] + strict = { + "kind": "fixed_block_reduce", + "row_factor": row_factor, + "col_factor": col_factor, + "reducer": "strict_mode", + } + if _fits(strict, demos): + programs.append(strict) + + common_colors = {cell for row in demos[0][0] for cell in row} + for source, _ in demos[1:]: + common_colors.intersection_update(cell for row in source for cell in row) + for background in sorted(common_colors): + program = { + "kind": "fixed_block_reduce", + "row_factor": row_factor, + "col_factor": col_factor, + "reducer": "unique_non_background", + "background": background, + } + if _fits(program, demos): + programs.append(program) + + if not programs: + return (), "NO_SUPPORTED_BLOCK_REDUCER" + return tuple(programs), None diff --git a/research/universal_core/holonomy_v2_2/component_outlier.py b/research/universal_core/holonomy_v2_2/component_outlier.py new file mode 100644 index 00000000..e0ce07d7 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/component_outlier.py @@ -0,0 +1,190 @@ +from __future__ import annotations + +from copy import deepcopy +from typing import Any, Mapping, Sequence + + +Grid = list[list[int]] +Program = dict[str, Any] +Point = tuple[int, int] +Component = tuple[Point, ...] + + +def _steps(connectivity: int) -> tuple[Point, ...]: + orthogonal: tuple[Point, ...] = ((1, 0), (-1, 0), (0, 1), (0, -1)) + if connectivity == 4: + return orthogonal + if connectivity == 8: + return orthogonal + ((1, 1), (1, -1), (-1, 1), (-1, -1)) + raise ValueError("component connectivity must be four or eight") + + +def _components(grid: Grid, background: int, connectivity: int) -> tuple[Component, ...]: + height = len(grid) + width = len(grid[0]) + seen: set[Point] = set() + found: list[Component] = [] + for row in range(height): + for col in range(width): + point = (row, col) + if point in seen or grid[row][col] == background: + continue + color = grid[row][col] + pending = [point] + seen.add(point) + cells: list[Point] = [] + while pending: + current = pending.pop() + cells.append(current) + for row_step, col_step in _steps(connectivity): + next_row = current[0] + row_step + next_col = current[1] + col_step + next_point = (next_row, next_col) + if not (0 <= next_row < height and 0 <= next_col < width): + continue + if next_point in seen or grid[next_row][next_col] != color: + continue + seen.add(next_point) + pending.append(next_point) + found.append(tuple(sorted(cells))) + return tuple(found) + + +def _partition( + components: Sequence[Component], + selector: str, +) -> tuple[Component, tuple[Component, ...]]: + if len(components) < 2: + raise ValueError("component outlier requires at least two components") + areas = [len(item) for item in components] + if selector == "minimum": + extreme = min(areas) + elif selector == "maximum": + extreme = max(areas) + else: + raise ValueError("unknown component outlier selector") + selected = [item for item in components if len(item) == extreme] + if len(selected) != 1: + raise ValueError("component outlier requires one unique area extreme") + peers = tuple(item for item in components if item != selected[0]) + if len({len(item) for item in peers}) != 1: + raise ValueError("component peer areas must be uniform") + return selected[0], peers + + +def apply_component_outlier(program: Mapping[str, Any], grid: Grid) -> Grid: + background = int(program["background"]) + connectivity = int(program["connectivity"]) + selector = str(program["selector"]) + selected_color = int(program["selected_color"]) + other_color = int(program["other_color"]) + if selected_color == other_color: + raise ValueError("component outlier colors must differ") + components = _components(grid, background, connectivity) + selected, _ = _partition(components, selector) + output = deepcopy(grid) + for component in components: + color = selected_color if component == selected else other_color + for row, col in component: + output[row][col] = color + return output + + +def _background_candidates( + demos: Sequence[tuple[Grid, Grid]], +) -> tuple[int, ...]: + common = set(cell for row in demos[0][0] for cell in row) + for source, _ in demos[1:]: + common.intersection_update(cell for row in source for cell in row) + candidates: list[int] = [] + for color in sorted(common): + unchanged = True + for source, target in demos: + if len(source) != len(target) or len(source[0]) != len(target[0]): + unchanged = False + break + for row_index, row in enumerate(source): + for col_index, cell in enumerate(row): + if cell == color and target[row_index][col_index] != color: + unchanged = False + break + if not unchanged: + break + if not unchanged: + break + if unchanged: + candidates.append(color) + return tuple(candidates) + + +def _derive_program( + demos: Sequence[tuple[Grid, Grid]], + background: int, + connectivity: int, + selector: str, +) -> Program | None: + selected_color: int | None = None + other_color: int | None = None + for source, target in demos: + if len(source) != len(target) or len(source[0]) != len(target[0]): + return None + components = _components(source, background, connectivity) + try: + selected, peers = _partition(components, selector) + except ValueError: + return None + covered = {point for component in components for point in component} + for row_index, row in enumerate(source): + for col_index, cell in enumerate(row): + if (row_index, col_index) not in covered and target[row_index][col_index] != cell: + return None + selected_colors = {target[row][col] for row, col in selected} + peer_colors = { + target[row][col] + for component in peers + for row, col in component + } + if len(selected_colors) != 1 or len(peer_colors) != 1: + return None + current_selected = next(iter(selected_colors)) + current_other = next(iter(peer_colors)) + if current_selected == current_other: + return None + if selected_color is None: + selected_color = current_selected + other_color = current_other + elif selected_color != current_selected or other_color != current_other: + return None + if selected_color is None or other_color is None: + return None + return { + "kind": "component_area_outlier_color", + "background": background, + "connectivity": connectivity, + "selector": selector, + "selected_color": selected_color, + "other_color": other_color, + } + + +def component_outlier_programs( + demos: Sequence[tuple[Grid, Grid]], +) -> tuple[tuple[Program, ...], str | None]: + backgrounds = _background_candidates(demos) + if not backgrounds: + return (), "NO_BACKGROUND_CANDIDATE" + programs: list[Program] = [] + for background in backgrounds: + for connectivity in (4, 8): + for selector in ("minimum", "maximum"): + program = _derive_program( + demos, + background, + connectivity, + selector, + ) + if program is not None: + programs.append(program) + if not programs: + return (), "NO_DERIVED_COMPONENT_OUTLIER_PROGRAM" + return tuple(programs), None diff --git a/research/universal_core/holonomy_v2_2/component_select.py b/research/universal_core/holonomy_v2_2/component_select.py new file mode 100644 index 00000000..67ac22c6 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/component_select.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +from collections import Counter +from typing import Any, Mapping, Sequence + + +Grid = list[list[int]] +Program = dict[str, Any] +Point = tuple[int, int] +Component = tuple[Point, ...] + + +def _steps(connectivity: int) -> tuple[Point, ...]: + orthogonal: tuple[Point, ...] = ((1, 0), (-1, 0), (0, 1), (0, -1)) + if connectivity == 4: + return orthogonal + if connectivity == 8: + return orthogonal + ((1, 1), (1, -1), (-1, 1), (-1, -1)) + raise ValueError("component connectivity must be four or eight") + + +def _components(grid: Grid, background: int, connectivity: int) -> tuple[Component, ...]: + height = len(grid) + width = len(grid[0]) + seen: set[Point] = set() + found: list[Component] = [] + for row in range(height): + for col in range(width): + point = (row, col) + if point in seen or grid[row][col] == background: + continue + color = grid[row][col] + pending = [point] + seen.add(point) + cells: list[Point] = [] + while pending: + current = pending.pop() + cells.append(current) + for row_step, col_step in _steps(connectivity): + next_row = current[0] + row_step + next_col = current[1] + col_step + next_point = (next_row, next_col) + if not (0 <= next_row < height and 0 <= next_col < width): + continue + if next_point in seen or grid[next_row][next_col] != color: + continue + seen.add(next_point) + pending.append(next_point) + found.append(tuple(sorted(cells))) + return tuple(found) + + +def _unique_extreme(components: Sequence[Component], selector: str) -> Component: + if not components: + raise ValueError("component selection requires a foreground component") + areas = [len(item) for item in components] + if selector == "minimum": + extreme = min(areas) + elif selector == "maximum": + extreme = max(areas) + else: + raise ValueError("unknown component area selector") + selected = [item for item in components if len(item) == extreme] + if len(selected) != 1: + raise ValueError("component area selector requires a unique extreme") + return selected[0] + + +def _crop_component(grid: Grid, cells: Component, background: int) -> Grid: + top = min(row for row, _ in cells) + bottom = max(row for row, _ in cells) + left = min(col for _, col in cells) + right = max(col for _, col in cells) + selected = set(cells) + return [ + [ + grid[row][col] if (row, col) in selected else background + for col in range(left, right + 1) + ] + for row in range(top, bottom + 1) + ] + + +def apply_component_select(program: Mapping[str, Any], grid: Grid) -> Grid: + background = int(program["background"]) + connectivity = int(program["connectivity"]) + selector = str(program["selector"]) + components = _components(grid, background, connectivity) + selected = _unique_extreme(components, selector) + return _crop_component(grid, selected, background) + + +def _fits(program: Program, demos: Sequence[tuple[Grid, Grid]]) -> bool: + try: + return all(apply_component_select(program, source) == target for source, target in demos) + except (KeyError, TypeError, ValueError): + return False + + +def _mode(values: Sequence[int]) -> int: + counts = Counter(values) + highest = max(counts.values()) + return min(color for color, count in counts.items() if count == highest) + + +def _modal_color(grid: Grid) -> int: + return _mode([cell for row in grid for cell in row]) + + +def _corner_mode(grid: Grid) -> int: + return _mode([grid[0][0], grid[0][-1], grid[-1][0], grid[-1][-1]]) + + +def _background_candidates(demos: Sequence[tuple[Grid, Grid]]) -> tuple[int, ...]: + common = set(cell for row in demos[0][0] for cell in row) + for source, _ in demos[1:]: + common.intersection_update(cell for row in source for cell in row) + if 0 in common: + return (0,) + candidates: set[int] = set() + modal_colors = {_modal_color(source) for source, _ in demos} + if len(modal_colors) == 1: + candidates.update(modal_colors & common) + corner_colors = {_corner_mode(source) for source, _ in demos} + if len(corner_colors) == 1: + candidates.update(corner_colors & common) + return tuple(sorted(candidates)) + + +def component_select_programs( + demos: Sequence[tuple[Grid, Grid]], +) -> tuple[tuple[Program, ...], str | None]: + backgrounds = _background_candidates(demos) + if not backgrounds: + return (), "NO_BACKGROUND_CANDIDATE" + programs: list[Program] = [] + for background in backgrounds: + for connectivity in (4, 8): + for selector in ("minimum", "maximum"): + programs.append({ + "kind": "component_area_select", + "background": background, + "connectivity": connectivity, + "selector": selector, + }) + if not programs: + return (), "NO_STRUCTURAL_PROGRAM" + return tuple(programs), None + + +def component_select_candidates(demos: Sequence[tuple[Grid, Grid]]) -> tuple[Program, ...]: + programs, _ = component_select_programs(demos) + return tuple(program for program in programs if _fits(program, demos)) diff --git a/research/universal_core/holonomy_v2_2/evidence/AXIS_COLLAPSE_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/AXIS_COLLAPSE_GREEN.json new file mode 100644 index 00000000..d3a86838 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/AXIS_COLLAPSE_GREEN.json @@ -0,0 +1,31 @@ +{ + "expected_new_exact_tasks": [ + "2dee498d", + "746b3537", + "7b7f7511", + "ce8d95cc", + "e1baa8a4", + "eb5a1d5d" + ], + "expected_training_delta": { + "accepted_rows": 6, + "correct_rows": 6, + "grammar_exhausted_rows": -6 + }, + "focused_output": "................................................. [100%]\n49 passed in 1.25s\n", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "frozen_output": "..... [100%]\n5 passed in 0.06s\n", + "outcomes": { + "focused": "success", + "frozen_byte_gate": "success", + "frozen_tests": "success", + "v2_2_cluster": "success" + }, + "ownership_boundary": "NO_DUPLICATE_AXIS_REDUCTION", + "passed": true, + "public_arc_evaluation_rerun": false, + "run_id": "30976236324", + "schema_version": 1, + "v2_2_output": "........................................................................ [ 78%]\n.................... [100%]\n92 passed in 1.26s\n", + "workflow_source_commit": "0752a6d9e9831befee72d26c44e717494187554f" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/AXIS_COLLAPSE_MODULE_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/AXIS_COLLAPSE_MODULE_GREEN.json new file mode 100644 index 00000000..b1747be3 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/AXIS_COLLAPSE_MODULE_GREEN.json @@ -0,0 +1,11 @@ +{ + "frozen_byte_gate": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "passed": true, + "public_arc_evaluation_rerun": false, + "run_id": "30976116069", + "schema_version": 1, + "source_commit": "6655e3ae0a02a31eb2a086ae133298fe3f6ecb14", + "test_count": 5, + "test_output": "..... [100%]\n5 passed in 0.09s\n" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/AXIS_COLLAPSE_RED.json b/research/universal_core/holonomy_v2_2/evidence/AXIS_COLLAPSE_RED.json new file mode 100644 index 00000000..d03f5443 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/AXIS_COLLAPSE_RED.json @@ -0,0 +1,25 @@ +{ + "captured_log": "\n==================================== ERRORS ====================================\n_ ERROR collecting research/universal_core/holonomy_v2_2/tests/test_axis_collapse.py _\nImportError while importing test module '/home/runner/work/MLE-agent/MLE-agent/research/universal_core/holonomy_v2_2/tests/test_axis_collapse.py'.\nHint: make sure your test modules/packages have valid Python names.\nTraceback:\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nresearch/universal_core/holonomy_v2_2/tests/test_axis_collapse.py:8: in \n from research.universal_core.holonomy_v2_2.axis_collapse import (\nE ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.axis_collapse'\n=========================== short test summary info ============================\nERROR research/universal_core/holonomy_v2_2/tests/test_axis_collapse.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.22s\n", + "expected_failure": "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.axis_collapse'", + "expected_new_exact_tasks": [ + "2dee498d", + "746b3537", + "7b7f7511", + "ce8d95cc", + "e1baa8a4", + "eb5a1d5d" + ], + "failure_confirmed": true, + "fixture_integrity": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "public_arc_evaluation_rerun": false, + "run_id": "30976057983", + "schema_version": 1, + "source_commit": "3cd55c78260143815f09aceba1588fd3b7094e53", + "training_scan": { + "ambiguous_hidden_outputs": 0, + "compatible_accepted_tasks": 0, + "compatible_tasks": 6, + "wrong_completed_hidden_outputs": 0 + } +} diff --git a/research/universal_core/holonomy_v2_2/evidence/AXIS_FIXTURE_INTEGRITY_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/AXIS_FIXTURE_INTEGRITY_GREEN.json new file mode 100644 index 00000000..3680c24d --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/AXIS_FIXTURE_INTEGRITY_GREEN.json @@ -0,0 +1,10 @@ +{ + "fixture_count": 6, + "fixture_integrity": true, + "public_arc_evaluation_rerun": false, + "run_id": "30976017322", + "schema_version": 1, + "source_artifact_run": 30929169392, + "test_output": ". [100%]\n1 passed in 0.02s\n", + "workflow_source_commit": "d2f0ebd922a2979052c34b7c549a33f4830e3815" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/BBOX_COMPLETE_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/BBOX_COMPLETE_GREEN.json new file mode 100644 index 00000000..1f7fdc74 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/BBOX_COMPLETE_GREEN.json @@ -0,0 +1,27 @@ +{ + "expected_new_exact_tasks": [ + "3aa6fb7a", + "6d75e8bb", + "e7639916" + ], + "expected_training_delta": { + "accepted_rows": 3, + "correct_rows": 3, + "grammar_exhausted_rows": -3 + }, + "focused_output": "..................... [100%]\n21 passed in 0.80s\n", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "frozen_output": "..... [100%]\n5 passed in 0.10s\n", + "outcomes": { + "focused": "success", + "frozen_byte_gate": "success", + "frozen_tests": "success", + "v2_2_cluster": "success" + }, + "passed": true, + "required_ambiguity": "60b61512", + "run_id": "30952811014", + "schema_version": 1, + "v2_2_output": "................................................................. [100%]\n65 passed in 0.91s\n", + "workflow_source_commit": "176c069ab8bfa160144232d113c24785221618c9" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/BBOX_COMPLETE_RED.json b/research/universal_core/holonomy_v2_2/evidence/BBOX_COMPLETE_RED.json new file mode 100644 index 00000000..a80b95f6 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/BBOX_COMPLETE_RED.json @@ -0,0 +1,30 @@ +{ + "expected_failure": "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.bbox_complete'", + "expected_new_exact_tasks": [ + "3aa6fb7a", + "6d75e8bb", + "e7639916" + ], + "failure_confirmed": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "mechanistic_family": { + "component_connectivity": [ + 4, + 8 + ], + "mode": [ + "fill", + "outline" + ], + "preserve_existing_cells": true, + "scope": [ + "global", + "components" + ] + }, + "red_commit": "679d54084d72ad4bca103f25b86d6557029bbf62", + "red_run_id": "30952677333", + "required_ambiguity": "60b61512", + "schema_version": 1, + "test_command": "python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_bbox_complete.py" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/BLOCK_REDUCE_FIXTURE_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/BLOCK_REDUCE_FIXTURE_GREEN.json new file mode 100644 index 00000000..1ee69450 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/BLOCK_REDUCE_FIXTURE_GREEN.json @@ -0,0 +1,10 @@ +{ + "fixture_count": 2, + "fixture_integrity": true, + "public_arc_evaluation_rerun": false, + "run_id": "30977196836", + "schema_version": 1, + "source_artifact_run": 30929169392, + "test_output": ". [100%]\n1 passed in 0.02s\n", + "workflow_source_commit": "cbd324e10f3a361577626981ee1eacc6265c4c84" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/BLOCK_REDUCE_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/BLOCK_REDUCE_GREEN.json new file mode 100644 index 00000000..c392989f --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/BLOCK_REDUCE_GREEN.json @@ -0,0 +1,33 @@ +{ + "expected_cumulative_metrics": { + "accepted": 90, + "correct": 90, + "failed": 0, + "incorrect_attempts": 0, + "rows": 1076 + }, + "expected_new_exact_tasks": [ + "5614dbcf", + "68b67ca3" + ], + "expected_training_delta": { + "accepted_rows": 2, + "correct_rows": 2, + "grammar_exhausted_rows": -2 + }, + "focused_output": "......................................................... [100%]\n57 passed in 1.83s\n", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "frozen_output": "..... [100%]\n5 passed in 0.09s\n", + "outcomes": { + "focused": "success", + "frozen_byte_gate": "success", + "frozen_tests": "success", + "v2_2_cluster": "success" + }, + "passed": true, + "public_arc_evaluation_rerun": false, + "run_id": "31910377325", + "schema_version": 1, + "v2_2_output": "........................................................................ [ 72%]\n............................ [100%]\n100 passed in 1.83s\n", + "workflow_source_commit": "93ea4c48f17215148c2d118104a9f9f002e5e360" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/BLOCK_REDUCE_MODULE_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/BLOCK_REDUCE_MODULE_GREEN.json new file mode 100644 index 00000000..6e06578a --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/BLOCK_REDUCE_MODULE_GREEN.json @@ -0,0 +1,15 @@ +{ + "expected_new_exact_tasks": [ + "5614dbcf", + "68b67ca3" + ], + "frozen_byte_gate": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "passed": true, + "public_arc_evaluation_rerun": false, + "run_id": "31910327227", + "schema_version": 1, + "source_commit": "48e1f11985e31bfacaeca4ab175b345b230395c4", + "test_count": 7, + "test_output": "....... [100%]\n7 passed in 0.13s\n" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/BLOCK_REDUCE_RED.json b/research/universal_core/holonomy_v2_2/evidence/BLOCK_REDUCE_RED.json new file mode 100644 index 00000000..b406cb95 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/BLOCK_REDUCE_RED.json @@ -0,0 +1,27 @@ +{ + "captured_log": "\n==================================== ERRORS ====================================\n_ ERROR collecting research/universal_core/holonomy_v2_2/tests/test_block_reduce.py _\nImportError while importing test module '/home/runner/work/MLE-agent/MLE-agent/research/universal_core/holonomy_v2_2/tests/test_block_reduce.py'.\nHint: make sure your test modules/packages have valid Python names.\nTraceback:\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nresearch/universal_core/holonomy_v2_2/tests/test_block_reduce.py:10: in \n from research.universal_core.holonomy_v2_2.block_reduce import (\nE ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.block_reduce'\n=========================== short test summary info ============================\nERROR research/universal_core/holonomy_v2_2/tests/test_block_reduce.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.17s\n", + "expected_cumulative_metrics": { + "accepted": 90, + "correct": 90, + "failed": 0, + "incorrect_attempts": 0, + "rows": 1076 + }, + "expected_failure": "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.block_reduce'", + "expected_new_exact_tasks": [ + "5614dbcf", + "68b67ca3" + ], + "expected_status_counts": { + "ACCEPTED": 90, + "AMBIGUOUS_PROGRAM": 1, + "GRAMMAR_EXHAUSTED": 985 + }, + "failure_confirmed": true, + "fixture_integrity": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "public_arc_evaluation_rerun": false, + "run_id": "31910245940", + "schema_version": 1, + "source_commit": "6aae8b87a738374a10d446aa71a2f7d61ec7ed49" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/COMPONENT_AREA_TDD.json b/research/universal_core/holonomy_v2_2/evidence/COMPONENT_AREA_TDD.json new file mode 100644 index 00000000..e1035750 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/COMPONENT_AREA_TDD.json @@ -0,0 +1,10 @@ +{ + "development_split": "ARC-AGI-2 training only", + "fixture_commit": "823036a1674e71f2470263dcbab7d0665f5b9764", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "implementation_module_commit": "31c3e39a0920523353f697d31ea2a60ea11f9586", + "integration_commit": "b0f8af1909741a2df83408830ebc624205ed085b", + "red_run_id": "30941409038", + "red_summary": "three extraction contracts were absent; two controls exposed existing panel and identity paths and were isolated before implementation", + "schema_version": 1 +} diff --git a/research/universal_core/holonomy_v2_2/evidence/COMPONENT_OUTLIER_DIAGNOSTIC.json b/research/universal_core/holonomy_v2_2/evidence/COMPONENT_OUTLIER_DIAGNOSTIC.json new file mode 100644 index 00000000..59d94ee5 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/COMPONENT_OUTLIER_DIAGNOSTIC.json @@ -0,0 +1,24 @@ +{ + "expected_training_delta": { + "accepted_rows": 1, + "correct_rows": 1, + "execution_failed_rows": -1, + "task_id": "b230c067" + }, + "focused_output": "......F........ [100%]\n=================================== FAILURES ===================================\n____________ test_hidden_component_rank_key_error_is_fully_recorded ____________\n\n def test_hidden_component_rank_key_error_is_fully_recorded() -> None:\n value = task(\n [{\n \"input\": [[5, 0, 5], [0, 0, 5]],\n \"output\": [[1, 0, 2], [0, 0, 2]],\n }],\n [[[5, 0, 5, 0, 5], [0, 0, 5, 0, 5], [0, 0, 0, 0, 5]]],\n \"unseen-rank\",\n )\n \n result = grid_induction.run_public_task_v2_2(value, None, mechanistic=True)\n failures = [\n item\n for item in events(result[\"mechanistic_trace\"], \"hidden_execution\")\n if item[\"outcome\"] == \"FAILED\"\n ]\n \n assert result[\"predictions\"] == [{\"status\": \"EXECUTION_FAILED\", \"prediction\": None}]\n assert failures\n> assert all(item[\"candidate_kind\"] == \"component_rank_color\" for item in failures)\nE assert False\nE + where False = all(. at 0x7f0e78a39560>)\n\nresearch/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py:94: AssertionError\n=========================== short test summary info ============================\nFAILED research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py::test_hidden_component_rank_key_error_is_fully_recorded - assert False\n + where False = all(. at 0x7f0e78a39560>)\n1 failed, 14 passed in 0.41s\n", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "frozen_output": "..... [100%]\n5 passed in 0.08s\n", + "mechanistic_invariant": "one unique area extreme and uniform-area peers", + "outcomes": { + "focused": "failure", + "frozen_byte_gate": "success", + "frozen_tests": "success", + "v2_2_cluster": "failure" + }, + "passed": false, + "run_id": "30951741137", + "schema_version": 1, + "source_trace_digest": "e881dc495aada3c875dd8016cc6768e37a2560e1588cb0bee0b46986b412f4ac", + "v2_2_output": "......................F.................................... [100%]\n=================================== FAILURES ===================================\n____________ test_hidden_component_rank_key_error_is_fully_recorded ____________\n\n def test_hidden_component_rank_key_error_is_fully_recorded() -> None:\n value = task(\n [{\n \"input\": [[5, 0, 5], [0, 0, 5]],\n \"output\": [[1, 0, 2], [0, 0, 2]],\n }],\n [[[5, 0, 5, 0, 5], [0, 0, 5, 0, 5], [0, 0, 0, 0, 5]]],\n \"unseen-rank\",\n )\n \n result = grid_induction.run_public_task_v2_2(value, None, mechanistic=True)\n failures = [\n item\n for item in events(result[\"mechanistic_trace\"], \"hidden_execution\")\n if item[\"outcome\"] == \"FAILED\"\n ]\n \n assert result[\"predictions\"] == [{\"status\": \"EXECUTION_FAILED\", \"prediction\": None}]\n assert failures\n> assert all(item[\"candidate_kind\"] == \"component_rank_color\" for item in failures)\nE assert False\nE + where False = all(. at 0x7f6fb37f2400>)\n\nresearch/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py:94: AssertionError\n=========================== short test summary info ============================\nFAILED research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py::test_hidden_component_rank_key_error_is_fully_recorded - assert False\n + where False = all(. at 0x7f6fb37f2400>)\n1 failed, 58 passed in 0.53s\n", + "workflow_source_commit": "179fe0e9b1f317f1b13a3a3bbe78ffb9245bbbb2" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/COMPONENT_OUTLIER_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/COMPONENT_OUTLIER_GREEN.json new file mode 100644 index 00000000..9290b7a5 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/COMPONENT_OUTLIER_GREEN.json @@ -0,0 +1,24 @@ +{ + "expected_training_delta": { + "accepted_rows": 1, + "correct_rows": 1, + "execution_failed_rows": -1, + "task_id": "b230c067" + }, + "focused_output": "............... [100%]\n15 passed in 0.42s\n", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "frozen_output": "..... [100%]\n5 passed in 0.11s\n", + "mechanistic_invariant": "one unique area extreme and uniform-area peers", + "outcomes": { + "focused": "success", + "frozen_byte_gate": "success", + "frozen_tests": "success", + "v2_2_cluster": "success" + }, + "passed": true, + "run_id": "30951907670", + "schema_version": 2, + "source_trace_digest": "e881dc495aada3c875dd8016cc6768e37a2560e1588cb0bee0b46986b412f4ac", + "v2_2_output": "........................................................... [100%]\n59 passed in 0.54s\n", + "workflow_source_commit": "a96557caf150291da56d2a333356cdf74bc91f49" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/COMPONENT_OUTLIER_RED.json b/research/universal_core/holonomy_v2_2/evidence/COMPONENT_OUTLIER_RED.json new file mode 100644 index 00000000..57774375 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/COMPONENT_OUTLIER_RED.json @@ -0,0 +1,20 @@ +{ + "captured_log": "\n==================================== ERRORS ====================================\n_ ERROR collecting research/universal_core/holonomy_v2_2/tests/test_component_outlier.py _\nImportError while importing test module '/home/runner/work/MLE-agent/MLE-agent/research/universal_core/holonomy_v2_2/tests/test_component_outlier.py'.\nHint: make sure your test modules/packages have valid Python names.\nTraceback:\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nresearch/universal_core/holonomy_v2_2/tests/test_component_outlier.py:8: in \n from research.universal_core.holonomy_v2_2.component_outlier import (\nE ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.component_outlier'\n=========================== short test summary info ============================\nERROR research/universal_core/holonomy_v2_2/tests/test_component_outlier.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.19s\n", + "expected_failure": "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.component_outlier'", + "failure_confirmed": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "mechanistic_hypothesis": { + "hidden_disambiguation": "four-connectivity violates peer-area uniformity; eight-connectivity completes", + "invariant": "one unique area extreme and uniform-area peers", + "primitive": "component_area_outlier_color" + }, + "red_commit": "57f53f663af783aab45fc2183ca83b0df319a7b6", + "red_run_id": "30951581526", + "schema_version": 1, + "source_trace": { + "failure": "two component_rank_color KeyErrors: 'unmapped component rank'", + "task_id": "b230c067", + "trace_digest": "e881dc495aada3c875dd8016cc6768e37a2560e1588cb0bee0b46986b412f4ac" + }, + "test_command": "python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_component_outlier.py" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/ENCLOSED_REGION_TDD.json b/research/universal_core/holonomy_v2_2/evidence/ENCLOSED_REGION_TDD.json new file mode 100644 index 00000000..d82940e6 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/ENCLOSED_REGION_TDD.json @@ -0,0 +1,10 @@ +{ + "development_split": "ARC-AGI-2 training only", + "fixture_commit": "b6c46649f6530a31041a72290ecdc3a651975c6c", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "implementation_module_commit": "825d11358afc678a56d12a3017a839d1638b3dce", + "integration_commit": "89d11fcd561d8f9a70b386f5922d74e98c39870e", + "red_run_id": "30940671616", + "red_summary": "three enclosed-region contracts failed before integration; the open-region ambiguity control passed", + "schema_version": 1 +} diff --git a/research/universal_core/holonomy_v2_2/evidence/GRAVITY_COMPACT_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/GRAVITY_COMPACT_GREEN.json new file mode 100644 index 00000000..6c649f4c --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/GRAVITY_COMPACT_GREEN.json @@ -0,0 +1,33 @@ +{ + "expected_cumulative_metrics": { + "accepted": 92, + "correct": 92, + "failed": 0, + "incorrect_attempts": 0, + "rows": 1076 + }, + "expected_new_exact_tasks": [ + "1e0a9b12", + "3906de3d" + ], + "expected_training_delta": { + "accepted_rows": 2, + "correct_rows": 2, + "grammar_exhausted_rows": -2 + }, + "focused_output": ".................................................................. [100%]\n66 passed in 1.95s\n", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "frozen_output": "..... [100%]\n5 passed in 0.08s\n", + "outcomes": { + "focused": "success", + "frozen_byte_gate": "success", + "frozen_tests": "success", + "v2_2_cluster": "success" + }, + "passed": true, + "public_arc_evaluation_rerun": false, + "run_id": "31911025910", + "schema_version": 1, + "v2_2_output": "........................................................................ [ 66%]\n..................................... [100%]\n109 passed in 1.95s\n", + "workflow_source_commit": "48c6d069d27073a2122bcd1b8f163de745c78b1f" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/GRAVITY_COMPACT_MODULE_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/GRAVITY_COMPACT_MODULE_GREEN.json new file mode 100644 index 00000000..abb95559 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/GRAVITY_COMPACT_MODULE_GREEN.json @@ -0,0 +1,15 @@ +{ + "expected_new_exact_tasks": [ + "1e0a9b12", + "3906de3d" + ], + "frozen_byte_gate": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "passed": true, + "public_arc_evaluation_rerun": false, + "run_id": "31910967175", + "schema_version": 1, + "source_commit": "5b7e92a0715f1b0bb555d49897f128e6c58eaca2", + "test_count": 8, + "test_output": "........ [100%]\n8 passed in 0.14s\n" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/GRAVITY_COMPACT_RED.json b/research/universal_core/holonomy_v2_2/evidence/GRAVITY_COMPACT_RED.json new file mode 100644 index 00000000..82f09dd6 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/GRAVITY_COMPACT_RED.json @@ -0,0 +1,34 @@ +{ + "captured_log": "\n==================================== ERRORS ====================================\n_ ERROR collecting research/universal_core/holonomy_v2_2/tests/test_gravity_compact.py _\nImportError while importing test module '/home/runner/work/MLE-agent/MLE-agent/research/universal_core/holonomy_v2_2/tests/test_gravity_compact.py'.\nHint: make sure your test modules/packages have valid Python names.\nTraceback:\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nresearch/universal_core/holonomy_v2_2/tests/test_gravity_compact.py:11: in \n from research.universal_core.holonomy_v2_2.gravity_compact import (\nE ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.gravity_compact'\n=========================== short test summary info ============================\nERROR research/universal_core/holonomy_v2_2/tests/test_gravity_compact.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.16s\n", + "expected_cumulative_metrics": { + "accepted": 92, + "correct": 92, + "failed": 0, + "incorrect_attempts": 0, + "rows": 1076 + }, + "expected_failure": "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.gravity_compact'", + "expected_new_exact_tasks": [ + "1e0a9b12", + "3906de3d" + ], + "expected_status_counts": { + "ACCEPTED": 92, + "AMBIGUOUS_PROGRAM": 1, + "GRAMMAR_EXHAUSTED": 983 + }, + "failure_confirmed": true, + "fixture_integrity": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "public_arc_evaluation_rerun": false, + "run_id": "31910927240", + "schema_version": 1, + "source_commit": "6a8de96f0dca952859568c17ba05588c487c9588", + "training_scan": { + "ambiguous_hidden_outputs": 0, + "compatible_accepted_tasks": 0, + "compatible_tasks": 2, + "fitting_programs": 2, + "wrong_completed_hidden_outputs": 0 + } +} diff --git a/research/universal_core/holonomy_v2_2/evidence/GRID_MECHANISTIC_DIAGNOSTIC.json b/research/universal_core/holonomy_v2_2/evidence/GRID_MECHANISTIC_DIAGNOSTIC.json new file mode 100644 index 00000000..9d4026ee --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/GRID_MECHANISTIC_DIAGNOSTIC.json @@ -0,0 +1,16 @@ +{ + "focused_output": "", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "frozen_output": "..... [100%]\n5 passed in 0.13s\n", + "outcomes": { + "focused": "skipped", + "frozen_byte_gate": "success", + "frozen_tests": "success", + "v2_2_cluster": "failure" + }, + "passed": false, + "run_id": "30948808307", + "schema_version": 1, + "v2_2_output": "\n==================================== ERRORS ====================================\n_ ERROR collecting research/universal_core/holonomy_v2_2/tests/test_component_rank.py _\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\nresearch/universal_core/holonomy_v2_2/tests/test_component_rank.py:3: in \n from research.universal_core.holonomy_v2_2.grid_induction import run_public_task_v2_2\nE File \"/home/runner/work/MLE-agent/MLE-agent/research/universal_core/holonomy_v2_2/grid_induction.py\", line 366\nE return result\nE ^\nE IndentationError: unindent does not match any outer indentation level\n_ ERROR collecting research/universal_core/holonomy_v2_2/tests/test_component_select.py _\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\nresearch/universal_core/holonomy_v2_2/tests/test_component_select.py:3: in \n from research.universal_core.holonomy_v2_2.grid_induction import run_public_task_v2_2\nE File \"/home/runner/work/MLE-agent/MLE-agent/research/universal_core/holonomy_v2_2/grid_induction.py\", line 366\nE return result\nE ^\nE IndentationError: unindent does not match any outer indentation level\n_ ERROR collecting research/universal_core/holonomy_v2_2/tests/test_grid_induction.py _\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\nresearch/universal_core/holonomy_v2_2/tests/test_grid_induction.py:5: in \n from research.universal_core.holonomy_v2_2 import grid_induction\nE File \"/home/runner/work/MLE-agent/MLE-agent/research/universal_core/holonomy_v2_2/grid_induction.py\", line 366\nE return result\nE ^\nE IndentationError: unindent does not match any outer indentation level\n_ ERROR collecting research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py _\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\nresearch/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py:6: in \n from research.universal_core.holonomy_v2_2 import grid_induction\nE File \"/home/runner/work/MLE-agent/MLE-agent/research/universal_core/holonomy_v2_2/grid_induction.py\", line 366\nE return result\nE ^\nE IndentationError: unindent does not match any outer indentation level\n_ ERROR collecting research/universal_core/holonomy_v2_2/tests/test_panel_combine.py _\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\nresearch/universal_core/holonomy_v2_2/tests/test_panel_combine.py:3: in \n from research.universal_core.holonomy_v2_2.grid_induction import run_public_task_v2_2\nE File \"/home/runner/work/MLE-agent/MLE-agent/research/universal_core/holonomy_v2_2/grid_induction.py\", line 366\nE return result\nE ^\nE IndentationError: unindent does not match any outer indentation level\n_ ERROR collecting research/universal_core/holonomy_v2_2/tests/test_region_fill.py _\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\nresearch/universal_core/holonomy_v2_2/tests/test_region_fill.py:3: in \n from research.universal_core.holonomy_v2_2.grid_induction import run_public_task_v2_2\nE File \"/home/runner/work/MLE-agent/MLE-agent/research/universal_core/holonomy_v2_2/grid_induction.py\", line 366\nE return result\nE ^\nE IndentationError: unindent does not match any outer indentation level\n_ ERROR collecting research/universal_core/holonomy_v2_2/tests/test_training_eval.py _\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/site-packages/_pytest/python.py:508: in importtestmodule\n mod = import_path(\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/site-packages/_pytest/pathlib.py:596: in import_path\n importlib.import_module(module_name)\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n:1387: in _gcd_import\n ???\n:1360: in _find_and_load\n ???\n:1331: in _find_and_load_unlocked\n ???\n:935: in _load_unlocked\n ???\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/site-packages/_pytest/assertion/rewrite.py:188: in exec_module\n exec(co, module.__dict__)\nresearch/universal_core/holonomy_v2_2/tests/test_training_eval.py:7: in \n from research.universal_core.holonomy_v2_2 import training_eval\nresearch/universal_core/holonomy_v2_2/training_eval.py:13: in \n from .grid_induction import run_public_task_v2_2\nE File \"/home/runner/work/MLE-agent/MLE-agent/research/universal_core/holonomy_v2_2/grid_induction.py\", line 366\nE return result\nE ^\nE IndentationError: unindent does not match any outer indentation level\n=========================== short test summary info ============================\nERROR research/universal_core/holonomy_v2_2/tests/test_component_rank.py\nERROR research/universal_core/holonomy_v2_2/tests/test_component_select.py\nERROR research/universal_core/holonomy_v2_2/tests/test_grid_induction.py\nERROR research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py\nERROR research/universal_core/holonomy_v2_2/tests/test_panel_combine.py\nERROR research/universal_core/holonomy_v2_2/tests/test_region_fill.py\nERROR research/universal_core/holonomy_v2_2/tests/test_training_eval.py\n!!!!!!!!!!!!!!!!!!! Interrupted: 7 errors during collection !!!!!!!!!!!!!!!!!!!!\n7 errors in 0.58s\n", + "workflow_source_commit": "7b51455b077950fb49b041cd83dcd773f8c7fa84" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/GRID_MECHANISTIC_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/GRID_MECHANISTIC_GREEN.json new file mode 100644 index 00000000..a631e919 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/GRID_MECHANISTIC_GREEN.json @@ -0,0 +1,16 @@ +{ + "focused_output": "............. [100%]\n13 passed in 0.27s\n", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "frozen_output": "..... [100%]\n5 passed in 0.07s\n", + "outcomes": { + "focused": "success", + "frozen_byte_gate": "success", + "frozen_tests": "success", + "v2_2_cluster": "success" + }, + "passed": true, + "run_id": "30948952495", + "schema_version": 2, + "v2_2_output": ".............................................. [100%]\n46 passed in 0.33s\n", + "workflow_source_commit": "73d05d58c70e8733bc80124fefef76e3d606e9a6" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/GRID_MECHANISTIC_RED.json b/research/universal_core/holonomy_v2_2/evidence/GRID_MECHANISTIC_RED.json new file mode 100644 index 00000000..a81258ca --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/GRID_MECHANISTIC_RED.json @@ -0,0 +1,10 @@ +{ + "captured_log": "FFFFFFF [100%]\n=================================== FAILURES ===================================\n________________ test_mechanistic_mode_has_exact_compact_parity ________________\n\n def test_mechanistic_mode_has_exact_compact_parity() -> None:\n value = task(\n [{\"input\": [[0, 1]], \"output\": [[2, 3]]}],\n [[[1, 0]]],\n \"parity\",\n )\n \n compact = grid_induction.run_public_task_v2_2(value, engine=None)\n> traced = grid_induction.run_public_task_v2_2(value, engine=None, mechanistic=True)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nE TypeError: run_public_task_v2_2() got an unexpected keyword argument 'mechanistic'\n\nresearch/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py:41: TypeError\n______ test_trace_records_every_candidate_demonstration_and_terminal_fit _______\n\n def test_trace_records_every_candidate_demonstration_and_terminal_fit() -> None:\n value = task(\n [{\"input\": [[1, 2]], \"output\": [[2, 1]]}],\n [[[3, 4]]],\n \"candidate-ledger\",\n )\n \n> trace = grid_induction.run_public_task_v2_2(\n value,\n engine=None,\n mechanistic=True,\n )[\"mechanistic_trace\"]\nE TypeError: run_public_task_v2_2() got an unexpected keyword argument 'mechanistic'\n\nresearch/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py:55: TypeError\n____________ test_hidden_component_rank_key_error_is_fully_recorded ____________\n\n def test_hidden_component_rank_key_error_is_fully_recorded() -> None:\n value = task(\n [{\n \"input\": [[5, 0, 5], [0, 0, 5]],\n \"output\": [[1, 0, 2], [0, 0, 2]],\n }],\n [[[5, 0, 5, 0, 5], [0, 0, 5, 0, 5], [0, 0, 0, 0, 5]]],\n \"unseen-rank\",\n )\n \n> result = grid_induction.run_public_task_v2_2(value, None, mechanistic=True)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nE TypeError: run_public_task_v2_2() got an unexpected keyword argument 'mechanistic'\n\nresearch/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py:85: TypeError\n__________ test_trace_replay_is_deterministic_and_task_id_independent __________\n\n def test_trace_replay_is_deterministic_and_task_id_independent() -> None:\n demos = [{\"input\": [[1, 2]], \"output\": [[2, 1]]}]\n> first = grid_induction.run_public_task_v2_2(\n task(demos, [[[3, 4]]], \"first\"),\n None,\n mechanistic=True,\n )[\"mechanistic_trace\"]\nE TypeError: run_public_task_v2_2() got an unexpected keyword argument 'mechanistic'\n\nresearch/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py:101: TypeError\n____________ test_ambiguous_program_trace_lists_every_output_class _____________\n\n def test_ambiguous_program_trace_lists_every_output_class() -> None:\n value = task(\n [{\"input\": [[1, 1], [1, 1]], \"output\": [[1, 1], [1, 1]]}],\n [[[1, 2], [3, 4]]],\n \"ambiguous\",\n )\n \n> result = grid_induction.run_public_task_v2_2(value, None, mechanistic=True)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nE TypeError: run_public_task_v2_2() got an unexpected keyword argument 'mechanistic'\n\nresearch/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py:129: TypeError\n____________ test_grammar_exhaustion_trace_has_no_fitting_candidate ____________\n\n def test_grammar_exhaustion_trace_has_no_fitting_candidate() -> None:\n value = task(\n [{\"input\": [[1, 2], [3, 4]], \"output\": [[9, 9, 9], [9, 9, 9], [9, 9, 9]]}],\n [[[5]]],\n \"exhausted\",\n )\n \n> result = grid_induction.run_public_task_v2_2(value, None, mechanistic=True)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nE TypeError: run_public_task_v2_2() got an unexpected keyword argument 'mechanistic'\n\nresearch/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py:156: TypeError\n________ test_non_grid_mechanistic_mode_marks_frozen_delegate_as_opaque ________\n\nmonkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x7f859f03f6b0>\n\n def test_non_grid_mechanistic_mode_marks_frozen_delegate_as_opaque(monkeypatch: Any) -> None:\n expected = {\"task_id\": \"text\", \"predictions\": [{\"status\": \"ACCEPTED\", \"prediction\": \"B\"}]}\n \n def fake_delegate(value: object, engine: object) -> dict[str, object]:\n return expected\n \n monkeypatch.setattr(grid_induction, \"run_public_task_v2\", fake_delegate)\n value = task([{\"input\": \"A\", \"output\": \"B\"}], [\"A\"], \"text\")\n \n> result = grid_induction.run_public_task_v2_2(value, object(), mechanistic=True)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nE TypeError: run_public_task_v2_2() got an unexpected keyword argument 'mechanistic'\n\nresearch/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py:177: TypeError\n=========================== short test summary info ============================\nFAILED research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py::test_mechanistic_mode_has_exact_compact_parity - TypeError: run_public_task_v2_2() got an unexpected keyword argument 'mechanistic'\nFAILED research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py::test_trace_records_every_candidate_demonstration_and_terminal_fit - TypeError: run_public_task_v2_2() got an unexpected keyword argument 'mechanistic'\nFAILED research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py::test_hidden_component_rank_key_error_is_fully_recorded - TypeError: run_public_task_v2_2() got an unexpected keyword argument 'mechanistic'\nFAILED research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py::test_trace_replay_is_deterministic_and_task_id_independent - TypeError: run_public_task_v2_2() got an unexpected keyword argument 'mechanistic'\nFAILED research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py::test_ambiguous_program_trace_lists_every_output_class - TypeError: run_public_task_v2_2() got an unexpected keyword argument 'mechanistic'\nFAILED research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py::test_grammar_exhaustion_trace_has_no_fitting_candidate - TypeError: run_public_task_v2_2() got an unexpected keyword argument 'mechanistic'\nFAILED research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py::test_non_grid_mechanistic_mode_marks_frozen_delegate_as_opaque - TypeError: run_public_task_v2_2() got an unexpected keyword argument 'mechanistic'\n7 failed in 0.16s\n", + "expected_failure": "TypeError: run_public_task_v2_2() got an unexpected keyword argument 'mechanistic'", + "failure_confirmed": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "red_commit": "2ad225d0cc5fa1ce6d575aabae70c7b08da5fbb7", + "red_run_id": "30948438874", + "schema_version": 1, + "test_command": "python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/LINE_CONNECT_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/LINE_CONNECT_GREEN.json new file mode 100644 index 00000000..b3ab3881 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/LINE_CONNECT_GREEN.json @@ -0,0 +1,35 @@ +{ + "expected_cumulative_metrics": { + "accepted": 96, + "correct": 96, + "failed": 0, + "incorrect_attempts": 0, + "rows": 1076 + }, + "expected_new_exact_tasks": [ + "1f876c06", + "22168020", + "22eb0ac0", + "ded97339" + ], + "expected_training_delta": { + "accepted_rows": 4, + "correct_rows": 4, + "grammar_exhausted_rows": -4 + }, + "focused_output": "........................................................................ [ 94%]\n.... [100%]\n76 passed in 2.71s\n", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "frozen_output": "..... [100%]\n5 passed in 0.08s\n", + "outcomes": { + "focused": "success", + "frozen_byte_gate": "success", + "frozen_tests": "success", + "v2_2_cluster": "success" + }, + "passed": true, + "public_arc_evaluation_rerun": false, + "run_id": "31911734711", + "schema_version": 1, + "v2_2_output": "........................................................................ [ 60%]\n............................................... [100%]\n119 passed in 2.68s\n", + "workflow_source_commit": "c3bea5c185a2213b29a3c74d2c01cf900f008a28" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/LINE_CONNECT_MODULE_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/LINE_CONNECT_MODULE_GREEN.json new file mode 100644 index 00000000..8a30726a --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/LINE_CONNECT_MODULE_GREEN.json @@ -0,0 +1,17 @@ +{ + "expected_new_exact_tasks": [ + "1f876c06", + "22168020", + "22eb0ac0", + "ded97339" + ], + "frozen_byte_gate": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "passed": true, + "public_arc_evaluation_rerun": false, + "run_id": "31911659286", + "schema_version": 1, + "source_commit": "121d127c415c51e30ba254723bd5c9da9cf0fe35", + "test_count": 9, + "test_output": "......... [100%]\n9 passed in 0.14s\n" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/LINE_CONNECT_RED.json b/research/universal_core/holonomy_v2_2/evidence/LINE_CONNECT_RED.json new file mode 100644 index 00000000..883b89d1 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/LINE_CONNECT_RED.json @@ -0,0 +1,35 @@ +{ + "captured_log": "\n==================================== ERRORS ====================================\n_ ERROR collecting research/universal_core/holonomy_v2_2/tests/test_line_connect.py _\nImportError while importing test module '/home/runner/work/MLE-agent/MLE-agent/research/universal_core/holonomy_v2_2/tests/test_line_connect.py'.\nHint: make sure your test modules/packages have valid Python names.\nTraceback:\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nresearch/universal_core/holonomy_v2_2/tests/test_line_connect.py:10: in \n from research.universal_core.holonomy_v2_2.line_connect import (\nE ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.line_connect'\n=========================== short test summary info ============================\nERROR research/universal_core/holonomy_v2_2/tests/test_line_connect.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.25s\n", + "expected_cumulative_metrics": { + "accepted": 96, + "correct": 96, + "failed": 0, + "incorrect_attempts": 0, + "rows": 1076 + }, + "expected_failure": "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.line_connect'", + "expected_new_exact_tasks": [ + "1f876c06", + "22168020", + "22eb0ac0", + "ded97339" + ], + "expected_status_counts": { + "ACCEPTED": 96, + "AMBIGUOUS_PROGRAM": 1, + "GRAMMAR_EXHAUSTED": 979 + }, + "failure_confirmed": true, + "fixture_integrity": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "public_arc_evaluation_rerun": false, + "run_id": "31911615768", + "schema_version": 1, + "source_commit": "f6411f125094e9e6d38162c35f9a37d300e2474c", + "training_scan": { + "ambiguous_hidden_output_classes": 0, + "compatible_accepted_tasks": 0, + "compatible_tasks": 4, + "wrong_completed_hidden_outputs": 0 + } +} diff --git a/research/universal_core/holonomy_v2_2/evidence/MARKER_RECOLOR_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/MARKER_RECOLOR_GREEN.json new file mode 100644 index 00000000..f93a9e61 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/MARKER_RECOLOR_GREEN.json @@ -0,0 +1,23 @@ +{ + "expected_training_delta": { + "accepted_rows": 1, + "correct_rows": 1, + "execution_failed_rows": -1, + "task_id": "aabf363d" + }, + "focused_output": "........... [100%]\n11 passed in 0.33s\n", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "frozen_output": "..... [100%]\n5 passed in 0.08s\n", + "outcomes": { + "focused": "success", + "frozen_byte_gate": "success", + "frozen_tests": "success", + "v2_2_cluster": "success" + }, + "passed": true, + "run_id": "30950841994", + "schema_version": 1, + "source_trace_digest": "29544f72d9511885b6ea71838162ee86d83437198841c2e4815a5ffb824f54e4", + "v2_2_output": "....................................................... [100%]\n55 passed in 0.45s\n", + "workflow_source_commit": "215ebf17475db1b518b1a8dbc4db707da08dc916" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/MARKER_RECOLOR_RED.json b/research/universal_core/holonomy_v2_2/evidence/MARKER_RECOLOR_RED.json new file mode 100644 index 00000000..e5d060a0 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/MARKER_RECOLOR_RED.json @@ -0,0 +1,15 @@ +{ + "captured_log": "\n==================================== ERRORS ====================================\n_ ERROR collecting research/universal_core/holonomy_v2_2/tests/test_marker_recolor.py _\nImportError while importing test module '/home/runner/work/MLE-agent/MLE-agent/research/universal_core/holonomy_v2_2/tests/test_marker_recolor.py'.\nHint: make sure your test modules/packages have valid Python names.\nTraceback:\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nresearch/universal_core/holonomy_v2_2/tests/test_marker_recolor.py:8: in \n from research.universal_core.holonomy_v2_2.marker_recolor import (\nE ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.marker_recolor'\n=========================== short test summary info ============================\nERROR research/universal_core/holonomy_v2_2/tests/test_marker_recolor.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.18s\n", + "expected_failure": "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.marker_recolor'", + "failure_confirmed": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "red_commit": "fd92b92d4cd3dbe62b09632a5b012698a71a4c59", + "red_run_id": "30950705042", + "schema_version": 1, + "source_trace": { + "failure": "color_map KeyError: 'unmapped grid cell'", + "task_id": "aabf363d", + "trace_digest": "29544f72d9511885b6ea71838162ee86d83437198841c2e4815a5ffb824f54e4" + }, + "test_command": "python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_marker_recolor.py" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/MECHANISTIC_TRACE_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/MECHANISTIC_TRACE_GREEN.json new file mode 100644 index 00000000..90328011 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/MECHANISTIC_TRACE_GREEN.json @@ -0,0 +1,14 @@ +{ + "branch": "universal-core-arc-training-v2-2", + "focused_command": "python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_mechanistic_trace.py", + "focused_output": "...... [100%]\n6 passed in 0.03s\n", + "frozen_byte_gate": true, + "frozen_command": "python -m pytest -q research/universal_core/holonomy_v2/tests/test_blind_adapter.py research/universal_core/holonomy_v2/tests/test_engine.py", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "frozen_output": "..... [100%]\n5 passed in 0.08s\n", + "green_commit": "bdf3baf7fbbf02076503640c8f27e8d3f2423917", + "green_run_id": "30947622159", + "schema_version": 1, + "v2_2_command": "python -m pytest -q research/universal_core/holonomy_v2_2/tests", + "v2_2_output": "................................. [100%]\n33 passed in 0.19s\n" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/MECHANISTIC_TRACE_RED.json b/research/universal_core/holonomy_v2_2/evidence/MECHANISTIC_TRACE_RED.json new file mode 100644 index 00000000..65503e6c --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/MECHANISTIC_TRACE_RED.json @@ -0,0 +1,11 @@ +{ + "branch": "universal-core-arc-training-v2-2", + "captured_log": "\n==================================== ERRORS ====================================\n_ ERROR collecting research/universal_core/holonomy_v2_2/tests/test_mechanistic_trace.py _\nImportError while importing test module '/home/runner/work/MLE-agent/MLE-agent/research/universal_core/holonomy_v2_2/tests/test_mechanistic_trace.py'.\nHint: make sure your test modules/packages have valid Python names.\nTraceback:\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nresearch/universal_core/holonomy_v2_2/tests/test_mechanistic_trace.py:8: in \n from research.universal_core.holonomy_v2_2.mechanistic_trace import (\nE ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.mechanistic_trace'\n=========================== short test summary info ============================\nERROR research/universal_core/holonomy_v2_2/tests/test_mechanistic_trace.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.12s\n", + "expected_failure": "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.mechanistic_trace'", + "failure_confirmed": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "red_commit": "ed0b66d72faab36df527bc656c52be56c720e439", + "red_run_id": "30947444119", + "schema_version": 1, + "test_command": "python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_mechanistic_trace.py" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/MIRROR_CONCAT_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/MIRROR_CONCAT_GREEN.json new file mode 100644 index 00000000..248ab1c7 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/MIRROR_CONCAT_GREEN.json @@ -0,0 +1,28 @@ +{ + "expected_new_exact_tasks": [ + "4c4377d9", + "6d0aefbc", + "6fa7a44f", + "8be77c9e", + "c9e6f938" + ], + "expected_training_delta": { + "accepted_rows": 5, + "correct_rows": 5, + "grammar_exhausted_rows": -5 + }, + "focused_output": ".......................... [100%]\n26 passed in 0.87s\n", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "frozen_output": "..... [100%]\n5 passed in 0.08s\n", + "outcomes": { + "focused": "success", + "frozen_byte_gate": "success", + "frozen_tests": "success", + "v2_2_cluster": "success" + }, + "passed": true, + "run_id": "30953728750", + "schema_version": 1, + "v2_2_output": "...................................................................... [100%]\n70 passed in 0.98s\n", + "workflow_source_commit": "c4a46c92ff96f2b13655aea02846e3a42416283b" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/MIRROR_CONCAT_RED.json b/research/universal_core/holonomy_v2_2/evidence/MIRROR_CONCAT_RED.json new file mode 100644 index 00000000..27f284b3 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/MIRROR_CONCAT_RED.json @@ -0,0 +1,20 @@ +{ + "expected_failure": "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.mirror_concat'", + "expected_new_exact_tasks": [ + "4c4377d9", + "6d0aefbc", + "6fa7a44f", + "8be77c9e", + "c9e6f938" + ], + "failure_confirmed": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "red_commit": "5bd85542a5f049043c9c428574a1ab36d5b75781", + "red_run_id": "30953619229", + "schema_version": 1, + "test_command": "python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_mirror_concat.py", + "training_scan": { + "ambiguous_hidden_outputs": 0, + "wrong_completed_hidden_outputs": 0 + } +} diff --git a/research/universal_core/holonomy_v2_2/evidence/PALETTE_FIXTURE_INTEGRITY_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/PALETTE_FIXTURE_INTEGRITY_GREEN.json new file mode 100644 index 00000000..2b8e078d --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/PALETTE_FIXTURE_INTEGRITY_GREEN.json @@ -0,0 +1,10 @@ +{ + "fixture_count": 4, + "fixture_integrity": true, + "public_arc_evaluation_rerun": false, + "run_id": "30974942803", + "schema_version": 1, + "source_artifact_run": 30929169392, + "test_output": ". [100%]\n1 passed in 0.02s\n", + "workflow_source_commit": "086d9417e0a69aa647219e9af6bd5e01191afa9a" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/PALETTE_REPEAT_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/PALETTE_REPEAT_GREEN.json new file mode 100644 index 00000000..2122c564 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/PALETTE_REPEAT_GREEN.json @@ -0,0 +1,29 @@ +{ + "expected_new_exact_tasks": [ + "a59b95c0", + "ac0a08a4", + "b91ae062", + "d4b1c2b1" + ], + "expected_training_delta": { + "accepted_rows": 4, + "correct_rows": 4, + "grammar_exhausted_rows": -4 + }, + "focused_output": "........................................... [100%]\n43 passed in 1.37s\n", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "frozen_output": "..... [100%]\n5 passed in 0.09s\n", + "outcomes": { + "focused": "success", + "frozen_byte_gate": "success", + "frozen_tests": "success", + "v2_2_cluster": "success" + }, + "ownership_boundary": "NO_FACTOR_VARIATION", + "passed": true, + "public_arc_evaluation_rerun": false, + "run_id": "30975148228", + "schema_version": 1, + "v2_2_output": "........................................................................ [ 83%]\n.............. [100%]\n86 passed in 1.41s\n", + "workflow_source_commit": "81923655a57ddcb686f4416706d479e90e09bed4" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/PALETTE_REPEAT_MODULE_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/PALETTE_REPEAT_MODULE_GREEN.json new file mode 100644 index 00000000..9105fb4d --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/PALETTE_REPEAT_MODULE_GREEN.json @@ -0,0 +1,11 @@ +{ + "frozen_byte_gate": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "passed": true, + "public_arc_evaluation_rerun": false, + "run_id": "30975042432", + "schema_version": 1, + "source_commit": "e0409e1f0c929480ddf114fd6fa643c1925dfe1b", + "test_count": 7, + "test_output": "....... [100%]\n7 passed in 0.12s\n" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/PALETTE_REPEAT_RED.json b/research/universal_core/holonomy_v2_2/evidence/PALETTE_REPEAT_RED.json new file mode 100644 index 00000000..b40d2491 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/PALETTE_REPEAT_RED.json @@ -0,0 +1,18 @@ +{ + "captured_log": "\n==================================== ERRORS ====================================\n_ ERROR collecting research/universal_core/holonomy_v2_2/tests/test_palette_repeat.py _\nImportError while importing test module '/home/runner/work/MLE-agent/MLE-agent/research/universal_core/holonomy_v2_2/tests/test_palette_repeat.py'.\nHint: make sure your test modules/packages have valid Python names.\nTraceback:\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nresearch/universal_core/holonomy_v2_2/tests/test_palette_repeat.py:10: in \n from research.universal_core.holonomy_v2_2.palette_repeat import (\nE ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.palette_repeat'\n=========================== short test summary info ============================\nERROR research/universal_core/holonomy_v2_2/tests/test_palette_repeat.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.24s\n", + "expected_failure": "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.palette_repeat'", + "expected_new_exact_tasks": [ + "a59b95c0", + "ac0a08a4", + "b91ae062", + "d4b1c2b1" + ], + "failure_confirmed": true, + "fixture_integrity": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "public_arc_evaluation_rerun": false, + "run_id": "30974984366", + "schema_version": 2, + "source_commit": "fa64b1ac64f33df194a72c09801b2ea45ff95f70", + "test_command": "python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_palette_repeat.py" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/PANEL_BOOLEAN_TDD.json b/research/universal_core/holonomy_v2_2/evidence/PANEL_BOOLEAN_TDD.json new file mode 100644 index 00000000..0fd31bef --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/PANEL_BOOLEAN_TDD.json @@ -0,0 +1,11 @@ +{ + "development_split": "ARC-AGI-2 training only", + "fixture_correction_commit": "0131bbe4945ab7f83fd0830237acdfbcb4ed0397", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "implementation_module_commit": "6c59d733c489c90fa2b29759ff727a06ef73630a", + "integration_commit": "8dc3685abf4cfb99c64a84cafa41585114b64f8e", + "red_commit": "a22dd06c7593cdd96c73c4a08bc3d483f93e629e", + "red_run_id": "30939098116", + "red_reason": "four panel-combination cases returned GRAMMAR_EXHAUSTED", + "schema_version": 1 +} diff --git a/research/universal_core/holonomy_v2_2/evidence/QUADRANT_FIXTURE_INTEGRITY_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/QUADRANT_FIXTURE_INTEGRITY_GREEN.json new file mode 100644 index 00000000..1a2346c9 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/QUADRANT_FIXTURE_INTEGRITY_GREEN.json @@ -0,0 +1,12 @@ +{ + "fixture_count": 9, + "frozen_byte_gate": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "passed": true, + "public_arc_evaluation_rerun": false, + "run_id": "30973384648", + "schema_version": 1, + "source_commit": "9bd9a9e3029176d184c498af8aa8b16d7e21a953", + "test_count": 5, + "test_output": "..... [100%]\n5 passed in 0.12s\n" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/QUADRANT_FIXTURE_INTEGRITY_RED.json b/research/universal_core/holonomy_v2_2/evidence/QUADRANT_FIXTURE_INTEGRITY_RED.json new file mode 100644 index 00000000..fa57d377 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/QUADRANT_FIXTURE_INTEGRITY_RED.json @@ -0,0 +1,10 @@ +{ + "captured_log": "F [100%]\n=================================== FAILURES ===================================\n_________ test_quadrant_mosaic_fixtures_match_pinned_training_payloads _________\n\n def test_quadrant_mosaic_fixtures_match_pinned_training_payloads() -> None:\n actual = {\n path.name: hashlib.sha256(path.read_bytes()).hexdigest()\n for path in sorted(_FIXTURES.glob(\"*.json\"))\n }\n \n> assert actual == _EXPECTED_SHA256\nE AssertionError: assert {'0c786b71.js...37324d6', ...} == {'0c786b71.js...4b48dc4', ...}\nE \nE Differing items:\nE {'0c786b71.json': 'b032ac7ccb2f10dbf34d58fecb1de91c2816d7becd632729910eb43389800377'} != {'0c786b71.json': 'df8a0f7356a4b200d7c9a4d8f58d40662f01563ebb25c85f67c718bdb487a3f2'}\nE {'ed98d772.json': '7290e0757633737ae70efa2cc63d9f26893cc4dbd4029debeff0c5ee1a04ed0c'} != {'ed98d772.json': 'f7f2569ee2b94b6e4d98ff72e3f36ff83441a802f3bc7c19ea8c04c81108b18b'}\nE {'62c24649.json': '0b783091ab9a5646dbf4d7ebd4c9c9c25b59cae81976d8f6ade61985137324d6'} != {'62c24649.json': 'bd4124704f8b1b02038f80c7cb1187224d1daf4dd528cea940486bb294b48dc4'}\nE {'7953d61e.json': '85389de113890e2e5e7170cfdb741770ec29b9b1e33ddb6e4e74f899a08b1bd0'} != {'7953d61e.json': '789c244eae24dc51956ed3beea9b530b9af8d895692b384bd6fa3eb04e3e6f12'}\nE {'7fe24cdd.json': 'c535869e1e18bab2fd0dade6e327d18745200cf507e5b7890a6f6bbf148a5ac9'} != {'7fe24cdd.json': '6deb4aa52a4cc3d63cf8b8b99846e50297837910a1b9c6d60f7e028cee8c68d7'}\nE {'67e8384a.json': 'e5c2f8360750e942338751000b3933c10a1da15ccda91829e5af658c686b7f8d'} != {'67e8384a.json': '3ad82285fa52a72081cfb705f80a224cf4f984232a1e002421d12b38b83a012d'}\nE {'3af2c5a8.json': '880f547cc521fc787b035629380b2a324c712917809f5241ac7377c42e799ad4'} != {'3af2c5a8.json': '969747e767a1af9f23689cc7e262abaadccf1e02ff8c2a446f738daaa4a24a48'}\nE {'833dafe3.json': 'e4c564f974362f596bfe722c946f469e847aebb11ec9eb0e50a66acafee30127'} != {'833dafe3.json': 'c3c9b31eb8def554991acfc0be966e49cb128568a5f948452be5b9bda1ce9dd2'}\nE {'46442a0e.json': 'f55c551b95a0e0e57c9d4a7e2ced030ef0f9f1d61ba3f812845b509e58b985a3'} != {'46442a0e.json': 'cd58b40fb6c991567f666e2d42a98a8a5ba9de6708011d3ffb36245487aba5ba'}\nE \nE Full diff:\nE {\nE - '0c786b71.json': 'df8a0f7356a4b200d7c9a4d8f58d40662f01563ebb25c85f67c718bdb487a3f2',\nE - '3af2c5a8.json': '969747e767a1af9f23689cc7e262abaadccf1e02ff8c2a446f738daaa4a24a48',\nE - '46442a0e.json': 'cd58b40fb6c991567f666e2d42a98a8a5ba9de6708011d3ffb36245487aba5ba',\nE - '62c24649.json': 'bd4124704f8b1b02038f80c7cb1187224d1daf4dd528cea940486bb294b48dc4',\nE - '67e8384a.json': '3ad82285fa52a72081cfb705f80a224cf4f984232a1e002421d12b38b83a012d',\nE - '7953d61e.json': '789c244eae24dc51956ed3beea9b530b9af8d895692b384bd6fa3eb04e3e6f12',\nE - '7fe24cdd.json': '6deb4aa52a4cc3d63cf8b8b99846e50297837910a1b9c6d60f7e028cee8c68d7',\nE - '833dafe3.json': 'c3c9b31eb8def554991acfc0be966e49cb128568a5f948452be5b9bda1ce9dd2',\nE - 'ed98d772.json': 'f7f2569ee2b94b6e4d98ff72e3f36ff83441a802f3bc7c19ea8c04c81108b18b',\nE + '0c786b71.json': 'b032ac7ccb2f10dbf34d58fecb1de91c2816d7becd632729910eb43389800377',\nE + '3af2c5a8.json': '880f547cc521fc787b035629380b2a324c712917809f5241ac7377c42e799ad4',\nE + '46442a0e.json': 'f55c551b95a0e0e57c9d4a7e2ced030ef0f9f1d61ba3f812845b509e58b985a3',\nE + '62c24649.json': '0b783091ab9a5646dbf4d7ebd4c9c9c25b59cae81976d8f6ade61985137324d6',\nE + '67e8384a.json': 'e5c2f8360750e942338751000b3933c10a1da15ccda91829e5af658c686b7f8d',\nE + '7953d61e.json': '85389de113890e2e5e7170cfdb741770ec29b9b1e33ddb6e4e74f899a08b1bd0',\nE + '7fe24cdd.json': 'c535869e1e18bab2fd0dade6e327d18745200cf507e5b7890a6f6bbf148a5ac9',\nE + '833dafe3.json': 'e4c564f974362f596bfe722c946f469e847aebb11ec9eb0e50a66acafee30127',\nE + 'ed98d772.json': '7290e0757633737ae70efa2cc63d9f26893cc4dbd4029debeff0c5ee1a04ed0c',\nE }\n\nresearch/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic_fixtures.py:27: AssertionError\n=========================== short test summary info ============================\nFAILED research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic_fixtures.py::test_quadrant_mosaic_fixtures_match_pinned_training_payloads - AssertionError: assert {'0c786b71.js...37324d6', ...} == {'0c786b71.js...4b48dc4', ...}\n \n Differing items:\n {'0c786b71.json': 'b032ac7ccb2f10dbf34d58fecb1de91c2816d7becd632729910eb43389800377'} != {'0c786b71.json': 'df8a0f7356a4b200d7c9a4d8f58d40662f01563ebb25c85f67c718bdb487a3f2'}\n {'ed98d772.json': '7290e0757633737ae70efa2cc63d9f26893cc4dbd4029debeff0c5ee1a04ed0c'} != {'ed98d772.json': 'f7f2569ee2b94b6e4d98ff72e3f36ff83441a802f3bc7c19ea8c04c81108b18b'}\n {'62c24649.json': '0b783091ab9a5646dbf4d7ebd4c9c9c25b59cae81976d8f6ade61985137324d6'} != {'62c24649.json': 'bd4124704f8b1b02038f80c7cb1187224d1daf4dd528cea940486bb294b48dc4'}\n {'7953d61e.json': '85389de113890e2e5e7170cfdb741770ec29b9b1e33ddb6e4e74f899a08b1bd0'} != {'7953d61e.json': '789c244eae24dc51956ed3beea9b530b9af8d895692b384bd6fa3eb04e3e6f12'}\n {'7fe24cdd.json': 'c535869e1e18bab2fd0dade6e327d18745200cf507e5b7890a6f6bbf148a5ac9'} != {'7fe24cdd.json': '6deb4aa52a4cc3d63cf8b8b99846e50297837910a1b9c6d60f7e028cee8c68d7'}\n {'67e8384a.json': 'e5c2f8360750e942338751000b3933c10a1da15ccda91829e5af658c686b7f8d'} != {'67e8384a.json': '3ad82285fa52a72081cfb705f80a224cf4f984232a1e002421d12b38b83a012d'}\n {'3af2c5a8.json': '880f547cc521fc787b035629380b2a324c712917809f5241ac7377c42e799ad4'} != {'3af2c5a8.json': '969747e767a1af9f23689cc7e262abaadccf1e02ff8c2a446f738daaa4a24a48'}\n {'833dafe3.json': 'e4c564f974362f596bfe722c946f469e847aebb11ec9eb0e50a66acafee30127'} != {'833dafe3.json': 'c3c9b31eb8def554991acfc0be966e49cb128568a5f948452be5b9bda1ce9dd2'}\n {'46442a0e.json': 'f55c551b95a0e0e57c9d4a7e2ced030ef0f9f1d61ba3f812845b509e58b985a3'} != {'46442a0e.json': 'cd58b40fb6c991567f666e2d42a98a8a5ba9de6708011d3ffb36245487aba5ba'}\n \n Full diff:\n {\n - '0c786b71.json': 'df8a0f7356a4b200d7c9a4d8f58d40662f01563ebb25c85f67c718bdb487a3f2',\n - '3af2c5a8.json': '969747e767a1af9f23689cc7e262abaadccf1e02ff8c2a446f738daaa4a24a48',\n - '46442a0e.json': 'cd58b40fb6c991567f666e2d42a98a8a5ba9de6708011d3ffb36245487aba5ba',\n - '62c24649.json': 'bd4124704f8b1b02038f80c7cb1187224d1daf4dd528cea940486bb294b48dc4',\n - '67e8384a.json': '3ad82285fa52a72081cfb705f80a224cf4f984232a1e002421d12b38b83a012d',\n - '7953d61e.json': '789c244eae24dc51956ed3beea9b530b9af8d895692b384bd6fa3eb04e3e6f12',\n - '7fe24cdd.json': '6deb4aa52a4cc3d63cf8b8b99846e50297837910a1b9c6d60f7e028cee8c68d7',\n - '833dafe3.json': 'c3c9b31eb8def554991acfc0be966e49cb128568a5f948452be5b9bda1ce9dd2',\n - 'ed98d772.json': 'f7f2569ee2b94b6e4d98ff72e3f36ff83441a802f3bc7c19ea8c04c81108b18b',\n + '0c786b71.json': 'b032ac7ccb2f10dbf34d58fecb1de91c2816d7becd632729910eb43389800377',\n + '3af2c5a8.json': '880f547cc521fc787b035629380b2a324c712917809f5241ac7377c42e799ad4',\n + '46442a0e.json': 'f55c551b95a0e0e57c9d4a7e2ced030ef0f9f1d61ba3f812845b509e58b985a3',\n + '62c24649.json': '0b783091ab9a5646dbf4d7ebd4c9c9c25b59cae81976d8f6ade61985137324d6',\n + '67e8384a.json': 'e5c2f8360750e942338751000b3933c10a1da15ccda91829e5af658c686b7f8d',\n + '7953d61e.json': '85389de113890e2e5e7170cfdb741770ec29b9b1e33ddb6e4e74f899a08b1bd0',\n + '7fe24cdd.json': 'c535869e1e18bab2fd0dade6e327d18745200cf507e5b7890a6f6bbf148a5ac9',\n + '833dafe3.json': 'e4c564f974362f596bfe722c946f469e847aebb11ec9eb0e50a66acafee30127',\n + 'ed98d772.json': '7290e0757633737ae70efa2cc63d9f26893cc4dbd4029debeff0c5ee1a04ed0c',\n }\n1 failed in 0.06s\n", + "failure_confirmed": true, + "public_arc_evaluation_rerun": false, + "root_cause": "fixture files were manually transcribed instead of copied byte-for-byte from the pinned ARC training corpus", + "run_id": "30973183086", + "schema_version": 1, + "source_commit": "b1e2a0ca5110ecdbbe7da57ce9a5351519f65d7c", + "test_command": "python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic_fixtures.py" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/QUADRANT_MOSAIC_DIAGNOSTIC.json b/research/universal_core/holonomy_v2_2/evidence/QUADRANT_MOSAIC_DIAGNOSTIC.json new file mode 100644 index 00000000..610f6f3c --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/QUADRANT_MOSAIC_DIAGNOSTIC.json @@ -0,0 +1,33 @@ +{ + "expected_new_exact_tasks": [ + "0c786b71", + "3af2c5a8", + "46442a0e", + "62c24649", + "67e8384a", + "7953d61e", + "7fe24cdd", + "833dafe3", + "ed98d772" + ], + "expected_training_delta": { + "accepted_rows": 9, + "correct_rows": 9, + "grammar_exhausted_rows": -9 + }, + "focused_output": "................................. [100%]\n33 passed in 2.99s\n", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "frozen_output": "..... [100%]\n5 passed in 0.08s\n", + "outcomes": { + "focused": "success", + "frozen_byte_gate": "success", + "frozen_tests": "success", + "v2_2_cluster": "failure" + }, + "passed": false, + "public_arc_evaluation_rerun": false, + "run_id": "30973470601", + "schema_version": 1, + "v2_2_output": "....................F................................................... [ 93%]\n..... [100%]\n=================================== FAILURES ===================================\n_______________ test_cell_scale_and_whole_grid_tile_are_distinct _______________\n\n def test_cell_scale_and_whole_grid_tile_are_distinct() -> None:\n scale = task(\n [{\"input\": [[1, 2]], \"output\": [[1, 1, 2, 2], [1, 1, 2, 2]]}],\n [[[3], [4]]],\n \"scale\",\n )\n tile = task(\n [{\"input\": [[1, 2]], \"output\": [[1, 2, 1, 2], [1, 2, 1, 2]]}],\n [[[3], [4]]],\n \"tile\",\n )\n \n scaled = grid_induction.run_public_task_v2_2(scale, engine=None)\n tiled = grid_induction.run_public_task_v2_2(tile, engine=None)\n \n assert scaled[\"predictions\"][0][\"prediction\"] == [[3, 3], [3, 3], [4, 4], [4, 4]]\n> assert tiled[\"predictions\"][0][\"prediction\"] == [[3, 3], [4, 4], [3, 3], [4, 4]]\nE assert None == [[3, 3], [4, 4], [3, 3], [4, 4]]\n\nresearch/universal_core/holonomy_v2_2/tests/test_grid_induction.py:67: AssertionError\n=========================== short test summary info ============================\nFAILED research/universal_core/holonomy_v2_2/tests/test_grid_induction.py::test_cell_scale_and_whole_grid_tile_are_distinct - assert None == [[3, 3], [4, 4], [3, 3], [4, 4]]\n1 failed, 76 passed in 3.10s\n", + "workflow_source_commit": "97133b1631258e0018b4c9428a28db68e1c359ac" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/QUADRANT_MOSAIC_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/QUADRANT_MOSAIC_GREEN.json new file mode 100644 index 00000000..c5f260c5 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/QUADRANT_MOSAIC_GREEN.json @@ -0,0 +1,34 @@ +{ + "expected_new_exact_tasks": [ + "0c786b71", + "3af2c5a8", + "46442a0e", + "62c24649", + "67e8384a", + "7953d61e", + "7fe24cdd", + "833dafe3", + "ed98d772" + ], + "expected_training_delta": { + "accepted_rows": 9, + "correct_rows": 9, + "grammar_exhausted_rows": -9 + }, + "focused_output": "................................... [100%]\n35 passed in 1.13s\n", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "frozen_output": "..... [100%]\n5 passed in 0.08s\n", + "outcomes": { + "focused": "success", + "frozen_byte_gate": "success", + "frozen_tests": "success", + "v2_2_cluster": "success" + }, + "passed": true, + "public_arc_evaluation_rerun": false, + "role_boundary": "NO_DISTINCT_QUADRANT_ROLES", + "run_id": "30973734130", + "schema_version": 2, + "v2_2_output": "........................................................................ [ 92%]\n...... [100%]\n78 passed in 1.18s\n", + "workflow_source_commit": "aee774ff11089dfc6d740ed0e78e3c81723e2953" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/QUADRANT_MOSAIC_MODULE_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/QUADRANT_MOSAIC_MODULE_GREEN.json new file mode 100644 index 00000000..f7439293 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/QUADRANT_MOSAIC_MODULE_GREEN.json @@ -0,0 +1,11 @@ +{ + "frozen_byte_gate": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "passed": true, + "public_arc_evaluation_rerun": false, + "run_id": "30973091597", + "schema_version": 1, + "source_commit": "7d006b98bb09ce03f3f8fe0317724af883060173", + "test_count": 4, + "test_output": ".... [100%]\n4 passed in 0.12s\n" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/QUADRANT_MOSAIC_RED.json b/research/universal_core/holonomy_v2_2/evidence/QUADRANT_MOSAIC_RED.json new file mode 100644 index 00000000..3127fcef --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/QUADRANT_MOSAIC_RED.json @@ -0,0 +1,25 @@ +{ + "expected_failure": "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.quadrant_mosaic'", + "expected_new_exact_tasks": [ + "0c786b71", + "3af2c5a8", + "46442a0e", + "62c24649", + "67e8384a", + "7953d61e", + "7fe24cdd", + "833dafe3", + "ed98d772" + ], + "failure_confirmed": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "public_arc_evaluation_rerun": false, + "red_commit": "74335d3309d8ea8f38a1e338f1f20d3fc1ccc858", + "red_run_id": "30972997473", + "schema_version": 1, + "test_command": "python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py", + "training_scan": { + "ambiguous_hidden_outputs": 0, + "wrong_completed_hidden_outputs": 0 + } +} diff --git a/research/universal_core/holonomy_v2_2/evidence/QUADRANT_ROLE_BOUNDARY_RED.json b/research/universal_core/holonomy_v2_2/evidence/QUADRANT_ROLE_BOUNDARY_RED.json new file mode 100644 index 00000000..bb98921d --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/QUADRANT_ROLE_BOUNDARY_RED.json @@ -0,0 +1,13 @@ +{ + "failure_confirmed": true, + "observed_candidate_count": 16, + "observed_failure": "quadrant_mosaic_programs returned repeated-transform programs instead of an empty program set", + "public_arc_evaluation_rerun": false, + "required_boundary": "NO_DISTINCT_QUADRANT_ROLES", + "root_cause": "a transform supported in all four quadrant roles overlaps whole-grid tiling and creates hidden ambiguity when source shape changes", + "run_id": 30973614403, + "schema_version": 1, + "source_commit": "bd04a806be01800eaea0acce866d5b6282b10177", + "test_command": "python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py::test_program_derivation_defers_uniform_quadrant_roles_to_tile", + "test_result": "1 failed; expected programs == (), observed 16 programs" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/RAY_EXTEND_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/RAY_EXTEND_GREEN.json new file mode 100644 index 00000000..ad544b43 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/RAY_EXTEND_GREEN.json @@ -0,0 +1,33 @@ +{ + "expected_cumulative_metrics": { + "accepted": 101, + "correct": 101, + "failed": 0, + "incorrect_attempts": 0, + "rows": 1076 + }, + "expected_new_exact_tasks": [ + "623ea044", + "d037b0a7" + ], + "expected_status_counts": { + "ACCEPTED": 101, + "AMBIGUOUS_PROGRAM": 2, + "GRAMMAR_EXHAUSTED": 973 + }, + "focused_output": "........................................................................ [ 72%]\n........................... [100%]\n99 passed in 5.78s\n", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "frozen_output": "..... [100%]\n5 passed in 0.11s\n", + "outcomes": { + "focused": "success", + "frozen_byte_gate": "success", + "frozen_tests": "success", + "v2_2_cluster": "success" + }, + "passed": true, + "public_arc_evaluation_rerun": false, + "run_id": "31912989436", + "schema_version": 1, + "v2_2_output": "........................................................................ [ 50%]\n...................................................................... [100%]\n142 passed in 5.65s\n", + "workflow_source_commit": "c932df158d31c65958822ac707000193ddbd9b98" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/RAY_EXTEND_MODULE_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/RAY_EXTEND_MODULE_GREEN.json new file mode 100644 index 00000000..aaaae53b --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/RAY_EXTEND_MODULE_GREEN.json @@ -0,0 +1,15 @@ +{ + "expected_new_exact_tasks": [ + "623ea044", + "d037b0a7" + ], + "frozen_byte_gate": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "passed": true, + "public_arc_evaluation_rerun": false, + "run_id": "31912958286", + "schema_version": 1, + "source_commit": "06e3ba8f39a8675679dd3e88b2e42181b72afd13", + "test_count": 10, + "test_output": ".......... [100%]\n10 passed in 0.14s\n" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/RAY_EXTEND_RED.json b/research/universal_core/holonomy_v2_2/evidence/RAY_EXTEND_RED.json new file mode 100644 index 00000000..3e4a1343 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/RAY_EXTEND_RED.json @@ -0,0 +1,27 @@ +{ + "captured_log": "\n==================================== ERRORS ====================================\n_ ERROR collecting research/universal_core/holonomy_v2_2/tests/test_ray_extend.py _\nImportError while importing test module '/home/runner/work/MLE-agent/MLE-agent/research/universal_core/holonomy_v2_2/tests/test_ray_extend.py'.\nHint: make sure your test modules/packages have valid Python names.\nTraceback:\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nresearch/universal_core/holonomy_v2_2/tests/test_ray_extend.py:10: in \n from research.universal_core.holonomy_v2_2.ray_extend import (\nE ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.ray_extend'\n=========================== short test summary info ============================\nERROR research/universal_core/holonomy_v2_2/tests/test_ray_extend.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.18s\n", + "expected_cumulative_metrics": { + "accepted": 101, + "correct": 101, + "failed": 0, + "incorrect_attempts": 0, + "rows": 1076 + }, + "expected_failure": "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.ray_extend'", + "expected_new_exact_tasks": [ + "623ea044", + "d037b0a7" + ], + "expected_status_counts": { + "ACCEPTED": 101, + "AMBIGUOUS_PROGRAM": 2, + "GRAMMAR_EXHAUSTED": 973 + }, + "failure_confirmed": true, + "fixture_integrity": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "public_arc_evaluation_rerun": false, + "run_id": "31912915079", + "schema_version": 1, + "source_commit": "0d0b98843b0d115852a6e4636271672a12069597" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/STRUCTURAL_ENUMERATION_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/STRUCTURAL_ENUMERATION_GREEN.json new file mode 100644 index 00000000..9b41cc4d --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/STRUCTURAL_ENUMERATION_GREEN.json @@ -0,0 +1,15 @@ +{ + "focused_output": "...... [100%]\n6 passed in 0.03s\n", + "frozen_byte_gate": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "frozen_output": "..... [100%]\n5 passed in 0.16s\n", + "green_commit": "438d55f277a41cb10b2cfa1fff3688f9e2cae0af", + "green_run_id": "30948261324", + "interfaces": [ + "panel_programs", + "region_programs", + "component_select_programs" + ], + "schema_version": 1, + "v2_2_output": "....................................... [100%]\n39 passed in 0.21s\n" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/STRUCTURAL_ENUMERATION_RED.json b/research/universal_core/holonomy_v2_2/evidence/STRUCTURAL_ENUMERATION_RED.json new file mode 100644 index 00000000..0c7cf47e --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/STRUCTURAL_ENUMERATION_RED.json @@ -0,0 +1,14 @@ +{ + "captured_log": "FFFFFF [100%]\n=================================== FAILURES ===================================\n_______________ test_panel_programs_expose_hypotheses_before_fit _______________\n\n def test_panel_programs_expose_hypotheses_before_fit() -> None:\n demos = (([[1, 0, 9, 2, 0]], [[4, 0]]),)\n \n> programs, reason = panel_combine.panel_programs(demos)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nE AttributeError: module 'research.universal_core.holonomy_v2_2.panel_combine' has no attribute 'panel_programs'\n\nresearch/universal_core/holonomy_v2_2/tests/test_structural_enumeration.py:13: AttributeError\n_________________ test_panel_programs_report_target_color_skip _________________\n\n def test_panel_programs_report_target_color_skip() -> None:\n> programs, reason = panel_combine.panel_programs((([[1, 0]], [[7, 7]]),))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nE AttributeError: module 'research.universal_core.holonomy_v2_2.panel_combine' has no attribute 'panel_programs'\n\nresearch/universal_core/holonomy_v2_2/tests/test_structural_enumeration.py:21: AttributeError\n______________ test_region_programs_expose_hypotheses_before_fit _______________\n\n def test_region_programs_expose_hypotheses_before_fit() -> None:\n source = [\n [0, 2, 2],\n [2, 0, 2],\n [2, 2, 2],\n ]\n demos = ((source, source),)\n \n> programs, reason = region_fill.region_programs(demos)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^\nE AttributeError: module 'research.universal_core.holonomy_v2_2.region_fill' has no attribute 'region_programs'\n\nresearch/universal_core/holonomy_v2_2/tests/test_structural_enumeration.py:35: AttributeError\n________________ test_region_programs_report_missing_background ________________\n\n def test_region_programs_report_missing_background() -> None:\n demos = (\n ([[1, 1], [1, 2]], [[1, 1], [1, 2]]),\n ([[2, 2], [2, 1]], [[2, 2], [2, 1]]),\n )\n \n> programs, reason = region_fill.region_programs(demos)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^\nE AttributeError: module 'research.universal_core.holonomy_v2_2.region_fill' has no attribute 'region_programs'\n\nresearch/universal_core/holonomy_v2_2/tests/test_structural_enumeration.py:48: AttributeError\n_________ test_component_select_programs_expose_hypotheses_before_fit __________\n\n def test_component_select_programs_expose_hypotheses_before_fit() -> None:\n demos = ((\n [\n [0, 2, 0, 3, 3, 0],\n [0, 2, 0, 3, 3, 0],\n [0, 0, 0, 3, 0, 0],\n ],\n [[3, 3], [3, 3], [3, 0]],\n ),)\n \n> programs, reason = component_select.component_select_programs(demos)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nE AttributeError: module 'research.universal_core.holonomy_v2_2.component_select' has no attribute 'component_select_programs'. Did you mean: 'component_select_candidates'?\n\nresearch/universal_core/holonomy_v2_2/tests/test_structural_enumeration.py:64: AttributeError\n___________ test_component_select_programs_report_missing_background ___________\n\n def test_component_select_programs_report_missing_background() -> None:\n demos = (\n ([[1, 1], [1, 2]], [[1]]),\n ([[2, 2], [2, 1]], [[2]]),\n )\n \n> programs, reason = component_select.component_select_programs(demos)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nE AttributeError: module 'research.universal_core.holonomy_v2_2.component_select' has no attribute 'component_select_programs'. Did you mean: 'component_select_candidates'?\n\nresearch/universal_core/holonomy_v2_2/tests/test_structural_enumeration.py:77: AttributeError\n=========================== short test summary info ============================\nFAILED research/universal_core/holonomy_v2_2/tests/test_structural_enumeration.py::test_panel_programs_expose_hypotheses_before_fit - AttributeError: module 'research.universal_core.holonomy_v2_2.panel_combine' has no attribute 'panel_programs'\nFAILED research/universal_core/holonomy_v2_2/tests/test_structural_enumeration.py::test_panel_programs_report_target_color_skip - AttributeError: module 'research.universal_core.holonomy_v2_2.panel_combine' has no attribute 'panel_programs'\nFAILED research/universal_core/holonomy_v2_2/tests/test_structural_enumeration.py::test_region_programs_expose_hypotheses_before_fit - AttributeError: module 'research.universal_core.holonomy_v2_2.region_fill' has no attribute 'region_programs'\nFAILED research/universal_core/holonomy_v2_2/tests/test_structural_enumeration.py::test_region_programs_report_missing_background - AttributeError: module 'research.universal_core.holonomy_v2_2.region_fill' has no attribute 'region_programs'\nFAILED research/universal_core/holonomy_v2_2/tests/test_structural_enumeration.py::test_component_select_programs_expose_hypotheses_before_fit - AttributeError: module 'research.universal_core.holonomy_v2_2.component_select' has no attribute 'component_select_programs'. Did you mean: 'component_select_candidates'?\nFAILED research/universal_core/holonomy_v2_2/tests/test_structural_enumeration.py::test_component_select_programs_report_missing_background - AttributeError: module 'research.universal_core.holonomy_v2_2.component_select' has no attribute 'component_select_programs'. Did you mean: 'component_select_candidates'?\n6 failed in 0.07s\n", + "expected_missing_interfaces": [ + "panel_programs", + "region_programs", + "component_select_programs" + ], + "failure_confirmed": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "red_commit": "f735346a50d3f93dc07f1e278df4094fb2ac7d3f", + "red_run_id": "30947791255", + "schema_version": 1, + "test_command": "python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_structural_enumeration.py" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/SYMMETRY_COMPLETE_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/SYMMETRY_COMPLETE_GREEN.json new file mode 100644 index 00000000..7fb0e7f1 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/SYMMETRY_COMPLETE_GREEN.json @@ -0,0 +1,37 @@ +{ + "expected_cumulative_metrics": { + "accepted": 99, + "correct": 99, + "failed": 0, + "incorrect_attempts": 0, + "rows": 1076 + }, + "expected_new_ambiguous_tasks": [ + "b8825c91" + ], + "expected_new_exact_tasks": [ + "496994bd", + "e729b7be", + "f25ffba3" + ], + "expected_status_counts": { + "ACCEPTED": 99, + "AMBIGUOUS_PROGRAM": 2, + "GRAMMAR_EXHAUSTED": 975 + }, + "focused_output": "........................................................................ [ 81%]\n................ [100%]\n88 passed in 5.39s\n", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "frozen_output": "..... [100%]\n5 passed in 0.08s\n", + "outcomes": { + "focused": "success", + "frozen_byte_gate": "success", + "frozen_tests": "success", + "v2_2_cluster": "success" + }, + "passed": true, + "public_arc_evaluation_rerun": false, + "run_id": "31912341617", + "schema_version": 1, + "v2_2_output": "........................................................................ [ 54%]\n........................................................... [100%]\n131 passed in 5.39s\n", + "workflow_source_commit": "13786767478a5d08b15371fda6d42025cc523026" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/SYMMETRY_COMPLETE_MODULE_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/SYMMETRY_COMPLETE_MODULE_GREEN.json new file mode 100644 index 00000000..301b7fc1 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/SYMMETRY_COMPLETE_MODULE_GREEN.json @@ -0,0 +1,19 @@ +{ + "expected_new_ambiguous_tasks": [ + "b8825c91" + ], + "expected_new_exact_tasks": [ + "496994bd", + "e729b7be", + "f25ffba3" + ], + "frozen_byte_gate": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "passed": true, + "public_arc_evaluation_rerun": false, + "run_id": "31912302330", + "schema_version": 1, + "source_commit": "8be28f0ef564f6472dcae0565ec765581dc70912", + "test_count": 10, + "test_output": ".......... [100%]\n10 passed in 0.11s\n" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/SYMMETRY_COMPLETE_RED.json b/research/universal_core/holonomy_v2_2/evidence/SYMMETRY_COMPLETE_RED.json new file mode 100644 index 00000000..0fe4bbca --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/SYMMETRY_COMPLETE_RED.json @@ -0,0 +1,31 @@ +{ + "captured_log": "\n==================================== ERRORS ====================================\n_ ERROR collecting research/universal_core/holonomy_v2_2/tests/test_symmetry_complete.py _\nImportError while importing test module '/home/runner/work/MLE-agent/MLE-agent/research/universal_core/holonomy_v2_2/tests/test_symmetry_complete.py'.\nHint: make sure your test modules/packages have valid Python names.\nTraceback:\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nresearch/universal_core/holonomy_v2_2/tests/test_symmetry_complete.py:10: in \n from research.universal_core.holonomy_v2_2.symmetry_complete import (\nE ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.symmetry_complete'\n=========================== short test summary info ============================\nERROR research/universal_core/holonomy_v2_2/tests/test_symmetry_complete.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.24s\n", + "expected_cumulative_metrics": { + "accepted": 99, + "correct": 99, + "failed": 0, + "incorrect_attempts": 0, + "rows": 1076 + }, + "expected_failure": "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.symmetry_complete'", + "expected_new_ambiguous_tasks": [ + "b8825c91" + ], + "expected_new_exact_tasks": [ + "496994bd", + "e729b7be", + "f25ffba3" + ], + "expected_status_counts": { + "ACCEPTED": 99, + "AMBIGUOUS_PROGRAM": 2, + "GRAMMAR_EXHAUSTED": 975 + }, + "failure_confirmed": true, + "fixture_integrity": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "public_arc_evaluation_rerun": false, + "run_id": "31912259997", + "schema_version": 1, + "source_commit": "5518a27cedc93bebee0fdd656225fa8bbc5ec88d" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/TRAINING_COMPONENT_RANK_0002.json b/research/universal_core/holonomy_v2_2/evidence/TRAINING_COMPONENT_RANK_0002.json new file mode 100644 index 00000000..86434898 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/TRAINING_COMPONENT_RANK_0002.json @@ -0,0 +1,49 @@ +{ + "baseline": { + "artifact_id": "8903788542", + "candidate_commit": "881e1263190f471ca079f686b3be922dbfe94fdf", + "correct": 16, + "coverage": 0.01486988847583643, + "execution_failed": 1, + "grammar_exhausted": 1059, + "incorrect_attempts": 0, + "report_digest": "1ec17044269cbcf94d6d43002165873fc67828cf752d3f8386001af94d4b1df9", + "rows": 1076, + "run_id": "30937670014" + }, + "component_rank_pass": { + "artifact_digest": "sha256:477a7cf4df1cd1b39bfa009c252c9c69fbc0e780ccc2328f34e0caa21cb5c12f", + "artifact_id": "8904087790", + "candidate_commit": "95c8160dd9a88c4bcd66063ab88c4701c93cf502", + "correct": 26, + "coverage": 0.024163568773234202, + "execution_failed": 2, + "grammar_exhausted": 1048, + "incorrect_attempts": 0, + "new_exact_tasks": [ + "08ed6ac7", + "62ab2642", + "6455b5f5", + "67385a82", + "6e82a1ae", + "a61f2674", + "aedd82e4", + "e0fb7511", + "e8593010", + "ea32f347" + ], + "report_digest": "304dcd2f18f74a83af07adb49b0b7e2abc306014fb6ef3ff3a6bb9707301501e", + "rows": 1076, + "run_id": "30938418028" + }, + "delta": { + "correct": 10, + "coverage": 0.009293680297397772, + "execution_failed": 1, + "incorrect_attempts": 0, + "lost_exact_tasks": 0 + }, + "development_split": "ARC-AGI-2 training only", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "schema_version": 1 +} diff --git a/research/universal_core/holonomy_v2_2/evidence/TRAINING_MECHANISTIC_GREEN.json b/research/universal_core/holonomy_v2_2/evidence/TRAINING_MECHANISTIC_GREEN.json new file mode 100644 index 00000000..38be9a58 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/TRAINING_MECHANISTIC_GREEN.json @@ -0,0 +1,16 @@ +{ + "focused_output": "........ [100%]\n8 passed in 0.15s\n", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "frozen_output": "..... [100%]\n5 passed in 0.08s\n", + "green_commit": "272f2319893d8fc2e1bbcb5cd0dbff2385bfde97", + "outcomes": { + "focused": "success", + "frozen_byte_gate": "success", + "frozen_tests": "success", + "v2_2_cluster": "success" + }, + "passed": true, + "run_id": "30949358599", + "schema_version": 1, + "v2_2_output": "................................................... [100%]\n51 passed in 0.40s\n" +} diff --git a/research/universal_core/holonomy_v2_2/evidence/TRAINING_MECHANISTIC_RED.json b/research/universal_core/holonomy_v2_2/evidence/TRAINING_MECHANISTIC_RED.json new file mode 100644 index 00000000..c1433d7f --- /dev/null +++ b/research/universal_core/holonomy_v2_2/evidence/TRAINING_MECHANISTIC_RED.json @@ -0,0 +1,10 @@ +{ + "captured_log": "\n==================================== ERRORS ====================================\n_ ERROR collecting research/universal_core/holonomy_v2_2/tests/test_trace_diagnosis.py _\nImportError while importing test module '/home/runner/work/MLE-agent/MLE-agent/research/universal_core/holonomy_v2_2/tests/test_trace_diagnosis.py'.\nHint: make sure your test modules/packages have valid Python names.\nTraceback:\n/opt/hostedtoolcache/Python/3.12.13/x64/lib/python3.12/importlib/__init__.py:90: in import_module\n return _bootstrap._gcd_import(name[level:], package, level)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nresearch/universal_core/holonomy_v2_2/tests/test_trace_diagnosis.py:4: in \n from research.universal_core.holonomy_v2_2.trace_diagnosis import build_trace_diagnosis\nE ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.trace_diagnosis'\n=========================== short test summary info ============================\nERROR research/universal_core/holonomy_v2_2/tests/test_trace_diagnosis.py\n!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!\n1 error in 0.19s\n", + "expected_failure": "ModuleNotFoundError: No module named 'research.universal_core.holonomy_v2_2.trace_diagnosis'", + "failure_confirmed": true, + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + "red_commit": "83b264802da77bc772b0e3fd5d10b6e2eadbad6c", + "red_run_id": "30949172439", + "schema_version": 1, + "test_command": "python -m pytest -q research/universal_core/holonomy_v2_2/tests/test_trace_diagnosis.py research/universal_core/holonomy_v2_2/tests/test_training_mechanistic.py" +} diff --git a/research/universal_core/holonomy_v2_2/gravity_compact.py b/research/universal_core/holonomy_v2_2/gravity_compact.py new file mode 100644 index 00000000..b5d65ef7 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/gravity_compact.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from typing import Any, Mapping, Sequence + + +Grid = list[list[int]] +Program = dict[str, Any] +_DIRECTION_ORDER = ("up", "down", "left", "right") + + +def _shape(grid: Grid) -> tuple[int, int]: + if not grid or not grid[0]: + raise ValueError("gravity compaction requires a nonempty grid") + width = len(grid[0]) + if any(len(row) != width for row in grid): + raise ValueError("gravity compaction requires a rectangular grid") + return len(grid), width + + +def apply_gravity_compact(program: Mapping[str, Any], grid: Grid) -> Grid: + height, width = _shape(grid) + background = int(program["background"]) + direction = str(program["direction"]) + if direction not in _DIRECTION_ORDER: + raise ValueError("unknown gravity direction") + + if direction in ("left", "right"): + output: Grid = [] + for source_row in grid: + values = [cell for cell in source_row if cell != background] + padding = [background] * (width - len(values)) + output.append(values + padding if direction == "left" else padding + values) + return output + + output = [[background for _ in range(width)] for _ in range(height)] + for col in range(width): + values = [grid[row][col] for row in range(height) if grid[row][col] != background] + start = 0 if direction == "up" else height - len(values) + for offset, value in enumerate(values): + output[start + offset][col] = value + return output + + +def gravity_compact_programs( + demos: Sequence[tuple[Grid, Grid]], +) -> tuple[tuple[Program, ...], str | None]: + if not demos: + return (), "NO_DEMONSTRATION" + + for source, target in demos: + if _shape(source) != _shape(target): + return (), "NO_SHAPE_PRESERVING_GRAVITY" + + common_backgrounds = {cell for row in demos[0][0] for cell in row} + for source, _ in demos[1:]: + common_backgrounds.intersection_update(cell for row in source for cell in row) + + programs: list[Program] = [] + for background in sorted(common_backgrounds): + for direction in _DIRECTION_ORDER: + program = { + "kind": "axis_gravity_compact", + "background": background, + "direction": direction, + } + predictions = [apply_gravity_compact(program, source) for source, _ in demos] + if not any(prediction != source for prediction, (source, _) in zip(predictions, demos)): + continue + if all(prediction == target for prediction, (_, target) in zip(predictions, demos)): + programs.append(program) + + if not programs: + return (), "NO_GRAVITY_COMPACTION" + return tuple(programs), None diff --git a/research/universal_core/holonomy_v2_2/grid_induction.py b/research/universal_core/holonomy_v2_2/grid_induction.py new file mode 100644 index 00000000..aa6b1b1b --- /dev/null +++ b/research/universal_core/holonomy_v2_2/grid_induction.py @@ -0,0 +1,408 @@ +from __future__ import annotations + +import hashlib +import json +from copy import deepcopy +from typing import Any, Mapping, Sequence + +from research.universal_core.holonomy_v2.blind_adapter import run_public_task_v2 + +from .panel_combine import apply_panel, panel_candidates +from .region_fill import apply_region, region_candidates +from .component_select import apply_component_select, component_select_candidates +from .marker_recolor import apply_marker_recolor +from .component_outlier import apply_component_outlier +from .bbox_complete import apply_bbox_complete +from .mirror_concat import apply_mirror_concat +from .quadrant_mosaic import apply_quadrant_mosaic +from .palette_repeat import apply_palette_repeat +from .axis_collapse import apply_axis_collapse +from .block_reduce import apply_block_reduce +from .gravity_compact import apply_gravity_compact +from .line_connect import apply_line_connect +from .symmetry_complete import apply_symmetry_complete +from .ray_extend import apply_ray_extend + + +Grid = list[list[int]] +Program = dict[str, Any] +Point = tuple[int, int] +_VERSION = "grid-v2.2-17" + + +def _canonical(value: Any) -> bytes: + return ( + json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + "\n" + ).encode("utf-8") + + +def _digest(value: Any) -> str: + return hashlib.sha256(_canonical(value)).hexdigest() + + +def _get(value: Any, key: str) -> Any: + if isinstance(value, Mapping): + return value[key] + return getattr(value, key) + + +def _grid(value: Any) -> Grid: + if not isinstance(value, (list, tuple)) or not 1 <= len(value) <= 30: + raise ValueError("grid height must be between 1 and 30") + width: int | None = None + copied: Grid = [] + for row in value: + if not isinstance(row, (list, tuple)) or not 1 <= len(row) <= 30: + raise ValueError("grid width must be between 1 and 30") + if width is None: + width = len(row) + elif len(row) != width: + raise ValueError("grid must be rectangular") + out_row: list[int] = [] + for cell in row: + if type(cell) is not int or not 0 <= cell <= 9: + raise ValueError("grid cells must be integers from 0 through 9") + out_row.append(cell) + copied.append(out_row) + return copied + + +def _demo_pair(item: Any) -> tuple[Grid, Grid]: + return _grid(_get(item, "input")), _grid(_get(item, "output")) + + +def _is_grid_task(task: Any) -> bool: + try: + demos = tuple(_get(task, "demonstrations")) + if not demos: + return False + for item in demos: + _demo_pair(item) + return True + except (KeyError, TypeError, ValueError): + return False + + +def _rotate(grid: Grid, turns: int) -> Grid: + current = deepcopy(grid) + for _ in range(turns % 4): + current = [list(row) for row in zip(*reversed(current))] + return current + + +def _reflect(grid: Grid, axis: str) -> Grid: + if axis == "columns": + return [list(reversed(row)) for row in grid] + if axis == "rows": + return [list(row) for row in reversed(grid)] + if axis == "main": + return [list(row) for row in zip(*grid)] + if axis == "anti": + return _rotate([list(row) for row in zip(*grid)], 2) + raise ValueError("unknown reflection axis") + + +def _scale(grid: Grid, row_factor: int, col_factor: int) -> Grid: + if row_factor < 1 or col_factor < 1: + raise ValueError("scale factors must be positive") + output: Grid = [] + for row in grid: + expanded = [cell for cell in row for _ in range(col_factor)] + for _ in range(row_factor): + output.append(list(expanded)) + if len(output) > 30 or len(output[0]) > 30: + raise ValueError("scaled grid exceeds ARC bounds") + return output + + +def _tile(grid: Grid, row_repeats: int, col_repeats: int) -> Grid: + if row_repeats < 1 or col_repeats < 1: + raise ValueError("tile repeats must be positive") + output = [row * col_repeats for _ in range(row_repeats) for row in grid] + if len(output) > 30 or len(output[0]) > 30: + raise ValueError("tiled grid exceeds ARC bounds") + return output + + +def _crop(grid: Grid, background: int) -> Grid: + cells = [ + (row_index, col_index) + for row_index, row in enumerate(grid) + for col_index, cell in enumerate(row) + if cell != background + ] + if not cells: + raise ValueError("crop requires a foreground cell") + top = min(item[0] for item in cells) + bottom = max(item[0] for item in cells) + left = min(item[1] for item in cells) + right = max(item[1] for item in cells) + return [row[left : right + 1] for row in grid[top : bottom + 1]] + + +def _components(grid: Grid, background: int) -> tuple[tuple[Point, ...], ...]: + height = len(grid) + width = len(grid[0]) + seen: set[Point] = set() + found: list[tuple[Point, ...]] = [] + for row in range(height): + for col in range(width): + if (row, col) in seen or grid[row][col] == background: + continue + color = grid[row][col] + pending = [(row, col)] + seen.add((row, col)) + cells: list[Point] = [] + while pending: + current = pending.pop() + cells.append(current) + for row_step, col_step in ((1, 0), (-1, 0), (0, 1), (0, -1)): + next_row = current[0] + row_step + next_col = current[1] + col_step + next_point = (next_row, next_col) + if not (0 <= next_row < height and 0 <= next_col < width): + continue + if next_point in seen or grid[next_row][next_col] != color: + continue + seen.add(next_point) + pending.append(next_point) + found.append(tuple(sorted(cells))) + return tuple(found) + + +def _rank_key(components: Sequence[Sequence[Point]], cells: Sequence[Point], mode: str) -> str: + sizes = sorted({len(item) for item in components}) + size = len(cells) + if mode == "rank": + return str(sizes.index(size)) + if mode == "extreme": + if len(sizes) == 1: + return "only" + if size == sizes[0]: + return "min" + if size == sizes[-1]: + return "max" + return "mid" + raise ValueError("unknown component rank mode") + + +def _apply_component_rank(program: Program, grid: Grid) -> Grid: + background = int(program["background"]) + mode = str(program["mode"]) + mapping = {str(key): int(value) for key, value in program["mapping"].items()} + components = _components(grid, background) + output = deepcopy(grid) + for cells in components: + key = _rank_key(components, cells, mode) + if key not in mapping: + raise KeyError("unmapped component rank") + for row, col in cells: + output[row][col] = mapping[key] + return output + + +def _apply(program: Program, value: Any) -> Grid: + grid = _grid(value) + kind = program["kind"] + if kind == "identity": + return grid + if kind == "rotate": + return _rotate(grid, int(program["turns"])) + if kind == "reflect": + return _reflect(grid, str(program["axis"])) + if kind == "scale": + return _scale(grid, int(program["rows"]), int(program["cols"])) + if kind == "tile": + return _tile(grid, int(program["rows"]), int(program["cols"])) + if kind == "crop": + return _crop(grid, int(program["background"])) + if kind == "component_rank_color": + return _apply_component_rank(program, grid) + if kind == "panel_boolean_combine": + return apply_panel(program, grid) + if kind == "enclosed_region_fill": + return apply_region(program, grid) + if kind == "component_area_select": + return apply_component_select(program, grid) + if kind == "anchor_ray_extend": + return apply_ray_extend(program, grid) + if kind == "reflective_symmetry_complete": + return apply_symmetry_complete(program, grid) + if kind == "same_color_line_connect": + return apply_line_connect(program, grid) + if kind == "axis_gravity_compact": + return apply_gravity_compact(program, grid) + if kind == "fixed_block_reduce": + return apply_block_reduce(program, grid) + if kind == "axis_duplicate_collapse": + return apply_axis_collapse(program, grid) + if kind == "palette_cardinality_repeat": + return apply_palette_repeat(program, grid) + if kind == "dihedral_quadrant_mosaic": + return apply_quadrant_mosaic(program, grid) + if kind == "mirror_concatenate": + return apply_mirror_concat(program, grid) + if kind == "bounding_box_complete": + return apply_bbox_complete(program, grid) + if kind == "component_area_outlier_color": + return apply_component_outlier(program, grid) + if kind == "marker_recolor": + return apply_marker_recolor(program, grid) + if kind == "color_map": + mapping = {int(key): int(item) for key, item in program["mapping"].items()} + if any(cell not in mapping for row in grid for cell in row): + raise KeyError("unmapped grid cell") + return [[mapping[cell] for cell in row] for row in grid] + raise ValueError("unknown grid program") + + +def _fits(program: Program, demos: Sequence[tuple[Grid, Grid]]) -> bool: + try: + return all(_apply(program, source) == target for source, target in demos) + except (KeyError, TypeError, ValueError): + return False + + +def _color_map(demos: Sequence[tuple[Grid, Grid]]) -> Program | None: + mapping: dict[int, int] = {} + for source, target in demos: + if len(source) != len(target) or len(source[0]) != len(target[0]): + return None + for source_row, target_row in zip(source, target): + for before, after in zip(source_row, target_row): + if before in mapping and mapping[before] != after: + return None + mapping[before] = after + return {"kind": "color_map", "mapping": {str(key): mapping[key] for key in sorted(mapping)}} + + +def _component_rank_program( + demos: Sequence[tuple[Grid, Grid]], + background: int, + mode: str, +) -> Program | None: + mapping: dict[str, int] = {} + for source, target in demos: + if len(source) != len(target) or len(source[0]) != len(target[0]): + return None + components = _components(source, background) + if not components: + return None + covered = {point for cells in components for point in cells} + for cells in components: + target_colors = {target[row][col] for row, col in cells} + if len(target_colors) != 1: + return None + key = _rank_key(components, cells, mode) + target_color = next(iter(target_colors)) + if key in mapping and mapping[key] != target_color: + return None + mapping[key] = target_color + for row, source_row in enumerate(source): + for col, source_color in enumerate(source_row): + if (row, col) not in covered and target[row][col] != source_color: + return None + return { + "kind": "component_rank_color", + "background": background, + "mode": mode, + "mapping": {key: mapping[key] for key in sorted(mapping)}, + } + + +def _dimension_program( + demos: Sequence[tuple[Grid, Grid]], + kind: str, +) -> Program | None: + first_source, first_target = demos[0] + if len(first_target) % len(first_source) or len(first_target[0]) % len(first_source[0]): + return None + rows = len(first_target) // len(first_source) + cols = len(first_target[0]) // len(first_source[0]) + if rows < 1 or cols < 1: + return None + data = {"kind": kind, "rows": rows, "cols": cols} + return data if _fits(data, demos) else None + + +def _candidate_programs(demos: Sequence[tuple[Grid, Grid]]) -> tuple[Program, ...]: + candidates: list[Program] = [ + {"kind": "identity"}, + {"kind": "rotate", "turns": 1}, + {"kind": "rotate", "turns": 2}, + {"kind": "rotate", "turns": 3}, + {"kind": "reflect", "axis": "columns"}, + {"kind": "reflect", "axis": "rows"}, + {"kind": "reflect", "axis": "main"}, + {"kind": "reflect", "axis": "anti"}, + ] + mapped = _color_map(demos) + if mapped is not None: + candidates.append(mapped) + for kind in ("scale", "tile"): + data = _dimension_program(demos, kind) + if data is not None: + candidates.append(data) + backgrounds = sorted({cell for source, _ in demos for row in source for cell in row}) + candidates.extend({"kind": "crop", "background": item} for item in backgrounds) + common_backgrounds = set(cell for row in demos[0][0] for cell in row) + for source, _ in demos[1:]: + common_backgrounds.intersection_update(cell for row in source for cell in row) + for background in sorted(common_backgrounds): + for mode in ("rank", "extreme"): + data = _component_rank_program(demos, background, mode) + if data is not None: + candidates.append(data) + + candidates.extend(panel_candidates(demos)) + candidates.extend(region_candidates(demos)) + candidates.extend(component_select_candidates(demos)) + + by_digest: dict[str, Program] = {} + for candidate in candidates: + if _fits(candidate, demos): + by_digest.setdefault(_digest(candidate), candidate) + return tuple(by_digest[key] for key in sorted(by_digest)) + + +def _evidence(candidates: Sequence[Program], batch_count: int) -> dict[str, Any]: + return { + "version": _VERSION, + "candidate_count": len(candidates), + "prediction_batch_count": batch_count, + "candidate_kinds": sorted({str(item["kind"]) for item in candidates}), + } + + +def _abstain(task_id: str, hidden_count: int, status: str, evidence: Mapping[str, Any]) -> dict[str, Any]: + return { + "task_id": task_id, + "tier": "V2.2-GRID", + "program_digest": None, + "freeze_digest": None, + "prediction_digest": None, + "evidence": dict(evidence), + "predictions": [ + {"status": status, "prediction": None} + for _ in range(hidden_count) + ], + } + +def run_public_task_v2_2( + task: Any, + engine: Any, + *, + mechanistic: bool = False, +) -> dict[str, Any]: + if not _is_grid_task(task): + result = run_public_task_v2(task, engine) + if not mechanistic: + return result + from .grid_mechanistic_runtime import attach_delegate_trace + + return attach_delegate_trace(task, result) + + from .grid_mechanistic_runtime import run_grid_task + + return run_grid_task(task, mechanistic=mechanistic) diff --git a/research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py b/research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py new file mode 100644 index 00000000..219f919e --- /dev/null +++ b/research/universal_core/holonomy_v2_2/grid_mechanistic_runtime.py @@ -0,0 +1,463 @@ +from __future__ import annotations + +from copy import deepcopy +from typing import Any, Mapping, Sequence + +from . import grid_induction as gi +from .component_select import component_select_programs +from .mechanistic_trace import TraceRecorder +from .marker_recolor import marker_recolor_programs +from .component_outlier import component_outlier_programs +from .bbox_complete import bbox_complete_programs +from .mirror_concat import mirror_concat_programs +from .quadrant_mosaic import quadrant_mosaic_programs +from .palette_repeat import palette_repeat_programs +from .axis_collapse import axis_collapse_programs +from .block_reduce import block_reduce_programs +from .gravity_compact import gravity_compact_programs +from .line_connect import line_connect_programs +from .symmetry_complete import symmetry_complete_programs +from .ray_extend import ray_extend_programs +from .panel_combine import panel_programs +from .region_fill import region_programs + + +Grid = list[list[int]] +Program = dict[str, Any] +Demo = tuple[Grid, Grid] +Proposal = dict[str, Any] + +_FROZEN_CORE_COMMIT = "1d7c489bcdbff755d638b35ad47ce62bd4fe829d" + + +def _emit(recorder: TraceRecorder | None, event: str, **data: Any) -> None: + if recorder is not None: + recorder.emit(event, **data) + + +def _store(recorder: TraceRecorder | None, value: Any) -> str | None: + return recorder.store(value) if recorder is not None else None + + +def _shape(grid: Grid) -> list[int]: + return [len(grid), len(grid[0])] + + +def _difference_coordinates(output: Grid, target: Grid) -> list[list[int]] | None: + if _shape(output) != _shape(target): + return None + return [ + [row, col] + for row in range(len(target)) + for col in range(len(target[0])) + if output[row][col] != target[row][col] + ] + + +def _dimension_program(demos: Sequence[Demo], kind: str) -> tuple[Program, ...]: + first_source, first_target = demos[0] + if len(first_target) % len(first_source) or len(first_target[0]) % len(first_source[0]): + return () + rows = len(first_target) // len(first_source) + cols = len(first_target[0]) // len(first_source[0]) + if rows < 1 or cols < 1: + return () + return ({"kind": kind, "rows": rows, "cols": cols},) + + +def _component_rank_programs(demos: Sequence[Demo]) -> tuple[Program, ...]: + common_backgrounds = set(cell for row in demos[0][0] for cell in row) + for source, _ in demos[1:]: + common_backgrounds.intersection_update(cell for row in source for cell in row) + programs: list[Program] = [] + for background in sorted(common_backgrounds): + for mode in ("rank", "extreme"): + program = gi._component_rank_program(demos, background, mode) + if program is not None: + programs.append(program) + return tuple(programs) + + +def _generator_batches(demos: Sequence[Demo]) -> tuple[tuple[str, tuple[Program, ...], str | None], ...]: + fixed: tuple[Program, ...] = ( + {"kind": "identity"}, + {"kind": "rotate", "turns": 1}, + {"kind": "rotate", "turns": 2}, + {"kind": "rotate", "turns": 3}, + {"kind": "reflect", "axis": "columns"}, + {"kind": "reflect", "axis": "rows"}, + {"kind": "reflect", "axis": "main"}, + {"kind": "reflect", "axis": "anti"}, + ) + mapped = gi._color_map(demos) + color_programs = (mapped,) if mapped is not None else () + crop_programs = tuple( + {"kind": "crop", "background": background} + for background in sorted({cell for source, _ in demos for row in source for cell in row}) + ) + panel, panel_reason = panel_programs(demos) + region, region_reason = region_programs(demos) + component_select, component_reason = component_select_programs(demos) + marker_recolor, marker_reason = marker_recolor_programs(demos) + component_outlier, outlier_reason = component_outlier_programs(demos) + bbox_complete, bbox_reason = bbox_complete_programs(demos) + mirror_concat, mirror_reason = mirror_concat_programs(demos) + quadrant_mosaic, quadrant_reason = quadrant_mosaic_programs(demos) + palette_repeat, palette_reason = palette_repeat_programs(demos) + axis_collapse, axis_reason = axis_collapse_programs(demos) + block_reduce, block_reason = block_reduce_programs(demos) + gravity_compact, gravity_reason = gravity_compact_programs(demos) + line_connect, line_reason = line_connect_programs(demos) + symmetry_complete, symmetry_reason = symmetry_complete_programs(demos) + ray_extend, ray_reason = ray_extend_programs(demos) + return ( + ("fixed", fixed, None), + ("color_map", color_programs, None if color_programs else "NO_CONSISTENT_COLOR_MAP"), + ("scale", _dimension_program(demos, "scale"), "NO_INTEGER_DIMENSION_RATIO"), + ("tile", _dimension_program(demos, "tile"), "NO_INTEGER_DIMENSION_RATIO"), + ("crop", crop_programs, None if crop_programs else "NO_SOURCE_COLOR"), + ( + "component_rank", + _component_rank_programs(demos), + None if _component_rank_programs(demos) else "NO_DERIVED_COMPONENT_RANK_PROGRAM", + ), + ("marker_recolor", marker_recolor, marker_reason), + ("component_outlier", component_outlier, outlier_reason), + ("bbox_complete", bbox_complete, bbox_reason), + ("mirror_concat", mirror_concat, mirror_reason), + ("quadrant_mosaic", quadrant_mosaic, quadrant_reason), + ("palette_repeat", palette_repeat, palette_reason), + ("axis_collapse", axis_collapse, axis_reason), + ("block_reduce", block_reduce, block_reason), + ("gravity_compact", gravity_compact, gravity_reason), + ("line_connect", line_connect, line_reason), + ("symmetry_complete", symmetry_complete, symmetry_reason), + ("ray_extend", ray_extend, ray_reason), + ("panel", panel, panel_reason), + ("region", region, region_reason), + ("component_select", component_select, component_reason), + ) + + +def _proposals(demos: Sequence[Demo], recorder: TraceRecorder | None) -> tuple[Proposal, ...]: + proposals: list[Proposal] = [] + for generator, programs, reason in _generator_batches(demos): + _emit(recorder, "generator_start", generator=generator) + if not programs: + _emit(recorder, "generator_skip", generator=generator, reason=reason or "NO_PROGRAM") + continue + for ordinal, program in enumerate(programs): + candidate_id = f"{generator}:{ordinal:04d}" + program_digest = gi._digest(program) + program_ref = _store(recorder, program) + if program_ref is not None and program_ref != program_digest: + raise ValueError("program object digest mismatch") + proposal: Proposal = { + "candidate_id": candidate_id, + "generator": generator, + "program": program, + "program_digest": program_digest, + } + proposals.append(proposal) + _emit( + recorder, + "candidate_proposed", + candidate_id=candidate_id, + generator=generator, + generator_ordinal=ordinal, + candidate_kind=str(program["kind"]), + program_digest=program_digest, + program_ref=program_ref, + ) + _emit(recorder, "generator_end", generator=generator, candidate_count=len(programs)) + return tuple(proposals) + + +def _fit_proposals( + proposals: Sequence[Proposal], + demos: Sequence[Demo], + recorder: TraceRecorder | None, +) -> tuple[Proposal, ...]: + fitting: list[Proposal] = [] + for proposal in proposals: + execution_failed = False + mismatch = False + for demo_index, (source, target) in enumerate(demos): + source_ref = _store(recorder, source) + target_ref = _store(recorder, target) + try: + output = gi._apply(proposal["program"], source) + except (KeyError, TypeError, ValueError) as exc: + execution_failed = True + _emit( + recorder, + "demo_execution", + candidate_id=proposal["candidate_id"], + candidate_kind=str(proposal["program"]["kind"]), + demo_index=demo_index, + outcome="FAILED", + source_ref=source_ref, + target_ref=target_ref, + exception_type=type(exc).__name__, + exception_message=str(exc), + ) + continue + matches = output == target + mismatch = mismatch or not matches + _emit( + recorder, + "demo_execution", + candidate_id=proposal["candidate_id"], + candidate_kind=str(proposal["program"]["kind"]), + demo_index=demo_index, + outcome="MATCH" if matches else "MISMATCH", + source_ref=source_ref, + target_ref=target_ref, + output_ref=_store(recorder, output), + source_shape=_shape(source), + target_shape=_shape(target), + output_shape=_shape(output), + difference_coordinates=[] if matches else _difference_coordinates(output, target), + ) + if execution_failed: + decision = "DEMO_EXECUTION_FAILED" + elif mismatch: + decision = "DEMO_MISMATCH" + else: + decision = "FITS_ALL_DEMOS" + fitting.append(proposal) + _emit( + recorder, + "candidate_fit_decision", + candidate_id=proposal["candidate_id"], + candidate_kind=str(proposal["program"]["kind"]), + decision=decision, + ) + return tuple(fitting) + + +def _deduplicate( + fitting: Sequence[Proposal], + recorder: TraceRecorder | None, +) -> tuple[Proposal, ...]: + by_digest: dict[str, Proposal] = {} + for proposal in fitting: + digest = str(proposal["program_digest"]) + if digest in by_digest: + _emit( + recorder, + "candidate_duplicate", + candidate_id=proposal["candidate_id"], + representative_id=by_digest[digest]["candidate_id"], + program_digest=digest, + ) + else: + by_digest[digest] = proposal + return tuple(by_digest[digest] for digest in sorted(by_digest)) + + +def _execute_hidden( + representatives: Sequence[Proposal], + hidden: Sequence[Grid], + recorder: TraceRecorder | None, +) -> dict[bytes, tuple[tuple[Grid, ...], list[Proposal], str]]: + batches: dict[bytes, tuple[tuple[Grid, ...], list[Proposal], str]] = {} + for proposal in representatives: + outputs: list[Grid] = [] + failed = False + for hidden_index, value in enumerate(hidden): + input_ref = _store(recorder, value) + try: + output = gi._apply(proposal["program"], value) + except (KeyError, TypeError, ValueError) as exc: + failed = True + _emit( + recorder, + "hidden_execution", + candidate_id=proposal["candidate_id"], + candidate_kind=str(proposal["program"]["kind"]), + hidden_index=hidden_index, + outcome="FAILED", + input_ref=input_ref, + exception_type=type(exc).__name__, + exception_message=str(exc), + ) + continue + outputs.append(output) + _emit( + recorder, + "hidden_execution", + candidate_id=proposal["candidate_id"], + candidate_kind=str(proposal["program"]["kind"]), + hidden_index=hidden_index, + outcome="SUCCEEDED", + input_ref=input_ref, + output_ref=_store(recorder, output), + output_shape=_shape(output), + ) + if failed: + _emit( + recorder, + "hidden_batch_decision", + candidate_id=proposal["candidate_id"], + candidate_kind=str(proposal["program"]["kind"]), + decision="FAILED", + ) + continue + output_batch = tuple(outputs) + key = gi._canonical(output_batch) + batch_ref = _store(recorder, output_batch) or gi._digest(output_batch) + _emit( + recorder, + "hidden_batch_decision", + candidate_id=proposal["candidate_id"], + candidate_kind=str(proposal["program"]["kind"]), + decision="COMPLETED", + batch_ref=batch_ref, + ) + if key not in batches: + batches[key] = (output_batch, [], batch_ref) + batches[key][1].append(proposal) + + for _, (_, members, batch_ref) in sorted(batches.items(), key=lambda item: item[0]): + _emit( + recorder, + "equivalence_class", + batch_ref=batch_ref, + candidate_ids=[item["candidate_id"] for item in members], + candidate_kinds=sorted({str(item["program"]["kind"]) for item in members}), + ) + return batches + + +def _finish( + result: dict[str, Any], + status: str, + recorder: TraceRecorder | None, + candidate_ids: Sequence[str], + **data: Any, +) -> dict[str, Any]: + if recorder is None: + return result + recorder.emit( + "final_decision", + status=status, + candidate_ids=list(candidate_ids), + program_digest=result.get("program_digest"), + freeze_digest=result.get("freeze_digest"), + prediction_digest=result.get("prediction_digest"), + **data, + ) + traced = dict(result) + traced["mechanistic_trace"] = recorder.seal(status) + return traced + + +def run_grid_task(task: Any, *, mechanistic: bool) -> dict[str, Any]: + task_id = str(gi._get(task, "task_id")) + demos = tuple(gi._demo_pair(item) for item in tuple(gi._get(task, "demonstrations"))) + hidden = tuple(gi._grid(item) for item in tuple(gi._get(task, "hidden_inputs"))) + recorder = TraceRecorder(task_id) if mechanistic else None + _emit( + recorder, + "run_start", + version=gi._VERSION, + demo_count=len(demos), + hidden_count=len(hidden), + mode="GRID", + ) + + proposed = _proposals(demos, recorder) + fitting = _fit_proposals(proposed, demos, recorder) + representatives = _deduplicate(fitting, recorder) + programs = tuple(item["program"] for item in representatives) + if not representatives: + result = gi._abstain( + task_id, + len(hidden), + "GRAMMAR_EXHAUSTED", + gi._evidence(programs, 0), + ) + return _finish(result, "GRAMMAR_EXHAUSTED", recorder, []) + + batches = _execute_hidden(representatives, hidden, recorder) + evidence = gi._evidence(programs, len(batches)) + if not batches: + result = gi._abstain(task_id, len(hidden), "EXECUTION_FAILED", evidence) + return _finish( + result, + "EXECUTION_FAILED", + recorder, + [item["candidate_id"] for item in representatives], + ) + completed_ids = [ + item["candidate_id"] + for _, (_, members, _) in sorted(batches.items(), key=lambda item: item[0]) + for item in members + ] + if len(batches) != 1: + result = gi._abstain(task_id, len(hidden), "AMBIGUOUS_PROGRAM", evidence) + return _finish( + result, + "AMBIGUOUS_PROGRAM", + recorder, + completed_ids, + output_class_count=len(batches), + ) + + outputs, agreeing, batch_ref = next(iter(batches.values())) + chosen = min(agreeing, key=lambda item: gi._digest(item["program"])) + program_digest = gi._digest(chosen["program"]) + freeze_digest = gi._digest({"version": gi._VERSION, "program_digest": program_digest}) + predictions = [ + {"status": "ACCEPTED", "prediction": deepcopy(item)} + for item in outputs + ] + result = { + "task_id": task_id, + "tier": "V2.2-GRID", + "program_digest": program_digest, + "freeze_digest": freeze_digest, + "prediction_digest": gi._digest(predictions), + "evidence": evidence, + "predictions": predictions, + } + return _finish( + result, + "ACCEPTED", + recorder, + [item["candidate_id"] for item in agreeing], + selected_candidate_id=chosen["candidate_id"], + output_batch_ref=batch_ref, + ) + + +def attach_delegate_trace(task: Any, result: Mapping[str, Any]) -> dict[str, Any]: + task_id = str(gi._get(task, "task_id")) + demos = tuple(gi._get(task, "demonstrations")) + hidden = tuple(gi._get(task, "hidden_inputs")) + recorder = TraceRecorder(task_id) + recorder.emit( + "run_start", + version=gi._VERSION, + demo_count=len(demos), + hidden_count=len(hidden), + mode="DELEGATED", + ) + recorder.emit( + "delegate_boundary", + delegate="research.universal_core.holonomy_v2.blind_adapter.run_public_task_v2", + visibility="OPAQUE_FROZEN_CORE", + frozen_core_commit=_FROZEN_CORE_COMMIT, + ) + recorder.emit( + "final_decision", + status="DELEGATED", + candidate_ids=[], + program_digest=result.get("program_digest"), + freeze_digest=result.get("freeze_digest"), + prediction_digest=result.get("prediction_digest"), + ) + traced = dict(result) + traced["mechanistic_trace"] = recorder.seal("DELEGATED") + return traced diff --git a/research/universal_core/holonomy_v2_2/line_connect.py b/research/universal_core/holonomy_v2_2/line_connect.py new file mode 100644 index 00000000..fbdbd471 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/line_connect.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +from collections import defaultdict +from typing import Any, Mapping, Sequence + + +Grid = list[list[int]] +Program = dict[str, Any] +Point = tuple[int, int] +_MODE_FAMILIES = { + "horizontal": ("horizontal",), + "vertical": ("vertical",), + "orthogonal": ("horizontal", "vertical"), + "diagonal": ("main_diagonal", "anti_diagonal"), + "all": ("horizontal", "vertical", "main_diagonal", "anti_diagonal"), +} + + +def _shape(grid: Grid) -> tuple[int, int]: + if not grid or not grid[0]: + raise ValueError("line connection requires a nonempty grid") + width = len(grid[0]) + if any(len(row) != width for row in grid): + raise ValueError("line connection requires a rectangular grid") + return len(grid), width + + +def _line_key(point: Point, family: str) -> int: + row, col = point + if family == "horizontal": + return row + if family == "vertical": + return col + if family == "main_diagonal": + return row - col + if family == "anti_diagonal": + return row + col + raise ValueError("unknown line family") + + +def _segment(first: Point, last: Point, family: str) -> tuple[Point, ...]: + if family == "horizontal": + row = first[0] + return tuple((row, col) for col in range(first[1], last[1] + 1)) + if family == "vertical": + col = first[1] + return tuple((row, col) for row in range(first[0], last[0] + 1)) + if family == "main_diagonal": + return tuple( + (row, first[1] + (row - first[0])) + for row in range(first[0], last[0] + 1) + ) + if family == "anti_diagonal": + return tuple( + (row, first[1] - (row - first[0])) + for row in range(first[0], last[0] + 1) + ) + raise ValueError("unknown line family") + + +def _ordered(points: Sequence[Point], family: str) -> list[Point]: + if family == "horizontal": + return sorted(points, key=lambda item: item[1]) + return sorted(points, key=lambda item: item[0]) + + +def apply_line_connect(program: Mapping[str, Any], grid: Grid) -> Grid: + _shape(grid) + background = int(program["background"]) + mode = str(program["mode"]) + if mode not in _MODE_FAMILIES: + raise ValueError("unknown line connection mode") + + colors = sorted({cell for row in grid for cell in row if cell != background}) + anchors = { + color: tuple( + (row, col) + for row, values in enumerate(grid) + for col, cell in enumerate(values) + if cell == color + ) + for color in colors + } + + claims: dict[Point, int] = {} + for color in colors: + for family in _MODE_FAMILIES[mode]: + groups: dict[int, list[Point]] = defaultdict(list) + for point in anchors[color]: + groups[_line_key(point, family)].append(point) + for points in groups.values(): + if len(points) < 2: + continue + ordered = _ordered(points, family) + for row, col in _segment(ordered[0], ordered[-1], family): + existing = grid[row][col] + if existing not in (background, color): + raise ValueError("line connection color conflict") + claimed = claims.get((row, col)) + if claimed is not None and claimed != color: + raise ValueError("line connection color conflict") + claims[(row, col)] = color + + output = [list(row) for row in grid] + for (row, col), color in claims.items(): + output[row][col] = color + return output + + +def line_connect_programs( + demos: Sequence[tuple[Grid, Grid]], +) -> tuple[tuple[Program, ...], str | None]: + if not demos: + return (), "NO_DEMONSTRATION" + + for source, target in demos: + if _shape(source) != _shape(target): + return (), "NO_SHAPE_PRESERVING_LINE_CONNECT" + + common_backgrounds = {cell for row in demos[0][0] for cell in row} + for source, _ in demos[1:]: + common_backgrounds.intersection_update(cell for row in source for cell in row) + + programs: list[Program] = [] + for background in sorted(common_backgrounds): + for mode in _MODE_FAMILIES: + program = { + "kind": "same_color_line_connect", + "background": background, + "mode": mode, + } + predictions: list[Grid] = [] + try: + predictions = [apply_line_connect(program, source) for source, _ in demos] + except (KeyError, TypeError, ValueError): + continue + if not any(prediction != source for prediction, (source, _) in zip(predictions, demos)): + continue + if all(prediction == target for prediction, (_, target) in zip(predictions, demos)): + programs.append(program) + + if not programs: + return (), "NO_LINE_CONNECTION" + return tuple(programs), None diff --git a/research/universal_core/holonomy_v2_2/marker_recolor.py b/research/universal_core/holonomy_v2_2/marker_recolor.py new file mode 100644 index 00000000..f2b3afd8 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/marker_recolor.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +from copy import deepcopy +from typing import Any, Mapping, Sequence + + +Grid = list[list[int]] +Program = dict[str, Any] +Point = tuple[int, int] +Component = tuple[Point, ...] + + +def _steps(connectivity: int) -> tuple[Point, ...]: + orthogonal: tuple[Point, ...] = ((1, 0), (-1, 0), (0, 1), (0, -1)) + if connectivity == 4: + return orthogonal + if connectivity == 8: + return orthogonal + ((1, 1), (1, -1), (-1, 1), (-1, -1)) + raise ValueError("marker connectivity must be four or eight") + + +def _components(grid: Grid, background: int, connectivity: int) -> tuple[Component, ...]: + height = len(grid) + width = len(grid[0]) + seen: set[Point] = set() + found: list[Component] = [] + for row in range(height): + for col in range(width): + point = (row, col) + if point in seen or grid[row][col] == background: + continue + color = grid[row][col] + pending = [point] + seen.add(point) + cells: list[Point] = [] + while pending: + current = pending.pop() + cells.append(current) + for row_step, col_step in _steps(connectivity): + next_row = current[0] + row_step + next_col = current[1] + col_step + next_point = (next_row, next_col) + if not (0 <= next_row < height and 0 <= next_col < width): + continue + if next_point in seen or grid[next_row][next_col] != color: + continue + seen.add(next_point) + pending.append(next_point) + found.append(tuple(sorted(cells))) + return tuple(found) + + +def apply_marker_recolor(program: Mapping[str, Any], grid: Grid) -> Grid: + background = int(program["background"]) + connectivity = int(program["connectivity"]) + components = _components(grid, background, connectivity) + singleton_components = [item for item in components if len(item) == 1] + if len(singleton_components) != 1: + raise ValueError("marker recolor requires exactly one singleton marker") + marker = singleton_components[0][0] + foreground_count = sum( + 1 + for row in grid + for cell in row + if cell != background + ) + if foreground_count <= 1: + raise ValueError("marker recolor requires another foreground cell") + marker_color = grid[marker[0]][marker[1]] + output = deepcopy(grid) + for row_index, row in enumerate(grid): + for col_index, cell in enumerate(row): + point = (row_index, col_index) + if point == marker: + output[row_index][col_index] = background + elif cell != background: + output[row_index][col_index] = marker_color + return output + + +def _background_candidates( + demos: Sequence[tuple[Grid, Grid]], +) -> tuple[int, ...]: + common = set(cell for row in demos[0][0] for cell in row) + for source, _ in demos[1:]: + common.intersection_update(cell for row in source for cell in row) + return tuple(sorted(common)) + + +def marker_recolor_programs( + demos: Sequence[tuple[Grid, Grid]], +) -> tuple[tuple[Program, ...], str | None]: + backgrounds = _background_candidates(demos) + if not backgrounds: + return (), "NO_BACKGROUND_CANDIDATE" + programs = tuple( + { + "kind": "marker_recolor", + "background": background, + "connectivity": connectivity, + } + for background in backgrounds + for connectivity in (4, 8) + ) + return programs, None diff --git a/research/universal_core/holonomy_v2_2/mechanistic_trace.py b/research/universal_core/holonomy_v2_2/mechanistic_trace.py new file mode 100644 index 00000000..55013719 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/mechanistic_trace.py @@ -0,0 +1,303 @@ +from __future__ import annotations + +import hashlib +import json +from copy import deepcopy +from pathlib import Path +from typing import Any, Mapping + + +_TRACE_SCHEMA = "mechanistic-grid-trace-v1" + + +def _canonical(value: Any) -> bytes: + return ( + json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ) + + "\n" + ).encode("utf-8") + + +def _digest(value: Any) -> str: + return hashlib.sha256(_canonical(value)).hexdigest() + + +def _normalized(value: Any) -> Any: + try: + return json.loads(_canonical(value).decode("utf-8")) + except (TypeError, ValueError) as exc: + raise TypeError("trace values must be JSON-safe") from exc + + +def _content_body(trace: Mapping[str, Any]) -> dict[str, Any]: + return { + "schema": trace["schema"], + "objects": trace["objects"], + "events": trace["events"], + "terminal_status": trace["terminal_status"], + } + + +def _object_references(value: Any) -> tuple[str, ...]: + found: list[str] = [] + if isinstance(value, Mapping): + for key, item in value.items(): + if isinstance(key, str) and key.endswith("_ref"): + if not isinstance(item, str): + raise ValueError("object reference must be a digest string") + found.append(item) + else: + found.extend(_object_references(item)) + elif isinstance(value, list): + for item in value: + found.extend(_object_references(item)) + return tuple(found) + + +def _events_named(events: list[Mapping[str, Any]], name: str) -> list[Mapping[str, Any]]: + return [item for item in events if item.get("event") == name] + + +def validate_trace(trace: Mapping[str, Any]) -> None: + if not isinstance(trace, Mapping): + raise ValueError("trace must be a mapping") + if trace.get("schema") != _TRACE_SCHEMA: + raise ValueError("unknown trace schema") + task_id = trace.get("task_id") + if not isinstance(task_id, str) or not task_id: + raise ValueError("trace task_id must be non-empty") + + objects = trace.get("objects") + if not isinstance(objects, Mapping): + raise ValueError("trace objects must be a mapping") + for key, value in objects.items(): + if not isinstance(key, str) or _digest(value) != key: + raise ValueError("trace object digest mismatch") + + events_value = trace.get("events") + if not isinstance(events_value, list): + raise ValueError("trace events must be a list") + events: list[Mapping[str, Any]] = [] + for ordinal, item in enumerate(events_value): + if not isinstance(item, Mapping): + raise ValueError("trace event must be a mapping") + if item.get("ordinal") != ordinal: + raise ValueError("trace event ordinals must be contiguous") + if not isinstance(item.get("event"), str) or not item["event"]: + raise ValueError("trace event name must be non-empty") + events.append(item) + for reference in _object_references(item): + if reference not in objects: + raise ValueError("unknown object reference") + + if trace.get("event_count") != len(events): + raise ValueError("trace event count mismatch") + + proposed_events = _events_named(events, "candidate_proposed") + proposed_ids: list[str] = [] + for item in proposed_events: + candidate_id = item.get("candidate_id") + if not isinstance(candidate_id, str) or not candidate_id: + raise ValueError("candidate proposal requires an id") + if candidate_id in proposed_ids: + raise ValueError("candidate ids must be unique") + proposed_ids.append(candidate_id) + proposed = set(proposed_ids) + + fit_events = _events_named(events, "candidate_fit_decision") + fit_by_candidate: dict[str, list[Mapping[str, Any]]] = {} + for item in fit_events: + candidate_id = item.get("candidate_id") + if candidate_id not in proposed: + raise ValueError("fit decision references unknown candidate") + fit_by_candidate.setdefault(str(candidate_id), []).append(item) + for candidate_id in proposed_ids: + if len(fit_by_candidate.get(candidate_id, [])) != 1: + raise ValueError("candidate requires exactly one terminal demonstration decision") + + duplicate_events = _events_named(events, "candidate_duplicate") + duplicates: dict[str, str] = {} + for item in duplicate_events: + candidate_id = item.get("candidate_id") + representative_id = item.get("representative_id") + if candidate_id not in proposed or representative_id not in proposed: + raise ValueError("duplicate event references unknown candidate") + if candidate_id == representative_id: + raise ValueError("duplicate candidate cannot represent itself") + if candidate_id in duplicates: + raise ValueError("candidate has multiple duplicate decisions") + duplicates[str(candidate_id)] = str(representative_id) + + fitting = { + candidate_id + for candidate_id, items in fit_by_candidate.items() + if items[0].get("decision") == "FITS_ALL_DEMOS" + } + representatives = fitting - set(duplicates) + + hidden_batch_events = _events_named(events, "hidden_batch_decision") + hidden_by_candidate: dict[str, list[Mapping[str, Any]]] = {} + for item in hidden_batch_events: + candidate_id = item.get("candidate_id") + if candidate_id not in representatives: + raise ValueError("hidden-batch decision references non-representative candidate") + hidden_by_candidate.setdefault(str(candidate_id), []).append(item) + for candidate_id in sorted(representatives): + if len(hidden_by_candidate.get(candidate_id, [])) != 1: + raise ValueError("fitting candidate requires exactly one terminal hidden-batch decision") + + run_starts = _events_named(events, "run_start") + if len(run_starts) > 1: + raise ValueError("trace has multiple run-start events") + if run_starts: + demo_count = run_starts[0].get("demo_count") + hidden_count = run_starts[0].get("hidden_count") + if type(demo_count) is not int or demo_count < 0: + raise ValueError("run-start demonstration count must be non-negative") + if type(hidden_count) is not int or hidden_count < 0: + raise ValueError("run-start hidden count must be non-negative") + + demo_events = _events_named(events, "demo_execution") + demo_by_candidate: dict[str, list[Mapping[str, Any]]] = {} + for item in demo_events: + candidate_id = item.get("candidate_id") + if candidate_id not in proposed: + raise ValueError("demonstration execution references unknown candidate") + demo_by_candidate.setdefault(str(candidate_id), []).append(item) + expected_demo_indices = list(range(demo_count)) + for candidate_id in proposed_ids: + indices = sorted( + int(item.get("demo_index", -1)) + for item in demo_by_candidate.get(candidate_id, []) + ) + if indices != expected_demo_indices: + raise ValueError("demonstration executions do not cover every candidate and example") + + hidden_events = _events_named(events, "hidden_execution") + by_candidate: dict[str, list[Mapping[str, Any]]] = {} + for item in hidden_events: + candidate_id = item.get("candidate_id") + if candidate_id not in representatives: + raise ValueError("hidden execution references non-representative candidate") + by_candidate.setdefault(str(candidate_id), []).append(item) + expected_indices = list(range(hidden_count)) + for candidate_id in sorted(representatives): + indices = sorted( + int(item.get("hidden_index", -1)) + for item in by_candidate.get(candidate_id, []) + ) + if indices != expected_indices: + raise ValueError("hidden executions do not cover every hidden input") + + completed: dict[str, str] = {} + for candidate_id, items in hidden_by_candidate.items(): + item = items[0] + decision = item.get("decision") + if decision == "COMPLETED": + batch_ref = item.get("batch_ref") + if not isinstance(batch_ref, str) or batch_ref not in objects: + raise ValueError("completed hidden batch requires an object reference") + completed[candidate_id] = batch_ref + elif decision != "FAILED": + raise ValueError("unknown hidden-batch decision") + + class_events = _events_named(events, "equivalence_class") + memberships: dict[str, list[str]] = {} + for item in class_events: + batch_ref = item.get("batch_ref") + candidate_ids = item.get("candidate_ids") + if not isinstance(batch_ref, str) or batch_ref not in objects: + raise ValueError("equivalence class requires a known batch") + if not isinstance(candidate_ids, list) or not candidate_ids: + raise ValueError("equivalence class requires candidate ids") + for candidate_id in candidate_ids: + if candidate_id not in completed: + raise ValueError("equivalence class references incomplete candidate") + if completed[str(candidate_id)] != batch_ref: + raise ValueError("equivalence class batch mismatch") + memberships.setdefault(str(candidate_id), []).append(batch_ref) + for candidate_id in sorted(completed): + if memberships.get(candidate_id) != [completed[candidate_id]]: + raise ValueError("completed candidate requires exactly one equivalence class") + + final_events = _events_named(events, "final_decision") + if len(final_events) != 1: + raise ValueError("trace requires exactly one final decision") + final = final_events[0] + terminal_status = trace.get("terminal_status") + if not isinstance(terminal_status, str) or final.get("status") != terminal_status: + raise ValueError("trace terminal status mismatch") + final_candidates = final.get("candidate_ids", []) + if not isinstance(final_candidates, list) or any(item not in proposed for item in final_candidates): + raise ValueError("final decision references unknown candidate") + + expected_trace_digest = _digest(_content_body(trace)) + if trace.get("trace_digest") != expected_trace_digest: + raise ValueError("trace digest mismatch") + expected_envelope = _digest({"task_id": task_id, "trace_digest": expected_trace_digest}) + if trace.get("envelope_digest") != expected_envelope: + raise ValueError("trace envelope digest mismatch") + + +class TraceRecorder: + def __init__(self, task_id: str) -> None: + if not isinstance(task_id, str) or not task_id: + raise ValueError("trace task_id must be non-empty") + self._task_id = task_id + self._objects: dict[str, Any] = {} + self._object_bytes: dict[str, bytes] = {} + self._events: list[dict[str, Any]] = [] + + def store(self, value: Any) -> str: + normalized = _normalized(value) + encoded = _canonical(normalized) + digest = hashlib.sha256(encoded).hexdigest() + if digest in self._object_bytes and self._object_bytes[digest] != encoded: + raise ValueError("trace object digest collision") + self._object_bytes[digest] = encoded + self._objects.setdefault(digest, normalized) + return digest + + def emit(self, event: str, **data: Any) -> None: + if not isinstance(event, str) or not event: + raise ValueError("trace event name must be non-empty") + if "ordinal" in data or "event" in data: + raise ValueError("trace event data contains reserved key") + item = _normalized({ + "ordinal": len(self._events), + "event": event, + **data, + }) + self._events.append(item) + + def seal(self, terminal_status: str) -> dict[str, Any]: + if not isinstance(terminal_status, str) or not terminal_status: + raise ValueError("terminal status must be non-empty") + trace: dict[str, Any] = { + "schema": _TRACE_SCHEMA, + "task_id": self._task_id, + "objects": {key: self._objects[key] for key in sorted(self._objects)}, + "events": deepcopy(self._events), + "event_count": len(self._events), + "terminal_status": terminal_status, + } + trace["trace_digest"] = _digest(_content_body(trace)) + trace["envelope_digest"] = _digest({ + "task_id": self._task_id, + "trace_digest": trace["trace_digest"], + }) + validate_trace(trace) + return trace + + +def write_trace(trace: Mapping[str, Any], path: Path | str) -> Path: + validate_trace(trace) + output = Path(path) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(_canonical(trace)) + return output diff --git a/research/universal_core/holonomy_v2_2/mirror_concat.py b/research/universal_core/holonomy_v2_2/mirror_concat.py new file mode 100644 index 00000000..074a0a99 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/mirror_concat.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from typing import Any, Mapping, Sequence + + +Grid = list[list[int]] +Program = dict[str, Any] + + +def apply_mirror_concat(program: Mapping[str, Any], grid: Grid) -> Grid: + axis = str(program["axis"]) + order = str(program["order"]) + if order not in {"source_first", "reflection_first"}: + raise ValueError("unknown mirror concatenation order") + + if axis == "rows": + if len(grid) * 2 > 30: + raise ValueError("row mirror concatenation exceeds ARC bounds") + source = [list(row) for row in grid] + reflected = [list(row) for row in reversed(grid)] + return source + reflected if order == "source_first" else reflected + source + + if axis == "columns": + if len(grid[0]) * 2 > 30: + raise ValueError("column mirror concatenation exceeds ARC bounds") + output: Grid = [] + for row in grid: + source = list(row) + reflected = list(reversed(row)) + output.append( + source + reflected + if order == "source_first" + else reflected + source + ) + return output + + raise ValueError("unknown mirror concatenation axis") + + +def mirror_concat_programs( + demos: Sequence[tuple[Grid, Grid]], +) -> tuple[tuple[Program, ...], str | None]: + if not demos: + return (), "NO_DEMONSTRATION" + return ( + {"kind": "mirror_concatenate", "axis": "rows", "order": "source_first"}, + {"kind": "mirror_concatenate", "axis": "rows", "order": "reflection_first"}, + {"kind": "mirror_concatenate", "axis": "columns", "order": "source_first"}, + {"kind": "mirror_concatenate", "axis": "columns", "order": "reflection_first"}, + ), None diff --git a/research/universal_core/holonomy_v2_2/palette_repeat.py b/research/universal_core/holonomy_v2_2/palette_repeat.py new file mode 100644 index 00000000..6166faf5 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/palette_repeat.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +from typing import Any, Mapping, Sequence + + +Grid = list[list[int]] +Program = dict[str, Any] + + +def _factor(program: Mapping[str, Any], grid: Grid) -> int: + colors = {cell for row in grid for cell in row} + statistic = str(program["statistic"]) + if statistic == "distinct_all": + return len(colors) + if statistic == "distinct_non_background": + return len(colors - {int(program["background"])}) + raise ValueError("unknown palette cardinality statistic") + + +def _scale(grid: Grid, factor: int) -> Grid: + return [ + [cell for cell in row for _ in range(factor)] + for row in grid + for _ in range(factor) + ] + + +def _tile(grid: Grid, factor: int) -> Grid: + return [ + list(row) * factor + for _ in range(factor) + for row in grid + ] + + +def apply_palette_repeat(program: Mapping[str, Any], grid: Grid) -> Grid: + factor = _factor(program, grid) + if factor < 1: + raise ValueError("palette repeat factor must be positive") + if len(grid) * factor > 30 or len(grid[0]) * factor > 30: + raise ValueError("palette repeat exceeds ARC bounds") + + operation = str(program["operation"]) + if operation == "scale": + return _scale(grid, factor) + if operation == "tile": + return _tile(grid, factor) + raise ValueError("unknown palette repeat operation") + + +def _candidate_programs( + demos: Sequence[tuple[Grid, Grid]], +) -> tuple[Program, ...]: + programs: list[Program] = [] + for operation in ("scale", "tile"): + programs.append({ + "kind": "palette_cardinality_repeat", + "operation": operation, + "statistic": "distinct_all", + }) + + common_backgrounds = set(cell for row in demos[0][0] for cell in row) + for source, _ in demos[1:]: + common_backgrounds.intersection_update(cell for row in source for cell in row) + for operation in ("scale", "tile"): + for background in sorted(common_backgrounds): + programs.append({ + "kind": "palette_cardinality_repeat", + "operation": operation, + "statistic": "distinct_non_background", + "background": background, + }) + return tuple(programs) + + +def palette_repeat_programs( + demos: Sequence[tuple[Grid, Grid]], +) -> tuple[tuple[Program, ...], str | None]: + if not demos: + return (), "NO_DEMONSTRATION" + + fitting: list[Program] = [] + constant_fit = False + for program in _candidate_programs(demos): + factors: list[int] = [] + exact = True + for source, target in demos: + try: + factors.append(_factor(program, source)) + output = apply_palette_repeat(program, source) + except (KeyError, TypeError, ValueError): + exact = False + break + if output != target: + exact = False + break + if not exact: + continue + if len(set(factors)) < 2: + constant_fit = True + continue + fitting.append(program) + + if fitting: + return tuple(fitting), None + if constant_fit: + return (), "NO_FACTOR_VARIATION" + return (), "NO_SUPPORTED_PALETTE_REPEAT" diff --git a/research/universal_core/holonomy_v2_2/panel_combine.py b/research/universal_core/holonomy_v2_2/panel_combine.py new file mode 100644 index 00000000..c53f3ce9 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/panel_combine.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +from collections import Counter +from typing import Any, Mapping, Sequence + + +Grid = list[list[int]] +Program = dict[str, Any] + +_PREDICATES = ( + "or", + "and", + "xor", + "equal_foreground", + "left_only", + "right_only", +) + + +def _split(grid: Grid, axis: str, gap: int) -> tuple[Grid, Grid]: + height = len(grid) + width = len(grid[0]) + if axis == "vertical": + if (width - gap) % 2: + raise ValueError("vertical panels must have equal width") + panel_width = (width - gap) // 2 + if panel_width < 1: + raise ValueError("vertical panel width must be positive") + left = [row[:panel_width] for row in grid] + right = [row[panel_width + gap :] for row in grid] + return left, right + if axis == "horizontal": + if (height - gap) % 2: + raise ValueError("horizontal panels must have equal height") + panel_height = (height - gap) // 2 + if panel_height < 1: + raise ValueError("horizontal panel height must be positive") + return grid[:panel_height], grid[panel_height + gap :] + raise ValueError("unknown panel axis") + + +def _predicate(name: str, left: int, right: int, background: int) -> bool: + left_foreground = left != background + right_foreground = right != background + if name == "or": + return left_foreground or right_foreground + if name == "and": + return left_foreground and right_foreground + if name == "xor": + return left_foreground != right_foreground + if name == "equal_foreground": + return left_foreground == right_foreground + if name == "left_only": + return left_foreground and not right_foreground + if name == "right_only": + return right_foreground and not left_foreground + raise ValueError("unknown panel predicate") + + +def apply_panel(program: Mapping[str, Any], grid: Grid) -> Grid: + left, right = _split(grid, str(program["axis"]), int(program["gap"])) + if len(left) != len(right) or len(left[0]) != len(right[0]): + raise ValueError("panel shapes differ") + background = int(program["background"]) + true_color = int(program["true_color"]) + false_color = int(program["false_color"]) + predicate = str(program["predicate"]) + return [ + [ + true_color + if _predicate(predicate, left[row][col], right[row][col], background) + else false_color + for col in range(len(left[0])) + ] + for row in range(len(left)) + ] + + +def _fits(program: Program, demos: Sequence[tuple[Grid, Grid]]) -> bool: + try: + return all(apply_panel(program, source) == target for source, target in demos) + except (KeyError, TypeError, ValueError): + return False + + +def _modal_color(grid: Grid) -> int: + counts = Counter(cell for row in grid for cell in row) + highest = max(counts.values()) + return min(color for color, count in counts.items() if count == highest) + + +def _background_candidates(demos: Sequence[tuple[Grid, Grid]]) -> tuple[int, ...]: + backgrounds = {0} + modal_colors = {_modal_color(source) for source, _ in demos} + if len(modal_colors) == 1: + backgrounds.update(modal_colors) + return tuple(sorted(backgrounds)) + + +def panel_programs( + demos: Sequence[tuple[Grid, Grid]], +) -> tuple[tuple[Program, ...], str | None]: + target_colors = sorted( + {cell for _, target in demos for row in target for cell in row} + ) + if len(target_colors) != 2: + return (), "TARGET_COLOR_CARDINALITY_NOT_TWO" + programs: list[Program] = [] + for axis in ("vertical", "horizontal"): + for gap in (0, 1, 2): + for background in _background_candidates(demos): + for predicate in _PREDICATES: + for true_color, false_color in ( + (target_colors[0], target_colors[1]), + (target_colors[1], target_colors[0]), + ): + programs.append({ + "kind": "panel_boolean_combine", + "axis": axis, + "gap": gap, + "background": background, + "predicate": predicate, + "true_color": true_color, + "false_color": false_color, + }) + if not programs: + return (), "NO_STRUCTURAL_PROGRAM" + return tuple(programs), None + + +def panel_candidates(demos: Sequence[tuple[Grid, Grid]]) -> tuple[Program, ...]: + programs, _ = panel_programs(demos) + return tuple(program for program in programs if _fits(program, demos)) diff --git a/research/universal_core/holonomy_v2_2/quadrant_mosaic.py b/research/universal_core/holonomy_v2_2/quadrant_mosaic.py new file mode 100644 index 00000000..eed73ca8 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/quadrant_mosaic.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +from itertools import product +from typing import Any, Mapping, Sequence + + +Grid = list[list[int]] +Program = dict[str, Any] + +_TRANSFORM_ORDER = ( + "identity", + "rotate_90", + "rotate_180", + "rotate_270", + "reflect_columns", + "reflect_rows", + "reflect_main_diagonal", + "reflect_anti_diagonal", +) +_QUADRANT_FIELDS = ( + "top_left", + "top_right", + "bottom_left", + "bottom_right", +) + + +def _shape(grid: Grid) -> tuple[int, int]: + return len(grid), len(grid[0]) + + +def _rotate_90(grid: Grid) -> Grid: + return [list(row) for row in zip(*reversed(grid))] + + +def _apply_transform(name: str, grid: Grid) -> Grid: + source = [list(row) for row in grid] + if name == "identity": + return source + if name == "rotate_90": + return _rotate_90(source) + if name == "rotate_180": + return [list(reversed(row)) for row in reversed(source)] + if name == "rotate_270": + return _rotate_90(_rotate_90(_rotate_90(source))) + if name == "reflect_columns": + return [list(reversed(row)) for row in source] + if name == "reflect_rows": + return [list(row) for row in reversed(source)] + if name == "reflect_main_diagonal": + return [list(row) for row in zip(*source)] + if name == "reflect_anti_diagonal": + transposed = [list(row) for row in zip(*source)] + return [list(reversed(row)) for row in reversed(transposed)] + raise ValueError("unknown quadrant mosaic transform") + + +def _split_quadrants(target: Grid, height: int, width: int) -> tuple[Grid, Grid, Grid, Grid]: + return ( + [row[:width] for row in target[:height]], + [row[width:] for row in target[:height]], + [row[:width] for row in target[height:]], + [row[width:] for row in target[height:]], + ) + + +def apply_quadrant_mosaic(program: Mapping[str, Any], grid: Grid) -> Grid: + height, width = _shape(grid) + if height * 2 > 30 or width * 2 > 30: + raise ValueError("quadrant mosaic exceeds ARC bounds") + + quadrants = tuple( + _apply_transform(str(program[field]), grid) + for field in _QUADRANT_FIELDS + ) + if any(_shape(item) != (height, width) for item in quadrants): + raise ValueError("quadrant transform must preserve source shape") + + top_left, top_right, bottom_left, bottom_right = quadrants + return ( + [top_left[row] + top_right[row] for row in range(height)] + + [bottom_left[row] + bottom_right[row] for row in range(height)] + ) + + +def quadrant_mosaic_programs( + demos: Sequence[tuple[Grid, Grid]], +) -> tuple[tuple[Program, ...], str | None]: + if not demos: + return (), "NO_DEMONSTRATION" + + supported: list[set[str]] = [set(_TRANSFORM_ORDER) for _ in _QUADRANT_FIELDS] + for source, target in demos: + height, width = _shape(source) + if _shape(target) != (height * 2, width * 2): + return (), "OUTPUT_NOT_DOUBLE_SOURCE_SHAPE" + quadrants = _split_quadrants(target, height, width) + for index, quadrant in enumerate(quadrants): + matches = { + name + for name in _TRANSFORM_ORDER + if _shape(_apply_transform(name, source)) == (height, width) + and _apply_transform(name, source) == quadrant + } + supported[index].intersection_update(matches) + + if any(not names for names in supported): + return (), "NO_SUPPORTED_QUADRANT_TRANSFORM" + if set.intersection(*supported): + return (), "NO_DISTINCT_QUADRANT_ROLES" + + ordered = [ + tuple(name for name in _TRANSFORM_ORDER if name in names) + for names in supported + ] + programs = tuple( + { + "kind": "dihedral_quadrant_mosaic", + **{ + field: transform + for field, transform in zip(_QUADRANT_FIELDS, transforms) + }, + } + for transforms in product(*ordered) + ) + return programs, None diff --git a/research/universal_core/holonomy_v2_2/ray_extend.py b/research/universal_core/holonomy_v2_2/ray_extend.py new file mode 100644 index 00000000..a988a541 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/ray_extend.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +from typing import Any, Mapping, Sequence + + +Grid = list[list[int]] +Program = dict[str, Any] +Point = tuple[int, int] +_DIRECTION = { + "N": (-1, 0), + "S": (1, 0), + "W": (0, -1), + "E": (0, 1), + "NW": (-1, -1), + "NE": (-1, 1), + "SW": (1, -1), + "SE": (1, 1), +} +_MODE_DIRECTIONS = { + "up": ("N",), + "down": ("S",), + "left": ("W",), + "right": ("E",), + "vertical": ("N", "S"), + "horizontal": ("W", "E"), + "orthogonal": ("N", "S", "W", "E"), + "diagonal": ("NW", "NE", "SW", "SE"), + "all": ("N", "S", "W", "E", "NW", "NE", "SW", "SE"), +} + + +def _shape(grid: Grid) -> tuple[int, int]: + if not grid or not grid[0]: + raise ValueError("ray extension requires a nonempty grid") + width = len(grid[0]) + if any(len(row) != width for row in grid): + raise ValueError("ray extension requires a rectangular grid") + return len(grid), width + + +def apply_ray_extend(program: Mapping[str, Any], grid: Grid) -> Grid: + height, width = _shape(grid) + background = int(program["background"]) + mode = str(program["mode"]) + if mode not in _MODE_DIRECTIONS: + raise ValueError("unknown ray extension mode") + + anchors = tuple( + (row, col, grid[row][col]) + for row in range(height) + for col in range(width) + if grid[row][col] != background + ) + claims: dict[Point, int] = {} + for row, col, color in anchors: + for direction in _MODE_DIRECTIONS[mode]: + delta_row, delta_col = _DIRECTION[direction] + target_row = row + delta_row + target_col = col + delta_col + while 0 <= target_row < height and 0 <= target_col < width: + existing = grid[target_row][target_col] + if existing not in (background, color): + raise ValueError("ray extension color conflict") + point = (target_row, target_col) + claimed = claims.get(point) + if claimed is not None and claimed != color: + raise ValueError("ray extension color conflict") + claims[point] = color + target_row += delta_row + target_col += delta_col + + output = [list(row) for row in grid] + for (row, col), color in claims.items(): + output[row][col] = color + return output + + +def ray_extend_programs( + demos: Sequence[tuple[Grid, Grid]], +) -> tuple[tuple[Program, ...], str | None]: + if not demos: + return (), "NO_DEMONSTRATION" + + for source, target in demos: + if _shape(source) != _shape(target): + return (), "NO_SHAPE_PRESERVING_RAY_EXTENSION" + + common_backgrounds = {cell for row in demos[0][0] for cell in row} + for source, _ in demos[1:]: + common_backgrounds.intersection_update(cell for row in source for cell in row) + + programs: list[Program] = [] + for background in sorted(common_backgrounds): + for mode in _MODE_DIRECTIONS: + program = { + "kind": "anchor_ray_extend", + "background": background, + "mode": mode, + } + try: + predictions = [apply_ray_extend(program, source) for source, _ in demos] + except (KeyError, TypeError, ValueError): + continue + if not any(prediction != source for prediction, (source, _) in zip(predictions, demos)): + continue + if all(prediction == target for prediction, (_, target) in zip(predictions, demos)): + programs.append(program) + + if not programs: + return (), "NO_RAY_EXTENSION" + return tuple(programs), None diff --git a/research/universal_core/holonomy_v2_2/region_fill.py b/research/universal_core/holonomy_v2_2/region_fill.py new file mode 100644 index 00000000..80ea7595 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/region_fill.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +from collections import Counter +from copy import deepcopy +from typing import Any, Mapping, Sequence + + +Grid = list[list[int]] +Program = dict[str, Any] +Point = tuple[int, int] + + +def _directions(connectivity: int) -> tuple[Point, ...]: + steps: tuple[Point, ...] = ((1, 0), (-1, 0), (0, 1), (0, -1)) + if connectivity == 8: + return steps + ((1, 1), (1, -1), (-1, 1), (-1, -1)) + if connectivity == 4: + return steps + raise ValueError("region connectivity must be four or eight") + + +def _exterior_background(grid: Grid, background: int, connectivity: int) -> set[Point]: + height = len(grid) + width = len(grid[0]) + pending: list[Point] = [] + seen: set[Point] = set() + border = ( + [(row, 0) for row in range(height)] + + [(row, width - 1) for row in range(height)] + + [(0, col) for col in range(width)] + + [(height - 1, col) for col in range(width)] + ) + for point in border: + if point not in seen and grid[point[0]][point[1]] == background: + seen.add(point) + pending.append(point) + while pending: + row, col = pending.pop() + for row_step, col_step in _directions(connectivity): + next_row = row + row_step + next_col = col + col_step + point = (next_row, next_col) + if not (0 <= next_row < height and 0 <= next_col < width): + continue + if point in seen or grid[next_row][next_col] != background: + continue + seen.add(point) + pending.append(point) + return seen + + +def _enclosed_background(grid: Grid, background: int, connectivity: int) -> set[Point]: + exterior = _exterior_background(grid, background, connectivity) + return { + (row, col) + for row, values in enumerate(grid) + for col, value in enumerate(values) + if value == background and (row, col) not in exterior + } + + +def apply_region(program: Mapping[str, Any], grid: Grid) -> Grid: + background = int(program["background"]) + fill_color = int(program["fill_color"]) + connectivity = int(program["connectivity"]) + mode = str(program["mode"]) + enclosed = _enclosed_background(grid, background, connectivity) + if mode == "fill_copy": + output = deepcopy(grid) + for row, col in enclosed: + output[row][col] = fill_color + return output + if mode == "highlight": + output = [[background for _ in values] for values in grid] + for row, col in enclosed: + output[row][col] = fill_color + return output + if mode == "solidify": + exterior = _exterior_background(grid, background, connectivity) + return [ + [background if (row, col) in exterior else fill_color for col in range(len(grid[0]))] + for row in range(len(grid)) + ] + raise ValueError("unknown enclosed region mode") + + +def _fits(program: Program, demos: Sequence[tuple[Grid, Grid]]) -> bool: + try: + return all(apply_region(program, source) == target for source, target in demos) + except (KeyError, TypeError, ValueError): + return False + + +def _modal_color(grid: Grid) -> int: + counts = Counter(cell for row in grid for cell in row) + highest = max(counts.values()) + return min(color for color, count in counts.items() if count == highest) + + +def _background_candidates(demos: Sequence[tuple[Grid, Grid]]) -> tuple[int, ...]: + common = set(cell for row in demos[0][0] for cell in row) + for source, _ in demos[1:]: + common.intersection_update(cell for row in source for cell in row) + candidates = {0} if 0 in common else set() + modal_colors = {_modal_color(source) for source, _ in demos} + if len(modal_colors) == 1: + candidates.update(modal_colors & common) + return tuple(sorted(candidates)) + + +def region_programs( + demos: Sequence[tuple[Grid, Grid]], +) -> tuple[tuple[Program, ...], str | None]: + backgrounds = _background_candidates(demos) + if not backgrounds: + return (), "NO_BACKGROUND_CANDIDATE" + target_colors = sorted({cell for _, target in demos for row in target for cell in row}) + programs: list[Program] = [] + for background in backgrounds: + for connectivity in (4, 8): + for mode in ("fill_copy", "highlight", "solidify"): + for fill_color in target_colors: + programs.append({ + "kind": "enclosed_region_fill", + "background": background, + "connectivity": connectivity, + "mode": mode, + "fill_color": fill_color, + }) + if not programs: + return (), "NO_STRUCTURAL_PROGRAM" + return tuple(programs), None + + +def region_candidates(demos: Sequence[tuple[Grid, Grid]]) -> tuple[Program, ...]: + programs, _ = region_programs(demos) + return tuple(program for program in programs if _fits(program, demos)) diff --git a/research/universal_core/holonomy_v2_2/symmetry_complete.py b/research/universal_core/holonomy_v2_2/symmetry_complete.py new file mode 100644 index 00000000..d0a1bf87 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/symmetry_complete.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +from typing import Any, Mapping, Sequence + + +Grid = list[list[int]] +Program = dict[str, Any] +Point = tuple[int, int] +_TRANSFORM_ORDER = ( + "vertical", + "horizontal", + "rotate_180", + "main_diagonal", + "anti_diagonal", +) + + +def _shape(grid: Grid) -> tuple[int, int]: + if not grid or not grid[0]: + raise ValueError("symmetry completion requires a nonempty grid") + width = len(grid[0]) + if any(len(row) != width for row in grid): + raise ValueError("symmetry completion requires a rectangular grid") + return len(grid), width + + +def _reflect(point: Point, height: int, width: int, transform: str) -> Point: + row, col = point + if transform == "vertical": + return row, width - 1 - col + if transform == "horizontal": + return height - 1 - row, col + if transform == "rotate_180": + return height - 1 - row, width - 1 - col + if transform == "main_diagonal": + if height != width: + raise ValueError("diagonal symmetry requires a square grid") + return col, row + if transform == "anti_diagonal": + if height != width: + raise ValueError("diagonal symmetry requires a square grid") + return width - 1 - col, height - 1 - row + raise ValueError("unknown symmetry transform") + + +def apply_symmetry_complete(program: Mapping[str, Any], grid: Grid) -> Grid: + height, width = _shape(grid) + background = int(program["background"]) + transform = str(program["transform"]) + if transform not in _TRANSFORM_ORDER: + raise ValueError("unknown symmetry transform") + + anchors = tuple( + (row, col, grid[row][col]) + for row in range(height) + for col in range(width) + if grid[row][col] != background + ) + claims: dict[Point, int] = {} + for row, col, color in anchors: + reflected = _reflect((row, col), height, width, transform) + target_row, target_col = reflected + existing = grid[target_row][target_col] + if existing not in (background, color): + raise ValueError("symmetry completion color conflict") + claimed = claims.get(reflected) + if claimed is not None and claimed != color: + raise ValueError("symmetry completion color conflict") + claims[reflected] = color + + output = [list(row) for row in grid] + for (row, col), color in claims.items(): + output[row][col] = color + return output + + +def symmetry_complete_programs( + demos: Sequence[tuple[Grid, Grid]], +) -> tuple[tuple[Program, ...], str | None]: + if not demos: + return (), "NO_DEMONSTRATION" + + for source, target in demos: + if _shape(source) != _shape(target): + return (), "NO_SHAPE_PRESERVING_SYMMETRY" + + common_backgrounds = {cell for row in demos[0][0] for cell in row} + for source, _ in demos[1:]: + common_backgrounds.intersection_update(cell for row in source for cell in row) + + programs: list[Program] = [] + for background in sorted(common_backgrounds): + for transform in _TRANSFORM_ORDER: + program = { + "kind": "reflective_symmetry_complete", + "background": background, + "transform": transform, + } + try: + predictions = [apply_symmetry_complete(program, source) for source, _ in demos] + except (KeyError, TypeError, ValueError): + continue + if not any(prediction != source for prediction, (source, _) in zip(predictions, demos)): + continue + if all(prediction == target for prediction, (_, target) in zip(predictions, demos)): + programs.append(program) + + if not programs: + return (), "NO_REFLECTIVE_SYMMETRY_COMPLETION" + return tuple(programs), None diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/axis_collapse/2dee498d.json b/research/universal_core/holonomy_v2_2/tests/fixtures/axis_collapse/2dee498d.json new file mode 100644 index 00000000..17ceb931 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/axis_collapse/2dee498d.json @@ -0,0 +1 @@ +{"train": [{"input": [[2, 0, 0, 1, 2, 0, 0, 1, 2, 0, 0, 1], [4, 2, 1, 4, 4, 2, 1, 4, 4, 2, 1, 4], [4, 1, 2, 4, 4, 1, 2, 4, 4, 1, 2, 4], [1, 0, 0, 2, 1, 0, 0, 2, 1, 0, 0, 2]], "output": [[2, 0, 0, 1], [4, 2, 1, 4], [4, 1, 2, 4], [1, 0, 0, 2]]}, {"input": [[4, 5, 1, 1, 5, 4, 4, 5, 1], [5, 5, 5, 5, 5, 5, 5, 5, 5], [1, 5, 4, 4, 5, 1, 1, 5, 4]], "output": [[4, 5, 1], [5, 5, 5], [1, 5, 4]]}, {"input": [[2, 1, 2, 1, 2, 1], [2, 3, 2, 3, 2, 3]], "output": [[2, 1], [2, 3]]}], "test": [{"input": [[0, 2, 0, 4, 4, 0, 2, 0, 4, 4, 0, 2, 0, 4, 4], [2, 2, 0, 4, 4, 2, 2, 0, 4, 4, 2, 2, 0, 4, 4], [0, 2, 2, 2, 0, 0, 2, 2, 2, 0, 0, 2, 2, 2, 0], [1, 1, 0, 2, 2, 1, 1, 0, 2, 2, 1, 1, 0, 2, 2], [1, 1, 0, 2, 0, 1, 1, 0, 2, 0, 1, 1, 0, 2, 0]], "output": [[0, 2, 0, 4, 4], [2, 2, 0, 4, 4], [0, 2, 2, 2, 0], [1, 1, 0, 2, 2], [1, 1, 0, 2, 0]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/axis_collapse/746b3537.json b/research/universal_core/holonomy_v2_2/tests/fixtures/axis_collapse/746b3537.json new file mode 100644 index 00000000..e5fa16b1 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/axis_collapse/746b3537.json @@ -0,0 +1 @@ +{"train": [{"input": [[2, 2], [6, 6], [8, 8], [8, 8]], "output": [[2], [6], [8]]}, {"input": [[3, 4, 6], [3, 4, 6], [3, 4, 6]], "output": [[3, 4, 6]]}, {"input": [[2, 3, 3, 8, 1], [2, 3, 3, 8, 1], [2, 3, 3, 8, 1]], "output": [[2, 3, 8, 1]]}, {"input": [[4, 4, 4, 4], [4, 4, 4, 4], [2, 2, 2, 2], [2, 2, 2, 2], [8, 8, 8, 8], [3, 3, 3, 3]], "output": [[4], [2], [8], [3]]}, {"input": [[1, 1, 1], [2, 2, 2], [1, 1, 1]], "output": [[1], [2], [1]]}], "test": [{"input": [[1, 1, 2, 3, 3, 3, 8, 8, 4], [1, 1, 2, 3, 3, 3, 8, 8, 4], [1, 1, 2, 3, 3, 3, 8, 8, 4], [1, 1, 2, 3, 3, 3, 8, 8, 4]], "output": [[1, 2, 3, 8, 4]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/axis_collapse/7b7f7511.json b/research/universal_core/holonomy_v2_2/tests/fixtures/axis_collapse/7b7f7511.json new file mode 100644 index 00000000..621f8368 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/axis_collapse/7b7f7511.json @@ -0,0 +1 @@ +{"train": [{"input": [[4, 4, 4, 4, 4, 4], [6, 4, 8, 6, 4, 8], [6, 6, 8, 6, 6, 8]], "output": [[4, 4, 4], [6, 4, 8], [6, 6, 8]]}, {"input": [[1, 1, 3, 2, 1, 1, 3, 2], [1, 1, 3, 3, 1, 1, 3, 3], [3, 3, 1, 1, 3, 3, 1, 1], [2, 3, 1, 1, 2, 3, 1, 1]], "output": [[1, 1, 3, 2], [1, 1, 3, 3], [3, 3, 1, 1], [2, 3, 1, 1]]}, {"input": [[2, 3], [3, 2], [4, 4], [2, 3], [3, 2], [4, 4]], "output": [[2, 3], [3, 2], [4, 4]]}], "test": [{"input": [[5, 4, 5], [4, 5, 4], [6, 6, 4], [2, 6, 2], [5, 4, 5], [4, 5, 4], [6, 6, 4], [2, 6, 2]], "output": [[5, 4, 5], [4, 5, 4], [6, 6, 4], [2, 6, 2]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/axis_collapse/ce8d95cc.json b/research/universal_core/holonomy_v2_2/tests/fixtures/axis_collapse/ce8d95cc.json new file mode 100644 index 00000000..6353b56c --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/axis_collapse/ce8d95cc.json @@ -0,0 +1 @@ +{"train": [{"input": [[0, 0, 4, 0, 0, 0, 0, 0, 0], [0, 0, 4, 0, 0, 0, 0, 0, 0], [0, 0, 4, 0, 0, 0, 0, 0, 0], [0, 0, 4, 0, 0, 0, 0, 0, 0], [3, 3, 4, 3, 3, 3, 3, 3, 3], [0, 0, 4, 0, 0, 0, 0, 0, 0], [0, 0, 4, 0, 0, 0, 0, 0, 0], [0, 0, 4, 0, 0, 0, 0, 0, 0], [0, 0, 4, 0, 0, 0, 0, 0, 0], [8, 8, 8, 8, 8, 8, 8, 8, 8], [0, 0, 4, 0, 0, 0, 0, 0, 0], [0, 0, 4, 0, 0, 0, 0, 0, 0]], "output": [[0, 4, 0], [3, 4, 3], [0, 4, 0], [8, 8, 8], [0, 4, 0]]}, {"input": [[0, 0, 1, 0, 0, 8, 0, 3, 0, 0, 0], [0, 0, 1, 0, 0, 8, 0, 3, 0, 0, 0], [0, 0, 1, 0, 0, 8, 0, 3, 0, 0, 0], [2, 2, 1, 2, 2, 8, 2, 3, 2, 2, 2], [0, 0, 1, 0, 0, 8, 0, 3, 0, 0, 0], [0, 0, 1, 0, 0, 8, 0, 3, 0, 0, 0], [0, 0, 1, 0, 0, 8, 0, 3, 0, 0, 0], [0, 0, 1, 0, 0, 8, 0, 3, 0, 0, 0], [0, 0, 1, 0, 0, 8, 0, 3, 0, 0, 0], [5, 5, 1, 5, 5, 8, 5, 3, 5, 5, 5], [0, 0, 1, 0, 0, 8, 0, 3, 0, 0, 0], [0, 0, 1, 0, 0, 8, 0, 3, 0, 0, 0]], "output": [[0, 1, 0, 8, 0, 3, 0], [2, 1, 2, 8, 2, 3, 2], [0, 1, 0, 8, 0, 3, 0], [5, 1, 5, 8, 5, 3, 5], [0, 1, 0, 8, 0, 3, 0]]}, {"input": [[0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0], [7, 7, 3, 7, 7, 7, 7, 1, 7, 7, 7], [0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0], [0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0], [0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0], [0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0], [2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2], [0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0], [0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0], [0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0], [8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8], [0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0]], "output": [[0, 3, 0, 1, 0], [7, 3, 7, 1, 7], [0, 3, 0, 1, 0], [2, 2, 2, 1, 2], [0, 3, 0, 1, 0], [8, 8, 8, 8, 8], [0, 3, 0, 1, 0]]}, {"input": [[0, 0, 0, 8, 0, 0, 0, 0, 6, 0, 0], [3, 3, 3, 8, 3, 3, 3, 3, 6, 3, 3], [0, 0, 0, 8, 0, 0, 0, 0, 6, 0, 0], [0, 0, 0, 8, 0, 0, 0, 0, 6, 0, 0], [0, 0, 0, 8, 0, 0, 0, 0, 6, 0, 0], [0, 0, 0, 8, 0, 0, 0, 0, 6, 0, 0], [5, 5, 5, 8, 5, 5, 5, 5, 6, 5, 5], [0, 0, 0, 8, 0, 0, 0, 0, 6, 0, 0], [0, 0, 0, 8, 0, 0, 0, 0, 6, 0, 0], [0, 0, 0, 8, 0, 0, 0, 0, 6, 0, 0]], "output": [[0, 8, 0, 6, 0], [3, 8, 3, 6, 3], [0, 8, 0, 6, 0], [5, 8, 5, 6, 5], [0, 8, 0, 6, 0]]}], "test": [{"input": [[0, 0, 3, 0, 0, 2, 0, 7, 0, 0, 4, 0, 0], [0, 0, 3, 0, 0, 2, 0, 7, 0, 0, 4, 0, 0], [0, 0, 3, 0, 0, 2, 0, 7, 0, 0, 4, 0, 0], [6, 6, 6, 6, 6, 2, 6, 7, 6, 6, 4, 6, 6], [0, 0, 3, 0, 0, 2, 0, 7, 0, 0, 4, 0, 0], [0, 0, 3, 0, 0, 2, 0, 7, 0, 0, 4, 0, 0], [1, 1, 1, 1, 1, 2, 1, 7, 1, 1, 4, 1, 1], [0, 0, 3, 0, 0, 2, 0, 7, 0, 0, 4, 0, 0], [0, 0, 3, 0, 0, 2, 0, 7, 0, 0, 4, 0, 0], [8, 8, 8, 8, 8, 8, 8, 7, 8, 8, 4, 8, 8], [0, 0, 3, 0, 0, 2, 0, 7, 0, 0, 4, 0, 0], [0, 0, 3, 0, 0, 2, 0, 7, 0, 0, 4, 0, 0]], "output": [[0, 3, 0, 2, 0, 7, 0, 4, 0], [6, 6, 6, 2, 6, 7, 6, 4, 6], [0, 3, 0, 2, 0, 7, 0, 4, 0], [1, 1, 1, 2, 1, 7, 1, 4, 1], [0, 3, 0, 2, 0, 7, 0, 4, 0], [8, 8, 8, 8, 8, 7, 8, 4, 8], [0, 3, 0, 2, 0, 7, 0, 4, 0]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/axis_collapse/e1baa8a4.json b/research/universal_core/holonomy_v2_2/tests/fixtures/axis_collapse/e1baa8a4.json new file mode 100644 index 00000000..6fc4a77c --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/axis_collapse/e1baa8a4.json @@ -0,0 +1 @@ +{"train": [{"input": [[1, 1, 1, 2, 2, 2, 2, 2, 8, 8, 8, 8, 8, 8], [1, 1, 1, 2, 2, 2, 2, 2, 8, 8, 8, 8, 8, 8], [1, 1, 1, 2, 2, 2, 2, 2, 8, 8, 8, 8, 8, 8], [1, 1, 1, 2, 2, 2, 2, 2, 8, 8, 8, 8, 8, 8], [1, 1, 1, 2, 2, 2, 2, 2, 8, 8, 8, 8, 8, 8], [5, 5, 5, 6, 6, 6, 6, 6, 3, 3, 3, 3, 3, 3], [5, 5, 5, 6, 6, 6, 6, 6, 3, 3, 3, 3, 3, 3], [5, 5, 5, 6, 6, 6, 6, 6, 3, 3, 3, 3, 3, 3], [5, 5, 5, 6, 6, 6, 6, 6, 3, 3, 3, 3, 3, 3], [5, 5, 5, 6, 6, 6, 6, 6, 3, 3, 3, 3, 3, 3], [5, 5, 5, 6, 6, 6, 6, 6, 3, 3, 3, 3, 3, 3], [5, 5, 5, 6, 6, 6, 6, 6, 3, 3, 3, 3, 3, 3], [5, 5, 5, 6, 6, 6, 6, 6, 3, 3, 3, 3, 3, 3], [5, 5, 5, 6, 6, 6, 6, 6, 3, 3, 3, 3, 3, 3], [5, 5, 5, 6, 6, 6, 6, 6, 3, 3, 3, 3, 3, 3]], "output": [[1, 2, 8], [5, 6, 3]]}, {"input": [[4, 4, 4, 4, 4, 5, 5, 5, 2, 2, 2, 2, 2, 2], [4, 4, 4, 4, 4, 5, 5, 5, 2, 2, 2, 2, 2, 2], [4, 4, 4, 4, 4, 5, 5, 5, 2, 2, 2, 2, 2, 2], [4, 4, 4, 4, 4, 5, 5, 5, 2, 2, 2, 2, 2, 2], [1, 1, 1, 1, 1, 3, 3, 3, 2, 2, 2, 2, 2, 2], [1, 1, 1, 1, 1, 3, 3, 3, 2, 2, 2, 2, 2, 2], [1, 1, 1, 1, 1, 3, 3, 3, 2, 2, 2, 2, 2, 2], [1, 1, 1, 1, 1, 3, 3, 3, 2, 2, 2, 2, 2, 2], [1, 1, 1, 1, 1, 3, 3, 3, 2, 2, 2, 2, 2, 2], [1, 1, 1, 1, 1, 3, 3, 3, 2, 2, 2, 2, 2, 2], [1, 1, 1, 1, 1, 3, 3, 3, 2, 2, 2, 2, 2, 2], [1, 1, 1, 1, 1, 3, 3, 3, 2, 2, 2, 2, 2, 2]], "output": [[4, 5, 2], [1, 3, 2]]}, {"input": [[8, 8, 8, 7, 7, 7, 7, 7, 9, 9, 9, 9, 8, 8, 8], [8, 8, 8, 7, 7, 7, 7, 7, 9, 9, 9, 9, 8, 8, 8], [8, 8, 8, 7, 7, 7, 7, 7, 9, 9, 9, 9, 8, 8, 8], [8, 8, 8, 7, 7, 7, 7, 7, 9, 9, 9, 9, 8, 8, 8], [3, 3, 3, 1, 1, 1, 1, 1, 6, 6, 6, 6, 4, 4, 4], [3, 3, 3, 1, 1, 1, 1, 1, 6, 6, 6, 6, 4, 4, 4], [3, 3, 3, 1, 1, 1, 1, 1, 6, 6, 6, 6, 4, 4, 4], [3, 3, 3, 1, 1, 1, 1, 1, 6, 6, 6, 6, 4, 4, 4], [3, 3, 3, 1, 1, 1, 1, 1, 6, 6, 6, 6, 4, 4, 4], [2, 2, 2, 4, 4, 4, 4, 4, 1, 1, 1, 1, 5, 5, 5], [2, 2, 2, 4, 4, 4, 4, 4, 1, 1, 1, 1, 5, 5, 5], [2, 2, 2, 4, 4, 4, 4, 4, 1, 1, 1, 1, 5, 5, 5], [2, 2, 2, 4, 4, 4, 4, 4, 1, 1, 1, 1, 5, 5, 5], [2, 2, 2, 4, 4, 4, 4, 4, 1, 1, 1, 1, 5, 5, 5], [2, 2, 2, 4, 4, 4, 4, 4, 1, 1, 1, 1, 5, 5, 5]], "output": [[8, 7, 9, 8], [3, 1, 6, 4], [2, 4, 1, 5]]}, {"input": [[2, 2, 2, 2, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8], [2, 2, 2, 2, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8], [2, 2, 2, 2, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8], [2, 2, 2, 2, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8], [2, 2, 2, 2, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8], [2, 2, 2, 2, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8], [2, 2, 2, 2, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8], [2, 2, 2, 2, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8], [2, 2, 2, 2, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8], [3, 3, 3, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5], [3, 3, 3, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5], [3, 3, 3, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5], [3, 3, 3, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5]], "output": [[2, 8], [3, 5]]}], "test": [{"input": [[8, 8, 8, 8, 7, 7, 7, 7, 4, 4, 4, 4, 4, 8, 8], [8, 8, 8, 8, 7, 7, 7, 7, 4, 4, 4, 4, 4, 8, 8], [3, 3, 3, 3, 1, 1, 1, 1, 2, 2, 2, 2, 2, 8, 8], [3, 3, 3, 3, 1, 1, 1, 1, 2, 2, 2, 2, 2, 8, 8], [3, 3, 3, 3, 1, 1, 1, 1, 2, 2, 2, 2, 2, 8, 8], [4, 4, 4, 4, 5, 5, 5, 5, 3, 3, 3, 3, 3, 9, 9], [4, 4, 4, 4, 5, 5, 5, 5, 3, 3, 3, 3, 3, 9, 9], [4, 4, 4, 4, 5, 5, 5, 5, 3, 3, 3, 3, 3, 9, 9], [4, 4, 4, 4, 5, 5, 5, 5, 3, 3, 3, 3, 3, 9, 9], [4, 4, 4, 4, 5, 5, 5, 5, 3, 3, 3, 3, 3, 9, 9], [2, 2, 2, 2, 6, 6, 6, 6, 1, 1, 1, 1, 1, 7, 7], [2, 2, 2, 2, 6, 6, 6, 6, 1, 1, 1, 1, 1, 7, 7], [1, 1, 1, 1, 5, 5, 5, 5, 2, 2, 2, 2, 2, 8, 8], [1, 1, 1, 1, 5, 5, 5, 5, 2, 2, 2, 2, 2, 8, 8], [1, 1, 1, 1, 5, 5, 5, 5, 2, 2, 2, 2, 2, 8, 8]], "output": [[8, 7, 4, 8], [3, 1, 2, 8], [4, 5, 3, 9], [2, 6, 1, 7], [1, 5, 2, 8]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/axis_collapse/eb5a1d5d.json b/research/universal_core/holonomy_v2_2/tests/fixtures/axis_collapse/eb5a1d5d.json new file mode 100644 index 00000000..59d0aec8 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/axis_collapse/eb5a1d5d.json @@ -0,0 +1 @@ +{"train": [{"input": [[5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5], [5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5], [5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5], [5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5], [5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5], [5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5], [5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5], [5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5], [5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5], [5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5], [5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5], [5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5], [5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5], [5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5]], "output": [[5, 5, 5], [5, 6, 5], [5, 5, 5]]}, {"input": [[8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8], [8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8], [8, 8, 8, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 8, 8, 8, 8, 8, 8], [8, 8, 8, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 8, 8, 8, 8, 8, 8], [8, 8, 8, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 8, 8, 8, 8, 8, 8], [8, 8, 8, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 8, 8, 8, 8, 8, 8], [8, 8, 8, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 8, 8, 8, 8, 8, 8], [8, 8, 8, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 8, 8, 8, 8, 8, 8], [8, 8, 8, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 8, 8, 8, 8, 8, 8], [8, 8, 8, 3, 3, 3, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 8, 8, 8, 8, 8, 8], [8, 8, 8, 3, 3, 3, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 8, 8, 8, 8, 8, 8], [8, 8, 8, 3, 3, 3, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 8, 8, 8, 8, 8, 8], [8, 8, 8, 3, 3, 3, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 8, 8, 8, 8, 8, 8], [8, 8, 8, 3, 3, 3, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 8, 8, 8, 8, 8, 8], [8, 8, 8, 3, 3, 3, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 8, 8, 8, 8, 8, 8], [8, 8, 8, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 8, 8, 8, 8, 8, 8], [8, 8, 8, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 8, 8, 8, 8, 8, 8], [8, 8, 8, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 8, 8, 8, 8, 8, 8], [8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8], [8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8], [8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8], [8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8], [8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8]], "output": [[8, 8, 8, 8, 8], [8, 3, 3, 3, 8], [8, 3, 2, 3, 8], [8, 3, 3, 3, 8], [8, 8, 8, 8, 8]]}, {"input": [[3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], [3, 3, 3, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 3, 3, 3, 3, 3], [3, 3, 3, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 3, 3, 3, 3, 3], [3, 3, 3, 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 8, 8, 8, 3, 3, 3, 3, 3], [3, 3, 3, 8, 8, 2, 2, 2, 1, 1, 1, 1, 1, 2, 8, 8, 8, 3, 3, 3, 3, 3], [3, 3, 3, 8, 8, 2, 2, 2, 1, 1, 1, 1, 1, 2, 8, 8, 8, 3, 3, 3, 3, 3], [3, 3, 3, 8, 8, 2, 2, 2, 1, 1, 1, 1, 1, 2, 8, 8, 8, 3, 3, 3, 3, 3], [3, 3, 3, 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 8, 8, 8, 3, 3, 3, 3, 3], [3, 3, 3, 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 8, 8, 8, 3, 3, 3, 3, 3], [3, 3, 3, 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 8, 8, 8, 3, 3, 3, 3, 3], [3, 3, 3, 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 8, 8, 8, 3, 3, 3, 3, 3], [3, 3, 3, 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 8, 8, 8, 3, 3, 3, 3, 3], [3, 3, 3, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 3, 3, 3, 3, 3], [3, 3, 3, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 3, 3, 3, 3, 3], [3, 3, 3, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 3, 3, 3, 3, 3], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]], "output": [[3, 3, 3, 3, 3, 3, 3], [3, 8, 8, 8, 8, 8, 3], [3, 8, 2, 2, 2, 8, 3], [3, 8, 2, 1, 2, 8, 3], [3, 8, 2, 2, 2, 8, 3], [3, 8, 8, 8, 8, 8, 3], [3, 3, 3, 3, 3, 3, 3]]}], "test": [{"input": [[2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2], [2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2], [2, 2, 2, 1, 1, 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 2, 2, 2, 2], [2, 2, 2, 1, 1, 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 2, 2, 2, 2], [2, 2, 2, 1, 1, 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 2, 2, 2, 2], [2, 2, 2, 1, 1, 1, 3, 3, 3, 8, 8, 8, 8, 8, 8, 8, 8, 8, 3, 3, 1, 1, 1, 2, 2, 2, 2], [2, 2, 2, 1, 1, 1, 3, 3, 3, 8, 8, 8, 2, 2, 2, 2, 8, 8, 3, 3, 1, 1, 1, 2, 2, 2, 2], [2, 2, 2, 1, 1, 1, 3, 3, 3, 8, 8, 8, 2, 2, 2, 2, 8, 8, 3, 3, 1, 1, 1, 2, 2, 2, 2], [2, 2, 2, 1, 1, 1, 3, 3, 3, 8, 8, 8, 2, 2, 2, 2, 8, 8, 3, 3, 1, 1, 1, 2, 2, 2, 2], [2, 2, 2, 1, 1, 1, 3, 3, 3, 8, 8, 8, 2, 2, 2, 2, 8, 8, 3, 3, 1, 1, 1, 2, 2, 2, 2], [2, 2, 2, 1, 1, 1, 3, 3, 3, 8, 8, 8, 8, 8, 8, 8, 8, 8, 3, 3, 1, 1, 1, 2, 2, 2, 2], [2, 2, 2, 1, 1, 1, 3, 3, 3, 8, 8, 8, 8, 8, 8, 8, 8, 8, 3, 3, 1, 1, 1, 2, 2, 2, 2], [2, 2, 2, 1, 1, 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 2, 2, 2, 2], [2, 2, 2, 1, 1, 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 2, 2, 2, 2], [2, 2, 2, 1, 1, 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 2, 2, 2, 2], [2, 2, 2, 1, 1, 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 2, 2, 2, 2], [2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2], [2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2], [2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2], [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]], "output": [[2, 2, 2, 2, 2, 2, 2, 2, 2], [2, 1, 1, 1, 1, 1, 1, 1, 2], [2, 1, 3, 3, 3, 3, 3, 1, 2], [2, 1, 3, 8, 8, 8, 3, 1, 2], [2, 1, 3, 8, 2, 8, 3, 1, 2], [2, 1, 3, 8, 8, 8, 3, 1, 2], [2, 1, 3, 3, 3, 3, 3, 1, 2], [2, 1, 1, 1, 1, 1, 1, 1, 2], [2, 2, 2, 2, 2, 2, 2, 2, 2]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/bbox/3aa6fb7a.json b/research/universal_core/holonomy_v2_2/tests/fixtures/bbox/3aa6fb7a.json new file mode 100644 index 00000000..7bdea593 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/bbox/3aa6fb7a.json @@ -0,0 +1 @@ +{"train": [{"input": [[0, 0, 0, 0, 8, 8, 0], [0, 0, 0, 0, 0, 8, 0], [0, 0, 8, 0, 0, 0, 0], [0, 0, 8, 8, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 8, 0, 0], [0, 0, 0, 8, 8, 0, 0]], "output": [[0, 0, 0, 0, 8, 8, 0], [0, 0, 0, 0, 1, 8, 0], [0, 0, 8, 1, 0, 0, 0], [0, 0, 8, 8, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 1, 8, 0, 0], [0, 0, 0, 8, 8, 0, 0]]}, {"input": [[0, 0, 0, 0, 0, 0, 0], [0, 8, 0, 0, 0, 0, 0], [0, 8, 8, 0, 0, 0, 0], [0, 0, 0, 0, 8, 8, 0], [0, 0, 0, 0, 0, 8, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0]], "output": [[0, 0, 0, 0, 0, 0, 0], [0, 8, 1, 0, 0, 0, 0], [0, 8, 8, 0, 0, 0, 0], [0, 0, 0, 0, 8, 8, 0], [0, 0, 0, 0, 1, 8, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0]]}], "test": [{"input": [[0, 0, 0, 0, 0, 8, 8], [8, 8, 0, 0, 0, 0, 8], [8, 0, 0, 0, 0, 0, 0], [0, 0, 0, 8, 0, 0, 0], [0, 0, 0, 8, 8, 0, 0], [0, 8, 0, 0, 0, 0, 0], [8, 8, 0, 0, 0, 0, 0]], "output": [[0, 0, 0, 0, 0, 8, 8], [8, 8, 0, 0, 0, 1, 8], [8, 1, 0, 0, 0, 0, 0], [0, 0, 0, 8, 1, 0, 0], [0, 0, 0, 8, 8, 0, 0], [1, 8, 0, 0, 0, 0, 0], [8, 8, 0, 0, 0, 0, 0]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/bbox/60b61512.json b/research/universal_core/holonomy_v2_2/tests/fixtures/bbox/60b61512.json new file mode 100644 index 00000000..30e4372b --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/bbox/60b61512.json @@ -0,0 +1 @@ +{"train": [{"input": [[0, 0, 0, 0, 0, 0, 0, 0, 0], [4, 4, 4, 0, 0, 0, 0, 0, 0], [0, 4, 4, 0, 0, 0, 0, 0, 0], [4, 4, 4, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 4, 4, 4, 0], [0, 0, 0, 0, 0, 0, 4, 0, 0], [0, 0, 0, 0, 0, 0, 4, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0]], "output": [[0, 0, 0, 0, 0, 0, 0, 0, 0], [4, 4, 4, 0, 0, 0, 0, 0, 0], [7, 4, 4, 0, 0, 0, 0, 0, 0], [4, 4, 4, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 4, 4, 4, 0], [0, 0, 0, 0, 0, 7, 4, 7, 0], [0, 0, 0, 0, 0, 7, 4, 7, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0]]}, {"input": [[0, 0, 0, 0, 0, 0, 0, 0, 0], [4, 4, 4, 0, 0, 0, 0, 0, 0], [4, 0, 4, 0, 0, 0, 0, 0, 0], [0, 0, 4, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 4, 4, 0, 0], [0, 0, 0, 0, 0, 0, 4, 4, 0], [0, 0, 0, 0, 0, 4, 0, 4, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0]], "output": [[0, 0, 0, 0, 0, 0, 0, 0, 0], [4, 4, 4, 0, 0, 0, 0, 0, 0], [4, 7, 4, 0, 0, 0, 0, 0, 0], [7, 7, 4, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 4, 4, 7, 0], [0, 0, 0, 0, 0, 7, 4, 4, 0], [0, 0, 0, 0, 0, 4, 7, 4, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0]]}], "test": [{"input": [[0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 4, 0, 0, 0, 0], [0, 0, 4, 4, 0, 0, 0, 0, 0], [0, 0, 4, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 4, 0, 0, 0], [0, 0, 0, 0, 0, 4, 4, 4, 0], [0, 0, 0, 0, 0, 0, 4, 0, 0]], "output": [[0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 7, 7, 4, 0, 0, 0, 0], [0, 0, 4, 4, 7, 0, 0, 0, 0], [0, 0, 4, 7, 7, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 4, 7, 7, 0], [0, 0, 0, 0, 0, 4, 4, 4, 0], [0, 0, 0, 0, 0, 7, 4, 7, 0]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/bbox/6d75e8bb.json b/research/universal_core/holonomy_v2_2/tests/fixtures/bbox/6d75e8bb.json new file mode 100644 index 00000000..9eef7408 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/bbox/6d75e8bb.json @@ -0,0 +1 @@ +{"train": [{"input": [[0, 0, 0, 0, 0, 0, 0, 0], [0, 8, 8, 8, 8, 8, 8, 0], [0, 8, 0, 8, 8, 0, 8, 0], [0, 8, 0, 8, 0, 0, 8, 0], [0, 0, 0, 8, 0, 8, 8, 0], [0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0]], "output": [[0, 0, 0, 0, 0, 0, 0, 0], [0, 8, 8, 8, 8, 8, 8, 0], [0, 8, 2, 8, 8, 2, 8, 0], [0, 8, 2, 8, 2, 2, 8, 0], [0, 2, 2, 8, 2, 8, 8, 0], [0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0]]}, {"input": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 8, 8, 8, 0, 0, 0, 0, 0, 0], [0, 8, 0, 0, 0, 0, 0, 0, 0, 0], [0, 8, 8, 8, 8, 0, 0, 0, 0, 0], [0, 8, 8, 0, 0, 0, 0, 0, 0, 0], [0, 8, 8, 8, 0, 0, 0, 0, 0, 0], [0, 8, 0, 0, 0, 0, 0, 0, 0, 0], [0, 8, 8, 8, 0, 0, 0, 0, 0, 0], [0, 8, 8, 8, 0, 0, 0, 0, 0, 0], [0, 8, 8, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "output": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 8, 8, 8, 2, 0, 0, 0, 0, 0], [0, 8, 2, 2, 2, 0, 0, 0, 0, 0], [0, 8, 8, 8, 8, 0, 0, 0, 0, 0], [0, 8, 8, 2, 2, 0, 0, 0, 0, 0], [0, 8, 8, 8, 2, 0, 0, 0, 0, 0], [0, 8, 2, 2, 2, 0, 0, 0, 0, 0], [0, 8, 8, 8, 2, 0, 0, 0, 0, 0], [0, 8, 8, 8, 2, 0, 0, 0, 0, 0], [0, 8, 8, 2, 2, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}, {"input": [[0, 0, 0, 0, 0, 0, 0, 0], [0, 8, 8, 8, 8, 8, 0, 0], [0, 0, 0, 8, 0, 8, 0, 0], [0, 0, 8, 8, 8, 8, 0, 0], [0, 0, 0, 8, 8, 8, 0, 0], [0, 0, 0, 0, 8, 8, 0, 0], [0, 0, 0, 8, 8, 8, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0]], "output": [[0, 0, 0, 0, 0, 0, 0, 0], [0, 8, 8, 8, 8, 8, 0, 0], [0, 2, 2, 8, 2, 8, 0, 0], [0, 2, 8, 8, 8, 8, 0, 0], [0, 2, 2, 8, 8, 8, 0, 0], [0, 2, 2, 2, 8, 8, 0, 0], [0, 2, 2, 8, 8, 8, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0]]}], "test": [{"input": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 8, 0, 0, 0, 8, 8, 0, 0, 0], [0, 0, 8, 0, 8, 0, 0, 8, 0, 0, 0], [0, 0, 8, 8, 8, 0, 0, 8, 0, 0, 0], [0, 0, 8, 8, 8, 8, 0, 8, 8, 0, 0], [0, 0, 8, 8, 8, 8, 8, 8, 8, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "output": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 8, 2, 2, 2, 2, 2, 2, 0, 0], [0, 0, 8, 2, 2, 2, 8, 8, 2, 0, 0], [0, 0, 8, 2, 8, 2, 2, 8, 2, 0, 0], [0, 0, 8, 8, 8, 2, 2, 8, 2, 0, 0], [0, 0, 8, 8, 8, 8, 2, 8, 8, 0, 0], [0, 0, 8, 8, 8, 8, 8, 8, 8, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/bbox/e7639916.json b/research/universal_core/holonomy_v2_2/tests/fixtures/bbox/e7639916.json new file mode 100644 index 00000000..69798b13 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/bbox/e7639916.json @@ -0,0 +1 @@ +{"train": [{"input": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "output": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 1, 8, 1, 1, 1, 1, 1, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 0, 8, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0], [0, 0, 8, 1, 1, 1, 1, 1, 1, 1, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}, {"input": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "output": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0], [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0], [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0], [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0], [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0], [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0], [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0], [0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 8, 1, 1, 1, 1, 1, 1, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}, {"input": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "output": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 1, 1, 1, 1, 1, 1, 8, 1, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0], [0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0], [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 8, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}], "test": [{"input": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "output": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 8, 1, 1, 1, 1, 1, 1, 1, 1, 0], [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0], [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8, 0], [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0], [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0], [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0], [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0], [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0], [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0], [0, 0, 0, 1, 1, 1, 1, 1, 8, 1, 1, 1, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/block_reduce/5614dbcf.json b/research/universal_core/holonomy_v2_2/tests/fixtures/block_reduce/5614dbcf.json new file mode 100644 index 00000000..7551d83b --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/block_reduce/5614dbcf.json @@ -0,0 +1 @@ +{"train": [{"input": [[0, 0, 0, 2, 2, 2, 0, 0, 0], [0, 5, 0, 2, 2, 2, 0, 0, 0], [0, 0, 0, 2, 2, 2, 0, 0, 0], [5, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 5, 0, 0, 0, 5, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 5, 0, 7, 7, 7, 0, 0, 0], [0, 0, 0, 7, 7, 5, 0, 0, 0], [0, 0, 0, 7, 7, 7, 0, 0, 0]], "output": [[0, 2, 0], [0, 0, 0], [0, 7, 0]]}, {"input": [[3, 3, 3, 0, 0, 0, 8, 8, 8], [3, 3, 3, 0, 0, 0, 8, 5, 8], [3, 3, 3, 0, 0, 0, 8, 8, 8], [0, 0, 0, 7, 5, 7, 0, 0, 0], [0, 0, 0, 7, 7, 7, 0, 0, 0], [0, 0, 0, 7, 7, 7, 0, 0, 0], [6, 6, 6, 0, 0, 5, 9, 9, 9], [6, 6, 6, 0, 0, 0, 9, 9, 9], [6, 5, 6, 0, 5, 0, 9, 9, 5]], "output": [[3, 0, 8], [0, 7, 0], [6, 0, 9]]}], "test": [{"input": [[4, 4, 4, 0, 0, 0, 0, 5, 0], [5, 4, 4, 0, 0, 0, 0, 0, 0], [4, 4, 4, 0, 5, 0, 0, 0, 0], [0, 0, 0, 3, 3, 3, 0, 5, 0], [0, 0, 0, 3, 3, 3, 0, 0, 0], [0, 0, 0, 3, 3, 3, 0, 0, 0], [0, 0, 5, 9, 9, 9, 0, 0, 0], [0, 0, 0, 9, 5, 9, 0, 0, 0], [0, 0, 0, 9, 9, 9, 0, 0, 0]], "output": [[4, 0, 0], [0, 3, 0], [0, 9, 0]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/block_reduce/68b67ca3.json b/research/universal_core/holonomy_v2_2/tests/fixtures/block_reduce/68b67ca3.json new file mode 100644 index 00000000..e8473c83 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/block_reduce/68b67ca3.json @@ -0,0 +1 @@ +{"train": [{"input": [[3, 0, 0, 0, 2, 0], [0, 0, 0, 0, 0, 0], [8, 0, 0, 0, 8, 0], [0, 0, 0, 0, 0, 0], [0, 0, 1, 0, 0, 0], [0, 0, 0, 0, 0, 0]], "output": [[3, 0, 2], [8, 0, 8], [0, 1, 0]]}, {"input": [[2, 0, 2, 0, 1, 0], [0, 0, 0, 0, 0, 0], [2, 0, 1, 0, 0, 0], [0, 0, 0, 0, 0, 0], [3, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0]], "output": [[2, 2, 1], [2, 1, 0], [3, 0, 0]]}, {"input": [[1, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 2, 0, 0, 0], [0, 0, 0, 0, 0, 0], [6, 0, 0, 0, 6, 0], [0, 0, 0, 0, 0, 0]], "output": [[1, 0, 0], [0, 2, 0], [6, 0, 6]]}], "test": [{"input": [[3, 0, 3, 0, 4, 0], [0, 0, 0, 0, 0, 0], [7, 0, 0, 0, 1, 0], [0, 0, 0, 0, 0, 0], [7, 0, 0, 0, 1, 0], [0, 0, 0, 0, 0, 0]], "output": [[3, 3, 4], [7, 0, 1], [7, 0, 1]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/gravity_compact/1e0a9b12.json b/research/universal_core/holonomy_v2_2/tests/fixtures/gravity_compact/1e0a9b12.json new file mode 100644 index 00000000..adef9ad0 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/gravity_compact/1e0a9b12.json @@ -0,0 +1 @@ +{"train": [{"input": [[0, 0, 0, 0, 0, 9], [0, 0, 0, 8, 0, 0], [0, 0, 0, 0, 0, 0], [4, 0, 0, 0, 0, 0], [4, 0, 7, 8, 0, 0], [4, 0, 7, 0, 0, 0]], "output": [[0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [4, 0, 0, 0, 0, 0], [4, 0, 7, 8, 0, 0], [4, 0, 7, 8, 0, 9]]}, {"input": [[0, 4, 0, 9], [0, 0, 0, 0], [0, 4, 6, 0], [1, 0, 0, 0]], "output": [[0, 0, 0, 0], [0, 0, 0, 0], [0, 4, 0, 0], [1, 4, 6, 9]]}, {"input": [[0, 0, 0, 1, 0], [0, 3, 0, 0, 0], [0, 3, 0, 1, 2], [6, 0, 0, 0, 0], [0, 3, 0, 0, 0]], "output": [[0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 3, 0, 0, 0], [0, 3, 0, 1, 0], [6, 3, 0, 1, 2]]}], "test": [{"input": [[0, 2, 0, 4, 3], [5, 0, 0, 0, 0], [0, 0, 6, 0, 0], [5, 2, 0, 4, 0], [5, 0, 0, 0, 0]], "output": [[0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [5, 0, 0, 0, 0], [5, 2, 0, 4, 0], [5, 2, 6, 4, 3]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/gravity_compact/3906de3d.json b/research/universal_core/holonomy_v2_2/tests/fixtures/gravity_compact/3906de3d.json new file mode 100644 index 00000000..e7c83cf2 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/gravity_compact/3906de3d.json @@ -0,0 +1 @@ +{"train": [{"input": [[0, 0, 1, 1, 1, 1, 1, 1, 1, 0], [0, 0, 1, 1, 0, 1, 1, 1, 1, 0], [0, 0, 1, 1, 0, 1, 0, 1, 1, 0], [0, 0, 1, 1, 0, 1, 0, 1, 1, 0], [0, 0, 0, 0, 0, 0, 0, 1, 1, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 2, 0, 0, 0], [0, 0, 0, 0, 0, 0, 2, 0, 0, 0], [0, 0, 0, 0, 0, 0, 2, 0, 0, 0], [0, 0, 0, 0, 2, 0, 2, 0, 0, 0]], "output": [[0, 0, 1, 1, 1, 1, 1, 1, 1, 0], [0, 0, 1, 1, 2, 1, 1, 1, 1, 0], [0, 0, 1, 1, 0, 1, 2, 1, 1, 0], [0, 0, 1, 1, 0, 1, 2, 1, 1, 0], [0, 0, 0, 0, 0, 0, 2, 1, 1, 0], [0, 0, 0, 0, 0, 0, 2, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}, {"input": [[0, 0, 1, 1, 1, 1, 1, 0, 0, 0], [0, 0, 1, 1, 1, 1, 1, 0, 0, 0], [0, 0, 1, 1, 0, 1, 1, 0, 0, 0], [0, 0, 1, 1, 0, 1, 1, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 2, 0, 0, 0, 0, 0], [0, 0, 0, 0, 2, 0, 0, 0, 0, 0], [0, 0, 0, 0, 2, 0, 0, 0, 0, 0]], "output": [[0, 0, 1, 1, 1, 1, 1, 0, 0, 0], [0, 0, 1, 1, 1, 1, 1, 0, 0, 0], [0, 0, 1, 1, 2, 1, 1, 0, 0, 0], [0, 0, 1, 1, 2, 1, 1, 0, 0, 0], [0, 0, 0, 0, 2, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}, {"input": [[0, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 1, 1, 0, 1, 1, 1, 1, 1, 1], [0, 1, 1, 0, 1, 1, 1, 1, 0, 1], [0, 1, 1, 0, 0, 1, 0, 1, 0, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 2, 0, 0, 0], [0, 0, 0, 2, 0, 0, 2, 0, 2, 0], [0, 0, 0, 2, 2, 0, 2, 0, 2, 0], [0, 0, 0, 2, 2, 0, 2, 0, 2, 0]], "output": [[0, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 1, 1, 2, 1, 1, 1, 1, 1, 1], [0, 1, 1, 2, 1, 1, 1, 1, 2, 1], [0, 1, 1, 2, 2, 1, 2, 1, 2, 1], [0, 0, 0, 0, 2, 0, 2, 0, 2, 0], [0, 0, 0, 0, 0, 0, 2, 0, 0, 0], [0, 0, 0, 0, 0, 0, 2, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}], "test": [{"input": [[0, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 1, 0, 1, 1, 1, 1, 1, 0, 1], [0, 1, 0, 1, 0, 1, 0, 1, 0, 1], [0, 1, 0, 1, 0, 0, 0, 1, 0, 1], [0, 0, 0, 1, 0, 0, 0, 0, 2, 1], [0, 0, 0, 0, 0, 0, 0, 0, 2, 0], [0, 0, 0, 0, 0, 2, 0, 0, 2, 0], [0, 0, 2, 0, 0, 2, 0, 0, 2, 0], [0, 0, 2, 0, 2, 2, 0, 0, 2, 0], [0, 0, 2, 0, 2, 2, 2, 0, 2, 0]], "output": [[0, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 1, 2, 1, 1, 1, 1, 1, 2, 1], [0, 1, 2, 1, 2, 1, 2, 1, 2, 1], [0, 1, 2, 1, 2, 2, 0, 1, 2, 1], [0, 0, 0, 1, 0, 2, 0, 0, 2, 1], [0, 0, 0, 0, 0, 2, 0, 0, 2, 0], [0, 0, 0, 0, 0, 2, 0, 0, 2, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/line_connect/1f876c06.json b/research/universal_core/holonomy_v2_2/tests/fixtures/line_connect/1f876c06.json new file mode 100644 index 00000000..5914e991 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/line_connect/1f876c06.json @@ -0,0 +1 @@ +{"train": [{"input": [[9, 0, 0, 0, 0, 0, 0, 3, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 8, 0, 0, 3], [0, 0, 0, 9, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 7, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 8, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 7]], "output": [[9, 0, 0, 0, 0, 0, 0, 3, 0, 0], [0, 9, 0, 0, 0, 0, 0, 0, 3, 0], [0, 0, 9, 0, 0, 0, 8, 0, 0, 3], [0, 0, 0, 9, 0, 8, 0, 0, 0, 0], [0, 0, 0, 0, 8, 0, 0, 0, 0, 0], [0, 0, 0, 8, 0, 7, 0, 0, 0, 0], [0, 0, 8, 0, 0, 0, 7, 0, 0, 0], [0, 8, 0, 0, 0, 0, 0, 7, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 7, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 7]]}, {"input": [[0, 0, 2, 0, 0, 6, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 6], [0, 0, 0, 4, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 4, 0, 0]], "output": [[0, 0, 2, 0, 0, 6, 0, 0, 0, 0], [0, 2, 0, 0, 0, 0, 6, 0, 0, 0], [2, 0, 0, 0, 0, 0, 0, 6, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 6, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 6], [0, 0, 0, 4, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 4, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 4, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 4, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 4, 0, 0]]}, {"input": [[0, 0, 0, 6, 0, 8, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 4, 0, 0, 0, 0, 0, 0, 0], [6, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 8], [0, 0, 0, 0, 9, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 4, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [9, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "output": [[0, 0, 0, 6, 0, 8, 0, 0, 0, 0], [0, 0, 6, 0, 0, 0, 8, 0, 0, 0], [0, 6, 4, 0, 0, 0, 0, 8, 0, 0], [6, 0, 0, 4, 0, 0, 0, 0, 8, 0], [0, 0, 0, 0, 4, 0, 0, 0, 0, 8], [0, 0, 0, 0, 9, 4, 0, 0, 0, 0], [0, 0, 0, 9, 0, 0, 4, 0, 0, 0], [0, 0, 9, 0, 0, 0, 0, 0, 0, 0], [0, 9, 0, 0, 0, 0, 0, 0, 0, 0], [9, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}], "test": [{"input": [[0, 0, 0, 0, 0, 0, 3, 0, 0, 9], [7, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 3, 0, 0, 9, 0, 0, 0], [6, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 4], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 6, 0, 0, 7, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 4, 0, 0, 0, 0]], "output": [[0, 0, 0, 0, 0, 0, 3, 0, 0, 9], [7, 0, 0, 0, 0, 3, 0, 0, 9, 0], [0, 7, 0, 0, 3, 0, 0, 9, 0, 0], [0, 0, 7, 3, 0, 0, 9, 0, 0, 0], [6, 0, 0, 7, 0, 0, 0, 0, 0, 0], [0, 6, 0, 0, 7, 0, 0, 0, 0, 4], [0, 0, 6, 0, 0, 7, 0, 0, 4, 0], [0, 0, 0, 6, 0, 0, 7, 4, 0, 0], [0, 0, 0, 0, 0, 0, 4, 0, 0, 0], [0, 0, 0, 0, 0, 4, 0, 0, 0, 0]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/line_connect/22168020.json b/research/universal_core/holonomy_v2_2/tests/fixtures/line_connect/22168020.json new file mode 100644 index 00000000..d1d44ca5 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/line_connect/22168020.json @@ -0,0 +1 @@ +{"train": [{"input": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 0, 0, 0, 0, 1, 0, 0, 0, 0], [0, 1, 0, 0, 1, 0, 0, 0, 0, 0], [0, 0, 1, 1, 0, 0, 0, 0, 0, 0], [0, 0, 1, 1, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 4, 0, 0, 0, 0, 4], [0, 0, 0, 0, 0, 4, 0, 0, 4, 0], [0, 0, 0, 0, 0, 0, 4, 4, 0, 0], [0, 0, 0, 0, 0, 0, 4, 4, 0, 0]], "output": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 0, 0, 0, 0], [0, 1, 1, 1, 1, 0, 0, 0, 0, 0], [0, 0, 1, 1, 0, 0, 0, 0, 0, 0], [0, 0, 1, 1, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 4, 4, 4, 4, 4, 4], [0, 0, 0, 0, 0, 4, 4, 4, 4, 0], [0, 0, 0, 0, 0, 0, 4, 4, 0, 0], [0, 0, 0, 0, 0, 0, 4, 4, 0, 0]]}, {"input": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 3, 0, 0, 0, 0, 0, 0, 3, 0], [0, 0, 3, 0, 0, 0, 0, 3, 0, 0], [0, 0, 0, 3, 0, 0, 3, 0, 0, 0], [0, 0, 0, 0, 3, 3, 0, 0, 0, 0], [0, 0, 0, 0, 3, 3, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "output": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 3, 3, 3, 3, 3, 3, 3, 3, 0], [0, 0, 3, 3, 3, 3, 3, 3, 0, 0], [0, 0, 0, 3, 3, 3, 3, 0, 0, 0], [0, 0, 0, 0, 3, 3, 0, 0, 0, 0], [0, 0, 0, 0, 3, 3, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}, {"input": [[6, 0, 0, 0, 0, 6, 0, 0, 0, 0], [0, 6, 0, 0, 6, 0, 0, 0, 0, 0], [0, 0, 6, 6, 0, 0, 0, 0, 0, 0], [0, 0, 6, 6, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 8, 0, 0, 0, 0, 0, 0, 8, 0], [0, 0, 8, 0, 0, 0, 0, 8, 0, 0], [0, 0, 0, 8, 0, 0, 8, 0, 0, 0], [0, 0, 0, 0, 8, 8, 0, 0, 0, 0], [0, 0, 0, 0, 8, 8, 0, 0, 0, 0]], "output": [[6, 6, 6, 6, 6, 6, 0, 0, 0, 0], [0, 6, 6, 6, 6, 0, 0, 0, 0, 0], [0, 0, 6, 6, 0, 0, 0, 0, 0, 0], [0, 0, 6, 6, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 8, 8, 8, 8, 8, 8, 8, 8, 0], [0, 0, 8, 8, 8, 8, 8, 8, 0, 0], [0, 0, 0, 8, 8, 8, 8, 0, 0, 0], [0, 0, 0, 0, 8, 8, 0, 0, 0, 0], [0, 0, 0, 0, 8, 8, 0, 0, 0, 0]]}], "test": [{"input": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [4, 0, 0, 4, 0, 0, 0, 0, 0, 0], [0, 4, 4, 0, 7, 0, 0, 0, 0, 7], [0, 4, 4, 0, 0, 7, 0, 0, 7, 0], [0, 0, 0, 0, 0, 0, 7, 7, 0, 0], [0, 0, 0, 0, 0, 0, 7, 7, 0, 0], [0, 3, 0, 0, 0, 0, 3, 0, 0, 0], [0, 0, 3, 0, 0, 3, 0, 0, 0, 0], [0, 0, 0, 3, 3, 0, 0, 0, 0, 0], [0, 0, 0, 3, 3, 0, 0, 0, 0, 0]], "output": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [4, 4, 4, 4, 0, 0, 0, 0, 0, 0], [0, 4, 4, 0, 7, 7, 7, 7, 7, 7], [0, 4, 4, 0, 0, 7, 7, 7, 7, 0], [0, 0, 0, 0, 0, 0, 7, 7, 0, 0], [0, 0, 0, 0, 0, 0, 7, 7, 0, 0], [0, 3, 3, 3, 3, 3, 3, 0, 0, 0], [0, 0, 3, 3, 3, 3, 0, 0, 0, 0], [0, 0, 0, 3, 3, 0, 0, 0, 0, 0], [0, 0, 0, 3, 3, 0, 0, 0, 0, 0]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/line_connect/22eb0ac0.json b/research/universal_core/holonomy_v2_2/tests/fixtures/line_connect/22eb0ac0.json new file mode 100644 index 00000000..b5f2d84d --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/line_connect/22eb0ac0.json @@ -0,0 +1 @@ +{"train": [{"input": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [8, 0, 0, 0, 0, 0, 0, 0, 0, 8], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [4, 0, 0, 0, 0, 0, 0, 0, 0, 2], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [3, 0, 0, 0, 0, 0, 0, 0, 0, 4], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 0, 0, 0, 0, 0, 0, 0, 0, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 0, 0, 0, 0, 0, 0, 0, 0, 3]], "output": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [8, 8, 8, 8, 8, 8, 8, 8, 8, 8], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [4, 0, 0, 0, 0, 0, 0, 0, 0, 2], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [3, 0, 0, 0, 0, 0, 0, 0, 0, 4], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 0, 0, 0, 0, 0, 0, 0, 0, 3]]}, {"input": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [9, 0, 0, 0, 0, 0, 0, 0, 0, 6], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [8, 0, 0, 0, 0, 0, 0, 0, 0, 9], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [4, 0, 0, 0, 0, 0, 0, 0, 0, 4], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [6, 0, 0, 0, 0, 0, 0, 0, 0, 8], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "output": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [9, 0, 0, 0, 0, 0, 0, 0, 0, 6], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [8, 0, 0, 0, 0, 0, 0, 0, 0, 9], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [4, 4, 4, 4, 4, 4, 4, 4, 4, 4], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [6, 0, 0, 0, 0, 0, 0, 0, 0, 8], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}, {"input": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 0, 0, 0, 0, 0, 0, 0, 0, 8], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [3, 0, 0, 0, 0, 0, 0, 0, 0, 4], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5, 0, 0, 0, 0, 0, 0, 0, 0, 3], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [8, 0, 0, 0, 0, 0, 0, 0, 0, 2], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "output": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 0, 0, 0, 0, 0, 0, 0, 0, 8], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [3, 0, 0, 0, 0, 0, 0, 0, 0, 4], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5, 0, 0, 0, 0, 0, 0, 0, 0, 3], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [8, 0, 0, 0, 0, 0, 0, 0, 0, 2], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}], "test": [{"input": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [4, 0, 0, 0, 0, 0, 0, 0, 0, 2], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [3, 0, 0, 0, 0, 0, 0, 0, 0, 3], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 0, 0, 0, 0, 0, 0, 0, 0, 9], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [6, 0, 0, 0, 0, 0, 0, 0, 0, 6], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [9, 0, 0, 0, 0, 0, 0, 0, 0, 4]], "output": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [4, 0, 0, 0, 0, 0, 0, 0, 0, 2], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 0, 0, 0, 0, 0, 0, 0, 0, 9], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [6, 6, 6, 6, 6, 6, 6, 6, 6, 6], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [9, 0, 0, 0, 0, 0, 0, 0, 0, 4]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/line_connect/ded97339.json b/research/universal_core/holonomy_v2_2/tests/fixtures/line_connect/ded97339.json new file mode 100644 index 00000000..5c042041 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/line_connect/ded97339.json @@ -0,0 +1 @@ +{"train": [{"input": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 8, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 8, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 8, 0, 0, 0, 8], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 8, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "output": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 8, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 8, 0, 0, 0, 0, 0, 0, 0], [0, 0, 8, 0, 0, 0, 0, 0, 0, 0], [0, 0, 8, 0, 0, 0, 0, 0, 0, 0], [0, 0, 8, 0, 0, 8, 8, 8, 8, 8], [0, 0, 8, 0, 0, 0, 0, 0, 0, 0], [0, 0, 8, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}, {"input": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 8, 0, 0, 0, 0, 8, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 8, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 8, 0, 0, 0]], "output": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 8, 8, 8, 8, 8, 8, 0, 0, 0], [0, 0, 0, 0, 0, 0, 8, 0, 0, 0], [0, 0, 0, 0, 0, 0, 8, 0, 0, 0], [0, 0, 0, 0, 0, 0, 8, 0, 0, 0], [0, 0, 0, 0, 8, 0, 8, 0, 0, 0], [0, 0, 0, 0, 0, 0, 8, 0, 0, 0], [0, 0, 0, 0, 0, 0, 8, 0, 0, 0]]}, {"input": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 8, 0, 0, 0, 8, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 8, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 8, 0, 0, 0, 8], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 8, 0, 0, 0, 0, 0, 0, 0]], "output": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 8, 8, 8, 8, 8, 0, 0, 0, 0], [0, 8, 0, 0, 0, 8, 0, 0, 0, 0], [0, 8, 0, 0, 0, 8, 0, 0, 0, 0], [0, 8, 0, 0, 0, 8, 0, 0, 0, 0], [0, 0, 0, 0, 0, 8, 0, 0, 0, 0], [0, 0, 0, 0, 0, 8, 0, 0, 0, 0], [0, 0, 0, 0, 0, 8, 8, 8, 8, 8], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 8, 0, 0, 0, 0, 0, 0, 0]]}], "test": [{"input": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 8, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 8, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 8, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 8, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 8, 0, 0, 8, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "output": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 8, 0, 0, 0, 0, 0, 0, 0, 0], [0, 8, 0, 8, 0, 0, 0, 0, 0, 0], [0, 8, 0, 0, 0, 0, 0, 8, 0, 0], [0, 8, 0, 0, 0, 0, 0, 8, 0, 0], [0, 8, 0, 0, 0, 0, 0, 8, 0, 0], [0, 0, 0, 0, 0, 0, 0, 8, 0, 0], [0, 0, 0, 0, 0, 0, 0, 8, 0, 0], [0, 0, 0, 0, 8, 8, 8, 8, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/mirror_concat/4c4377d9.json b/research/universal_core/holonomy_v2_2/tests/fixtures/mirror_concat/4c4377d9.json new file mode 100644 index 00000000..7fc0ba67 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/mirror_concat/4c4377d9.json @@ -0,0 +1 @@ +{"train": [{"input": [[9, 4, 9, 4], [9, 9, 4, 4], [4, 4, 4, 4]], "output": [[4, 4, 4, 4], [9, 9, 4, 4], [9, 4, 9, 4], [9, 4, 9, 4], [9, 9, 4, 4], [4, 4, 4, 4]]}, {"input": [[4, 1, 1, 4], [1, 1, 1, 1], [4, 4, 4, 1]], "output": [[4, 4, 4, 1], [1, 1, 1, 1], [4, 1, 1, 4], [4, 1, 1, 4], [1, 1, 1, 1], [4, 4, 4, 1]]}, {"input": [[3, 3, 5, 5], [3, 5, 5, 3], [5, 5, 3, 3]], "output": [[5, 5, 3, 3], [3, 5, 5, 3], [3, 3, 5, 5], [3, 3, 5, 5], [3, 5, 5, 3], [5, 5, 3, 3]]}, {"input": [[9, 9, 5, 9], [5, 5, 9, 9], [9, 5, 9, 9]], "output": [[9, 5, 9, 9], [5, 5, 9, 9], [9, 9, 5, 9], [9, 9, 5, 9], [5, 5, 9, 9], [9, 5, 9, 9]]}], "test": [{"input": [[4, 4, 9, 9], [4, 4, 4, 4], [4, 4, 9, 9]], "output": [[4, 4, 9, 9], [4, 4, 4, 4], [4, 4, 9, 9], [4, 4, 9, 9], [4, 4, 4, 4], [4, 4, 9, 9]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/mirror_concat/6d0aefbc.json b/research/universal_core/holonomy_v2_2/tests/fixtures/mirror_concat/6d0aefbc.json new file mode 100644 index 00000000..4950b1ee --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/mirror_concat/6d0aefbc.json @@ -0,0 +1 @@ +{"train": [{"input": [[1, 1, 1], [8, 1, 6], [6, 8, 8]], "output": [[1, 1, 1, 1, 1, 1], [8, 1, 6, 6, 1, 8], [6, 8, 8, 8, 8, 6]]}, {"input": [[6, 8, 1], [6, 1, 1], [1, 1, 6]], "output": [[6, 8, 1, 1, 8, 6], [6, 1, 1, 1, 1, 6], [1, 1, 6, 6, 1, 1]]}, {"input": [[1, 1, 1], [1, 6, 6], [6, 6, 6]], "output": [[1, 1, 1, 1, 1, 1], [1, 6, 6, 6, 6, 1], [6, 6, 6, 6, 6, 6]]}, {"input": [[6, 6, 6], [1, 6, 1], [8, 8, 6]], "output": [[6, 6, 6, 6, 6, 6], [1, 6, 1, 1, 6, 1], [8, 8, 6, 6, 8, 8]]}], "test": [{"input": [[6, 8, 6], [8, 6, 8], [1, 6, 1]], "output": [[6, 8, 6, 6, 8, 6], [8, 6, 8, 8, 6, 8], [1, 6, 1, 1, 6, 1]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/mirror_concat/6fa7a44f.json b/research/universal_core/holonomy_v2_2/tests/fixtures/mirror_concat/6fa7a44f.json new file mode 100644 index 00000000..3533a817 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/mirror_concat/6fa7a44f.json @@ -0,0 +1 @@ +{"train": [{"input": [[7, 7, 7], [9, 5, 5], [5, 1, 7]], "output": [[7, 7, 7], [9, 5, 5], [5, 1, 7], [5, 1, 7], [9, 5, 5], [7, 7, 7]]}, {"input": [[4, 8, 4], [7, 6, 7], [8, 7, 8]], "output": [[4, 8, 4], [7, 6, 7], [8, 7, 8], [8, 7, 8], [7, 6, 7], [4, 8, 4]]}, {"input": [[2, 6, 9], [2, 6, 9], [2, 9, 2]], "output": [[2, 6, 9], [2, 6, 9], [2, 9, 2], [2, 9, 2], [2, 6, 9], [2, 6, 9]]}, {"input": [[9, 1, 4], [9, 1, 4], [2, 1, 1]], "output": [[9, 1, 4], [9, 1, 4], [2, 1, 1], [2, 1, 1], [9, 1, 4], [9, 1, 4]]}], "test": [{"input": [[2, 9, 2], [8, 5, 2], [2, 2, 8]], "output": [[2, 9, 2], [8, 5, 2], [2, 2, 8], [2, 2, 8], [8, 5, 2], [2, 9, 2]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/mirror_concat/8be77c9e.json b/research/universal_core/holonomy_v2_2/tests/fixtures/mirror_concat/8be77c9e.json new file mode 100644 index 00000000..b429cee4 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/mirror_concat/8be77c9e.json @@ -0,0 +1 @@ +{"train": [{"input": [[0, 0, 0], [1, 0, 1], [1, 1, 0]], "output": [[0, 0, 0], [1, 0, 1], [1, 1, 0], [1, 1, 0], [1, 0, 1], [0, 0, 0]]}, {"input": [[1, 1, 0], [1, 1, 1], [0, 0, 0]], "output": [[1, 1, 0], [1, 1, 1], [0, 0, 0], [0, 0, 0], [1, 1, 1], [1, 1, 0]]}, {"input": [[0, 0, 0], [0, 0, 1], [0, 0, 1]], "output": [[0, 0, 0], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 0, 1], [0, 0, 0]]}], "test": [{"input": [[0, 0, 0], [0, 0, 1], [1, 0, 0]], "output": [[0, 0, 0], [0, 0, 1], [1, 0, 0], [1, 0, 0], [0, 0, 1], [0, 0, 0]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/mirror_concat/c9e6f938.json b/research/universal_core/holonomy_v2_2/tests/fixtures/mirror_concat/c9e6f938.json new file mode 100644 index 00000000..23e2cfb8 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/mirror_concat/c9e6f938.json @@ -0,0 +1 @@ +{"train": [{"input": [[0, 0, 0], [0, 7, 7], [0, 0, 0]], "output": [[0, 0, 0, 0, 0, 0], [0, 7, 7, 7, 7, 0], [0, 0, 0, 0, 0, 0]]}, {"input": [[0, 7, 0], [0, 0, 7], [0, 7, 7]], "output": [[0, 7, 0, 0, 7, 0], [0, 0, 7, 7, 0, 0], [0, 7, 7, 7, 7, 0]]}, {"input": [[0, 0, 0], [7, 0, 0], [0, 0, 0]], "output": [[0, 0, 0, 0, 0, 0], [7, 0, 0, 0, 0, 7], [0, 0, 0, 0, 0, 0]]}], "test": [{"input": [[7, 7, 0], [0, 7, 0], [0, 0, 7]], "output": [[7, 7, 0, 0, 7, 7], [0, 7, 0, 0, 7, 0], [0, 0, 7, 7, 0, 0]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/palette_repeat/a59b95c0.json b/research/universal_core/holonomy_v2_2/tests/fixtures/palette_repeat/a59b95c0.json new file mode 100644 index 00000000..84e33c84 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/palette_repeat/a59b95c0.json @@ -0,0 +1 @@ +{"train": [{"input": [[7, 7, 7], [7, 2, 2], [7, 7, 2]], "output": [[7, 7, 7, 7, 7, 7], [7, 2, 2, 7, 2, 2], [7, 7, 2, 7, 7, 2], [7, 7, 7, 7, 7, 7], [7, 2, 2, 7, 2, 2], [7, 7, 2, 7, 7, 2]]}, {"input": [[3, 4, 4], [3, 3, 3], [3, 4, 4]], "output": [[3, 4, 4, 3, 4, 4], [3, 3, 3, 3, 3, 3], [3, 4, 4, 3, 4, 4], [3, 4, 4, 3, 4, 4], [3, 3, 3, 3, 3, 3], [3, 4, 4, 3, 4, 4]]}, {"input": [[8, 2, 1], [1, 8, 3], [2, 1, 3]], "output": [[8, 2, 1, 8, 2, 1, 8, 2, 1, 8, 2, 1], [1, 8, 3, 1, 8, 3, 1, 8, 3, 1, 8, 3], [2, 1, 3, 2, 1, 3, 2, 1, 3, 2, 1, 3], [8, 2, 1, 8, 2, 1, 8, 2, 1, 8, 2, 1], [1, 8, 3, 1, 8, 3, 1, 8, 3, 1, 8, 3], [2, 1, 3, 2, 1, 3, 2, 1, 3, 2, 1, 3], [8, 2, 1, 8, 2, 1, 8, 2, 1, 8, 2, 1], [1, 8, 3, 1, 8, 3, 1, 8, 3, 1, 8, 3], [2, 1, 3, 2, 1, 3, 2, 1, 3, 2, 1, 3], [8, 2, 1, 8, 2, 1, 8, 2, 1, 8, 2, 1], [1, 8, 3, 1, 8, 3, 1, 8, 3, 1, 8, 3], [2, 1, 3, 2, 1, 3, 2, 1, 3, 2, 1, 3]]}, {"input": [[2, 3, 2], [3, 3, 2], [2, 2, 1]], "output": [[2, 3, 2, 2, 3, 2, 2, 3, 2], [3, 3, 2, 3, 3, 2, 3, 3, 2], [2, 2, 1, 2, 2, 1, 2, 2, 1], [2, 3, 2, 2, 3, 2, 2, 3, 2], [3, 3, 2, 3, 3, 2, 3, 3, 2], [2, 2, 1, 2, 2, 1, 2, 2, 1], [2, 3, 2, 2, 3, 2, 2, 3, 2], [3, 3, 2, 3, 3, 2, 3, 3, 2], [2, 2, 1, 2, 2, 1, 2, 2, 1]]}, {"input": [[9, 7, 9], [9, 6, 7], [7, 6, 6]], "output": [[9, 7, 9, 9, 7, 9, 9, 7, 9], [9, 6, 7, 9, 6, 7, 9, 6, 7], [7, 6, 6, 7, 6, 6, 7, 6, 6], [9, 7, 9, 9, 7, 9, 9, 7, 9], [9, 6, 7, 9, 6, 7, 9, 6, 7], [7, 6, 6, 7, 6, 6, 7, 6, 6], [9, 7, 9, 9, 7, 9, 9, 7, 9], [9, 6, 7, 9, 6, 7, 9, 6, 7], [7, 6, 6, 7, 6, 6, 7, 6, 6]]}], "test": [{"input": [[4, 3, 2], [2, 1, 4], [3, 1, 2]], "output": [[4, 3, 2, 4, 3, 2, 4, 3, 2, 4, 3, 2], [2, 1, 4, 2, 1, 4, 2, 1, 4, 2, 1, 4], [3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2], [4, 3, 2, 4, 3, 2, 4, 3, 2, 4, 3, 2], [2, 1, 4, 2, 1, 4, 2, 1, 4, 2, 1, 4], [3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2], [4, 3, 2, 4, 3, 2, 4, 3, 2, 4, 3, 2], [2, 1, 4, 2, 1, 4, 2, 1, 4, 2, 1, 4], [3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2], [4, 3, 2, 4, 3, 2, 4, 3, 2, 4, 3, 2], [2, 1, 4, 2, 1, 4, 2, 1, 4, 2, 1, 4], [3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/palette_repeat/ac0a08a4.json b/research/universal_core/holonomy_v2_2/tests/fixtures/palette_repeat/ac0a08a4.json new file mode 100644 index 00000000..978133c3 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/palette_repeat/ac0a08a4.json @@ -0,0 +1 @@ +{"train": [{"input": [[0, 4, 0], [0, 0, 8], [6, 0, 0]], "output": [[0, 0, 0, 4, 4, 4, 0, 0, 0], [0, 0, 0, 4, 4, 4, 0, 0, 0], [0, 0, 0, 4, 4, 4, 0, 0, 0], [0, 0, 0, 0, 0, 0, 8, 8, 8], [0, 0, 0, 0, 0, 0, 8, 8, 8], [0, 0, 0, 0, 0, 0, 8, 8, 8], [6, 6, 6, 0, 0, 0, 0, 0, 0], [6, 6, 6, 0, 0, 0, 0, 0, 0], [6, 6, 6, 0, 0, 0, 0, 0, 0]]}, {"input": [[2, 0, 0], [0, 0, 7], [0, 0, 0]], "output": [[2, 2, 0, 0, 0, 0], [2, 2, 0, 0, 0, 0], [0, 0, 0, 0, 7, 7], [0, 0, 0, 0, 7, 7], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0]]}, {"input": [[0, 6, 9], [3, 0, 2], [0, 7, 0]], "output": [[0, 0, 0, 0, 0, 6, 6, 6, 6, 6, 9, 9, 9, 9, 9], [0, 0, 0, 0, 0, 6, 6, 6, 6, 6, 9, 9, 9, 9, 9], [0, 0, 0, 0, 0, 6, 6, 6, 6, 6, 9, 9, 9, 9, 9], [0, 0, 0, 0, 0, 6, 6, 6, 6, 6, 9, 9, 9, 9, 9], [0, 0, 0, 0, 0, 6, 6, 6, 6, 6, 9, 9, 9, 9, 9], [3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2], [3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2], [3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2], [3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2], [3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2], [0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 0, 0, 0, 0, 0]]}], "test": [{"input": [[1, 0, 0], [0, 9, 6], [8, 0, 0]], "output": [[1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 9, 9, 9, 9, 6, 6, 6, 6], [0, 0, 0, 0, 9, 9, 9, 9, 6, 6, 6, 6], [0, 0, 0, 0, 9, 9, 9, 9, 6, 6, 6, 6], [0, 0, 0, 0, 9, 9, 9, 9, 6, 6, 6, 6], [8, 8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0], [8, 8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0], [8, 8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0], [8, 8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/palette_repeat/b91ae062.json b/research/universal_core/holonomy_v2_2/tests/fixtures/palette_repeat/b91ae062.json new file mode 100644 index 00000000..a5f2fc40 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/palette_repeat/b91ae062.json @@ -0,0 +1 @@ +{"train": [{"input": [[0, 8, 0], [0, 6, 6], [9, 8, 0]], "output": [[0, 0, 0, 8, 8, 8, 0, 0, 0], [0, 0, 0, 8, 8, 8, 0, 0, 0], [0, 0, 0, 8, 8, 8, 0, 0, 0], [0, 0, 0, 6, 6, 6, 6, 6, 6], [0, 0, 0, 6, 6, 6, 6, 6, 6], [0, 0, 0, 6, 6, 6, 6, 6, 6], [9, 9, 9, 8, 8, 8, 0, 0, 0], [9, 9, 9, 8, 8, 8, 0, 0, 0], [9, 9, 9, 8, 8, 8, 0, 0, 0]]}, {"input": [[1, 0, 4], [0, 4, 0], [0, 1, 0]], "output": [[1, 1, 0, 0, 4, 4], [1, 1, 0, 0, 4, 4], [0, 0, 4, 4, 0, 0], [0, 0, 4, 4, 0, 0], [0, 0, 1, 1, 0, 0], [0, 0, 1, 1, 0, 0]]}, {"input": [[3, 2, 0], [0, 7, 3], [0, 0, 0]], "output": [[3, 3, 3, 2, 2, 2, 0, 0, 0], [3, 3, 3, 2, 2, 2, 0, 0, 0], [3, 3, 3, 2, 2, 2, 0, 0, 0], [0, 0, 0, 7, 7, 7, 3, 3, 3], [0, 0, 0, 7, 7, 7, 3, 3, 3], [0, 0, 0, 7, 7, 7, 3, 3, 3], [0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0]]}, {"input": [[4, 0, 3], [2, 2, 0], [0, 0, 8]], "output": [[4, 4, 4, 4, 0, 0, 0, 0, 3, 3, 3, 3], [4, 4, 4, 4, 0, 0, 0, 0, 3, 3, 3, 3], [4, 4, 4, 4, 0, 0, 0, 0, 3, 3, 3, 3], [4, 4, 4, 4, 0, 0, 0, 0, 3, 3, 3, 3], [2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0], [2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0], [2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0], [2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8], [0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8], [0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8], [0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8]]}, {"input": [[6, 7, 0], [0, 6, 6], [0, 0, 0]], "output": [[6, 6, 7, 7, 0, 0], [6, 6, 7, 7, 0, 0], [0, 0, 6, 6, 6, 6], [0, 0, 6, 6, 6, 6], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0]]}], "test": [{"input": [[0, 1, 0], [0, 8, 7], [9, 9, 0]], "output": [[0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0], [0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0], [0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0], [0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0], [0, 0, 0, 0, 8, 8, 8, 8, 7, 7, 7, 7], [0, 0, 0, 0, 8, 8, 8, 8, 7, 7, 7, 7], [0, 0, 0, 0, 8, 8, 8, 8, 7, 7, 7, 7], [0, 0, 0, 0, 8, 8, 8, 8, 7, 7, 7, 7], [9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0], [9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0], [9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0], [9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/palette_repeat/d4b1c2b1.json b/research/universal_core/holonomy_v2_2/tests/fixtures/palette_repeat/d4b1c2b1.json new file mode 100644 index 00000000..a02bb471 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/palette_repeat/d4b1c2b1.json @@ -0,0 +1 @@ +{"train": [{"input": [[4, 4, 7], [8, 7, 7], [8, 8, 4]], "output": [[4, 4, 4, 4, 4, 4, 7, 7, 7], [4, 4, 4, 4, 4, 4, 7, 7, 7], [4, 4, 4, 4, 4, 4, 7, 7, 7], [8, 8, 8, 7, 7, 7, 7, 7, 7], [8, 8, 8, 7, 7, 7, 7, 7, 7], [8, 8, 8, 7, 7, 7, 7, 7, 7], [8, 8, 8, 8, 8, 8, 4, 4, 4], [8, 8, 8, 8, 8, 8, 4, 4, 4], [8, 8, 8, 8, 8, 8, 4, 4, 4]]}, {"input": [[8, 8, 8], [8, 8, 8], [8, 8, 8]], "output": [[8, 8, 8], [8, 8, 8], [8, 8, 8]]}, {"input": [[3, 3, 3], [3, 3, 3], [3, 3, 3]], "output": [[3, 3, 3], [3, 3, 3], [3, 3, 3]]}, {"input": [[4, 2, 8], [2, 2, 5], [8, 5, 4]], "output": [[4, 4, 4, 4, 2, 2, 2, 2, 8, 8, 8, 8], [4, 4, 4, 4, 2, 2, 2, 2, 8, 8, 8, 8], [4, 4, 4, 4, 2, 2, 2, 2, 8, 8, 8, 8], [4, 4, 4, 4, 2, 2, 2, 2, 8, 8, 8, 8], [2, 2, 2, 2, 2, 2, 2, 2, 5, 5, 5, 5], [2, 2, 2, 2, 2, 2, 2, 2, 5, 5, 5, 5], [2, 2, 2, 2, 2, 2, 2, 2, 5, 5, 5, 5], [2, 2, 2, 2, 2, 2, 2, 2, 5, 5, 5, 5], [8, 8, 8, 8, 5, 5, 5, 5, 4, 4, 4, 4], [8, 8, 8, 8, 5, 5, 5, 5, 4, 4, 4, 4], [8, 8, 8, 8, 5, 5, 5, 5, 4, 4, 4, 4], [8, 8, 8, 8, 5, 5, 5, 5, 4, 4, 4, 4]]}, {"input": [[2, 2, 4], [4, 4, 4], [2, 4, 2]], "output": [[2, 2, 2, 2, 4, 4], [2, 2, 2, 2, 4, 4], [4, 4, 4, 4, 4, 4], [4, 4, 4, 4, 4, 4], [2, 2, 4, 4, 2, 2], [2, 2, 4, 4, 2, 2]]}, {"input": [[1, 1, 1], [6, 6, 6], [6, 1, 6]], "output": [[1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], [6, 6, 6, 6, 6, 6], [6, 6, 6, 6, 6, 6], [6, 6, 1, 1, 6, 6], [6, 6, 1, 1, 6, 6]]}, {"input": [[3, 6, 6], [3, 6, 6], [3, 3, 3]], "output": [[3, 3, 6, 6, 6, 6], [3, 3, 6, 6, 6, 6], [3, 3, 6, 6, 6, 6], [3, 3, 6, 6, 6, 6], [3, 3, 3, 3, 3, 3], [3, 3, 3, 3, 3, 3]]}], "test": [{"input": [[7, 1, 7], [3, 3, 6], [8, 8, 6]], "output": [[7, 7, 7, 7, 7, 1, 1, 1, 1, 1, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 1, 1, 1, 1, 1, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 1, 1, 1, 1, 1, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 1, 1, 1, 1, 1, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 1, 1, 1, 1, 1, 7, 7, 7, 7, 7], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 6, 6, 6, 6, 6], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 6, 6, 6, 6, 6], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 6, 6, 6, 6, 6], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 6, 6, 6, 6, 6], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 6, 6, 6, 6, 6], [8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 6, 6, 6, 6], [8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 6, 6, 6, 6], [8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 6, 6, 6, 6], [8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 6, 6, 6, 6], [8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 6, 6, 6, 6, 6]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/0c786b71.json b/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/0c786b71.json new file mode 100644 index 00000000..224da8af --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/0c786b71.json @@ -0,0 +1 @@ +{"test":[{"input":[[8,5,7,8],[7,7,8,8],[5,5,8,5]],"output":[[5,8,5,5,5,5,8,5],[8,8,7,7,7,7,8,8],[8,7,5,8,8,5,7,8],[8,7,5,8,8,5,7,8],[8,8,7,7,7,7,8,8],[5,8,5,5,5,5,8,5]]}],"train":[{"input":[[5,5,9,9],[9,5,5,5],[5,7,5,7]],"output":[[7,5,7,5,5,7,5,7],[5,5,5,9,9,5,5,5],[9,9,5,5,5,5,9,9],[9,9,5,5,5,5,9,9],[5,5,5,9,9,5,5,5],[7,5,7,5,5,7,5,7]]},{"input":[[6,2,4,2],[2,2,6,6],[6,4,2,4]],"output":[[4,2,4,6,6,4,2,4],[6,6,2,2,2,2,6,6],[2,4,2,6,6,2,4,2],[2,4,2,6,6,2,4,2],[6,6,2,2,2,2,6,6],[4,2,4,6,6,4,2,4]]},{"input":[[3,3,5,5],[5,8,5,8],[8,8,5,8]],"output":[[8,5,8,8,8,8,5,8],[8,5,8,5,5,8,5,8],[5,5,3,3,3,3,5,5],[5,5,3,3,3,3,5,5],[8,5,8,5,5,8,5,8],[8,5,8,8,8,8,5,8]]}]} diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/3af2c5a8.json b/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/3af2c5a8.json new file mode 100644 index 00000000..2947a659 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/3af2c5a8.json @@ -0,0 +1 @@ +{"test":[{"input":[[4,0,0,0],[0,0,0,4],[4,4,0,0]],"output":[[4,0,0,0,0,0,0,4],[0,0,0,4,4,0,0,0],[4,4,0,0,0,0,4,4],[4,4,0,0,0,0,4,4],[0,0,0,4,4,0,0,0],[4,0,0,0,0,0,0,4]]}],"train":[{"input":[[0,0,3,3],[0,3,0,3],[3,3,3,0]],"output":[[0,0,3,3,3,3,0,0],[0,3,0,3,3,0,3,0],[3,3,3,0,0,3,3,3],[3,3,3,0,0,3,3,3],[0,3,0,3,3,0,3,0],[0,0,3,3,3,3,0,0]]},{"input":[[0,0,8,0],[0,8,0,8],[0,0,8,0]],"output":[[0,0,8,0,0,8,0,0],[0,8,0,8,8,0,8,0],[0,0,8,0,0,8,0,0],[0,0,8,0,0,8,0,0],[0,8,0,8,8,0,8,0],[0,0,8,0,0,8,0,0]]},{"input":[[3,3,3,3],[3,0,0,0],[3,0,0,0]],"output":[[3,3,3,3,3,3,3,3],[3,0,0,0,0,0,0,3],[3,0,0,0,0,0,0,3],[3,0,0,0,0,0,0,3],[3,0,0,0,0,0,0,3],[3,3,3,3,3,3,3,3]]}]} diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/46442a0e.json b/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/46442a0e.json new file mode 100644 index 00000000..0dbf0271 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/46442a0e.json @@ -0,0 +1 @@ +{"test":[{"input":[[1,4,1],[4,9,4],[9,1,9]],"output":[[1,4,1,9,4,1],[4,9,4,1,9,4],[9,1,9,9,4,1],[1,4,9,9,1,9],[4,9,1,4,9,4],[1,4,9,1,4,1]]}],"train":[{"input":[[7,7,8],[7,7,8],[8,8,8]],"output":[[7,7,8,8,7,7],[7,7,8,8,7,7],[8,8,8,8,8,8],[8,8,8,8,8,8],[7,7,8,8,7,7],[7,7,8,8,7,7]]},{"input":[[8,6],[6,8]],"output":[[8,6,6,8],[6,8,8,6],[6,8,8,6],[8,6,6,8]]},{"input":[[6,9,9],[6,4,4],[6,4,4]],"output":[[6,9,9,6,6,6],[6,4,4,4,4,9],[6,4,4,4,4,9],[9,4,4,4,4,6],[9,4,4,4,4,6],[6,6,6,9,9,6]]}]} diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/62c24649.json b/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/62c24649.json new file mode 100644 index 00000000..155b67e7 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/62c24649.json @@ -0,0 +1 @@ +{"test":[{"input":[[1,1,0],[0,3,2],[3,3,0]],"output":[[1,1,0,0,1,1],[0,3,2,2,3,0],[3,3,0,0,3,3],[3,3,0,0,3,3],[0,3,2,2,3,0],[1,1,0,0,1,1]]}],"train":[{"input":[[3,3,1],[1,3,0],[0,2,2]],"output":[[3,3,1,1,3,3],[1,3,0,0,3,1],[0,2,2,2,2,0],[0,2,2,2,2,0],[1,3,0,0,3,1],[3,3,1,1,3,3]]},{"input":[[3,3,3],[0,2,2],[1,1,0]],"output":[[3,3,3,3,3,3],[0,2,2,2,2,0],[1,1,0,0,1,1],[1,1,0,0,1,1],[0,2,2,2,2,0],[3,3,3,3,3,3]]},{"input":[[2,1,0],[0,2,3],[0,3,0]],"output":[[2,1,0,0,1,2],[0,2,3,3,2,0],[0,3,0,0,3,0],[0,3,0,0,3,0],[0,2,3,3,2,0],[2,1,0,0,1,2]]}]} diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/67e8384a.json b/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/67e8384a.json new file mode 100644 index 00000000..b4722bcf --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/67e8384a.json @@ -0,0 +1 @@ +{"test":[{"input":[[1,6,6],[5,2,2],[2,2,2]],"output":[[1,6,6,6,6,1],[5,2,2,2,2,5],[2,2,2,2,2,2],[2,2,2,2,2,2],[5,2,2,2,2,5],[1,6,6,6,6,1]]}],"train":[{"input":[[2,5,2],[2,6,4],[2,2,2]],"output":[[2,5,2,2,5,2],[2,6,4,4,6,2],[2,2,2,2,2,2],[2,2,2,2,2,2],[2,6,4,4,6,2],[2,5,2,2,5,2]]},{"input":[[7,1,5],[7,7,1],[5,3,1]],"output":[[7,1,5,5,1,7],[7,7,1,1,7,7],[5,3,1,1,3,5],[5,3,1,1,3,5],[7,7,1,1,7,7],[7,1,5,5,1,7]]},{"input":[[1,2,1],[2,8,1],[8,1,6]],"output":[[1,2,1,1,2,1],[2,8,1,1,8,2],[8,1,6,6,1,8],[8,1,6,6,1,8],[2,8,1,1,8,2],[1,2,1,1,2,1]]},{"input":[[5,3,4],[3,4,5],[3,4,4]],"output":[[5,3,4,4,3,5],[3,4,5,5,4,3],[3,4,4,4,4,3],[3,4,4,4,4,3],[3,4,5,5,4,3],[5,3,4,4,3,5]]}]} diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/7953d61e.json b/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/7953d61e.json new file mode 100644 index 00000000..b7f8cf5f --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/7953d61e.json @@ -0,0 +1 @@ +{"test":[{"input":[[4,6,4,4],[4,6,4,4],[7,6,7,9],[9,4,9,7]],"output":[[4,6,4,4,4,4,9,7],[4,6,4,4,4,4,7,9],[7,6,7,9,6,6,6,4],[9,4,9,7,4,4,7,9],[7,9,4,9,9,7,4,4],[9,7,6,7,4,6,6,6],[4,4,6,4,9,7,4,4],[4,4,6,4,7,9,4,4]]}],"train":[{"input":[[4,9,1,8],[8,4,1,8],[4,8,8,1],[1,1,1,8]],"output":[[4,9,1,8,8,8,1,8],[8,4,1,8,1,1,8,1],[4,8,8,1,9,4,8,1],[1,1,1,8,4,8,4,1],[8,1,1,1,1,4,8,4],[1,8,8,4,1,8,4,9],[8,1,4,8,1,8,1,1],[8,1,9,4,8,1,8,8]]},{"input":[[6,2,6,2],[6,6,5,5],[1,1,1,2],[5,1,2,1]],"output":[[6,2,6,2,2,5,2,1],[6,6,5,5,6,5,1,2],[1,1,1,2,2,6,1,1],[5,1,2,1,6,6,1,5],[1,2,1,5,5,1,6,6],[2,1,1,1,1,1,6,2],[5,5,6,6,2,1,5,6],[2,6,2,6,1,2,5,2]]},{"input":[[6,7,7,6],[7,1,6,6],[9,1,6,6],[9,1,6,1]],"output":[[6,7,7,6,6,6,6,1],[7,1,6,6,7,6,6,6],[9,1,6,6,7,1,1,1],[9,1,6,1,6,7,9,9],[1,6,1,9,9,9,7,6],[6,6,1,9,1,1,1,7],[6,6,1,7,6,6,6,7],[6,7,7,6,1,6,6,6]]},{"input":[[1,1,2,1],[6,6,7,6],[7,6,2,1],[1,6,2,6]],"output":[[1,1,2,1,1,6,1,6],[6,6,7,6,2,7,2,2],[7,6,2,1,1,6,6,6],[1,6,2,6,1,6,7,1],[6,2,6,1,1,7,6,1],[1,2,6,7,6,6,6,1],[6,7,6,6,2,2,7,2],[1,2,1,1,6,1,6,1]]},{"input":[[4,1,9,1],[1,9,1,4],[9,1,4,6],[4,1,6,6]],"output":[[4,1,9,1,1,4,6,6],[1,9,1,4,9,1,4,6],[9,1,4,6,1,9,1,1],[4,1,6,6,4,1,9,4],[6,6,1,4,4,9,1,4],[6,4,1,9,1,1,9,1],[4,1,9,1,6,4,1,9],[1,9,1,4,6,6,4,1]]}]} diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/7fe24cdd.json b/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/7fe24cdd.json new file mode 100644 index 00000000..5b4c0495 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/7fe24cdd.json @@ -0,0 +1 @@ +{"test":[{"input":[[2,5,0],[2,5,1],[3,1,1]],"output":[[2,5,0,3,2,2],[2,5,1,1,5,5],[3,1,1,1,1,0],[0,1,1,1,1,3],[5,5,1,1,5,2],[2,2,3,0,5,2]]}],"train":[{"input":[[3,8,2],[3,2,2],[8,5,2]],"output":[[3,8,2,8,3,3],[3,2,2,5,2,8],[8,5,2,2,2,2],[2,2,2,2,5,8],[8,2,5,2,2,3],[3,3,8,2,8,3]]},{"input":[[8,5,0],[8,5,3],[0,3,2]],"output":[[8,5,0,0,8,8],[8,5,3,3,5,5],[0,3,2,2,3,0],[0,3,2,2,3,0],[5,5,3,3,5,8],[8,8,0,0,5,8]]},{"input":[[0,3,0],[6,6,6],[0,3,0]],"output":[[0,3,0,0,6,0],[6,6,6,3,6,3],[0,3,0,0,6,0],[0,6,0,0,3,0],[3,6,3,6,6,6],[0,6,0,0,3,0]]}]} diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/833dafe3.json b/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/833dafe3.json new file mode 100644 index 00000000..07586428 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/833dafe3.json @@ -0,0 +1 @@ +{"test":[{"input":[[0,0,1,0,0,0,0,0],[0,2,1,0,9,0,0,0],[0,2,1,0,9,0,0,0],[0,2,1,0,9,1,1,1],[9,2,0,0,9,0,0,0],[9,2,0,0,9,0,0,9],[1,2,0,0,9,0,0,9],[9,9,0,0,9,0,0,9]],"output":[[9,0,0,9,0,0,9,9,9,9,0,0,9,0,0,9],[9,0,0,9,0,0,2,1,1,2,0,0,9,0,0,9],[9,0,0,9,0,0,2,9,9,2,0,0,9,0,0,9],[0,0,0,9,0,0,2,9,9,2,0,0,9,0,0,0],[1,1,1,9,0,1,2,0,0,2,1,0,9,1,1,1],[0,0,0,9,0,1,2,0,0,2,1,0,9,0,0,0],[0,0,0,9,0,1,2,0,0,2,1,0,9,0,0,0],[0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0],[0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0],[0,0,0,9,0,1,2,0,0,2,1,0,9,0,0,0],[0,0,0,9,0,1,2,0,0,2,1,0,9,0,0,0],[1,1,1,9,0,1,2,0,0,2,1,0,9,1,1,1],[0,0,0,9,0,0,2,9,9,2,0,0,9,0,0,0],[9,0,0,9,0,0,2,9,9,2,0,0,9,0,0,9],[9,0,0,9,0,0,2,1,1,2,0,0,9,0,0,9],[9,0,0,9,0,0,9,9,9,9,0,0,9,0,0,9]]}],"train":[{"input":[[0,6,0,0],[4,6,0,3],[4,6,3,0],[4,3,3,0]],"output":[[0,3,3,4,4,3,3,0],[0,3,6,4,4,6,3,0],[3,0,6,4,4,6,0,3],[0,0,6,0,0,6,0,0],[0,0,6,0,0,6,0,0],[3,0,6,4,4,6,0,3],[0,3,6,4,4,6,3,0],[0,3,3,4,4,3,3,0]]},{"input":[[3,0,0],[3,4,0],[3,4,2]],"output":[[2,4,3,3,4,2],[0,4,3,3,4,0],[0,0,3,3,0,0],[0,0,3,3,0,0],[0,4,3,3,4,0],[2,4,3,3,4,2]]}]} diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/ed98d772.json b/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/ed98d772.json new file mode 100644 index 00000000..f92f073c --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/quadrant_mosaic/ed98d772.json @@ -0,0 +1 @@ +{"test":[{"input":[[6,6,0],[6,6,0],[0,0,6]],"output":[[6,6,0,0,0,6],[6,6,0,6,6,0],[0,0,6,6,6,0],[6,0,0,0,6,6],[0,6,6,0,6,6],[0,6,6,6,0,0]]}],"train":[{"input":[[8,0,8],[8,0,0],[8,0,0]],"output":[[8,0,8,8,0,0],[8,0,0,0,0,0],[8,0,0,8,8,8],[0,0,8,8,8,8],[0,0,8,0,0,0],[8,0,8,0,0,8]]},{"input":[[3,0,3],[0,3,3],[3,3,3]],"output":[[3,0,3,3,3,3],[0,3,3,0,3,3],[3,3,3,3,0,3],[3,3,3,3,0,3],[3,3,0,3,3,0],[3,0,3,3,3,3]]},{"input":[[3,3,3],[0,0,3],[3,0,0]],"output":[[3,3,3,3,3,0],[0,0,3,3,0,0],[3,0,0,3,0,3],[0,0,3,3,0,3],[3,0,0,0,0,3],[3,3,3,0,3,3]]},{"input":[[0,7,7],[0,0,0],[7,7,0]],"output":[[0,7,7,7,0,0],[0,0,0,7,0,7],[7,7,0,0,0,7],[0,7,7,7,0,0],[0,0,0,7,0,7],[7,7,0,0,0,7]]},{"input":[[9,9,9],[0,0,0],[9,9,0]],"output":[[9,9,9,9,0,0],[0,0,0,9,0,9],[9,9,0,9,0,9],[0,9,9,9,0,9],[0,0,0,9,0,9],[9,9,9,0,0,9]]}]} diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/ray_extend/623ea044.json b/research/universal_core/holonomy_v2_2/tests/fixtures/ray_extend/623ea044.json new file mode 100644 index 00000000..890b4955 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/ray_extend/623ea044.json @@ -0,0 +1 @@ +{"train": [{"input": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "output": [[0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 7], [0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 7, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 7, 0], [0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 7], [0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}, {"input": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "output": [[2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0], [0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]]}, {"input": [[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 8, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0]], "output": [[0, 0, 0, 0, 0, 8, 0], [8, 0, 0, 0, 8, 0, 0], [0, 8, 0, 8, 0, 0, 0], [0, 0, 8, 0, 0, 0, 0], [0, 8, 0, 8, 0, 0, 0], [8, 0, 0, 0, 8, 0, 0], [0, 0, 0, 0, 0, 8, 0]]}], "test": [{"input": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "output": [[0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6], [0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 6, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 6, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 6, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 6, 0], [0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 6], [0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/ray_extend/d037b0a7.json b/research/universal_core/holonomy_v2_2/tests/fixtures/ray_extend/d037b0a7.json new file mode 100644 index 00000000..41e45dfe --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/ray_extend/d037b0a7.json @@ -0,0 +1 @@ +{"train": [{"input": [[0, 2, 0], [7, 0, 8], [0, 0, 0]], "output": [[0, 2, 0], [7, 2, 8], [7, 2, 8]]}, {"input": [[0, 0, 6], [0, 4, 0], [3, 0, 0]], "output": [[0, 0, 6], [0, 4, 6], [3, 4, 6]]}, {"input": [[4, 0, 0], [0, 2, 0], [0, 0, 0]], "output": [[4, 0, 0], [4, 2, 0], [4, 2, 0]]}], "test": [{"input": [[4, 0, 8], [0, 0, 0], [0, 7, 0]], "output": [[4, 0, 8], [4, 0, 8], [4, 7, 8]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/symmetry_complete/496994bd.json b/research/universal_core/holonomy_v2_2/tests/fixtures/symmetry_complete/496994bd.json new file mode 100644 index 00000000..2c3bfe8e --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/symmetry_complete/496994bd.json @@ -0,0 +1 @@ +{"train": [{"input": [[2, 2, 2, 2, 2], [8, 8, 8, 8, 8], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0]], "output": [[2, 2, 2, 2, 2], [8, 8, 8, 8, 8], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [8, 8, 8, 8, 8], [2, 2, 2, 2, 2]]}, {"input": [[2, 2, 2], [2, 2, 2], [3, 3, 3], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0]], "output": [[2, 2, 2], [2, 2, 2], [3, 3, 3], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [3, 3, 3], [2, 2, 2], [2, 2, 2]]}], "test": [{"input": [[3, 3, 3, 3, 3, 3], [5, 5, 5, 5, 5, 5], [5, 5, 5, 5, 5, 5], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0]], "output": [[3, 3, 3, 3, 3, 3], [5, 5, 5, 5, 5, 5], [5, 5, 5, 5, 5, 5], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [5, 5, 5, 5, 5, 5], [5, 5, 5, 5, 5, 5], [3, 3, 3, 3, 3, 3]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/symmetry_complete/b8825c91.json b/research/universal_core/holonomy_v2_2/tests/fixtures/symmetry_complete/b8825c91.json new file mode 100644 index 00000000..e0d7ea53 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/symmetry_complete/b8825c91.json @@ -0,0 +1 @@ +{"train": [{"input": [[9, 3, 9, 9, 2, 8, 7, 8, 8, 7, 8, 2, 9, 9, 3, 9], [3, 9, 9, 3, 8, 8, 8, 5, 5, 8, 8, 8, 3, 9, 9, 3], [9, 9, 2, 8, 7, 8, 2, 2, 2, 2, 8, 7, 8, 2, 9, 9], [9, 3, 8, 8, 8, 5, 2, 1, 1, 2, 5, 8, 8, 8, 3, 9], [2, 8, 7, 8, 2, 5, 9, 7, 7, 9, 5, 2, 8, 7, 8, 2], [8, 8, 8, 5, 5, 5, 7, 6, 6, 7, 5, 5, 5, 8, 8, 8], [7, 8, 2, 2, 9, 7, 1, 1, 1, 1, 7, 9, 4, 4, 8, 7], [8, 5, 2, 1, 7, 6, 1, 3, 3, 1, 6, 7, 4, 4, 5, 8], [8, 5, 2, 1, 7, 6, 1, 3, 3, 1, 6, 7, 4, 4, 5, 8], [7, 8, 2, 2, 9, 7, 1, 1, 1, 1, 7, 9, 4, 4, 8, 7], [8, 8, 8, 5, 5, 5, 7, 6, 6, 7, 5, 5, 5, 8, 8, 8], [2, 8, 4, 4, 4, 4, 9, 7, 7, 9, 5, 2, 8, 7, 8, 2], [9, 3, 4, 4, 4, 4, 2, 1, 1, 2, 5, 8, 8, 8, 3, 9], [9, 9, 4, 4, 4, 4, 2, 2, 2, 2, 8, 7, 8, 2, 9, 9], [3, 9, 4, 4, 4, 4, 8, 5, 5, 8, 8, 8, 3, 9, 9, 3], [9, 3, 9, 9, 2, 8, 7, 8, 8, 7, 8, 2, 9, 9, 3, 9]], "output": [[9, 3, 9, 9, 2, 8, 7, 8, 8, 7, 8, 2, 9, 9, 3, 9], [3, 9, 9, 3, 8, 8, 8, 5, 5, 8, 8, 8, 3, 9, 9, 3], [9, 9, 2, 8, 7, 8, 2, 2, 2, 2, 8, 7, 8, 2, 9, 9], [9, 3, 8, 8, 8, 5, 2, 1, 1, 2, 5, 8, 8, 8, 3, 9], [2, 8, 7, 8, 2, 5, 9, 7, 7, 9, 5, 2, 8, 7, 8, 2], [8, 8, 8, 5, 5, 5, 7, 6, 6, 7, 5, 5, 5, 8, 8, 8], [7, 8, 2, 2, 9, 7, 1, 1, 1, 1, 7, 9, 2, 2, 8, 7], [8, 5, 2, 1, 7, 6, 1, 3, 3, 1, 6, 7, 1, 2, 5, 8], [8, 5, 2, 1, 7, 6, 1, 3, 3, 1, 6, 7, 1, 2, 5, 8], [7, 8, 2, 2, 9, 7, 1, 1, 1, 1, 7, 9, 2, 2, 8, 7], [8, 8, 8, 5, 5, 5, 7, 6, 6, 7, 5, 5, 5, 8, 8, 8], [2, 8, 7, 8, 2, 5, 9, 7, 7, 9, 5, 2, 8, 7, 8, 2], [9, 3, 8, 8, 8, 5, 2, 1, 1, 2, 5, 8, 8, 8, 3, 9], [9, 9, 2, 8, 7, 8, 2, 2, 2, 2, 8, 7, 8, 2, 9, 9], [3, 9, 9, 3, 8, 8, 8, 5, 5, 8, 8, 8, 3, 9, 9, 3], [9, 3, 9, 9, 2, 8, 7, 8, 8, 7, 8, 2, 9, 9, 3, 9]]}, {"input": [[9, 9, 6, 1, 8, 9, 6, 6, 6, 6, 9, 8, 1, 6, 9, 9], [9, 6, 1, 3, 9, 6, 6, 1, 1, 6, 6, 9, 3, 1, 6, 9], [6, 4, 4, 2, 6, 6, 8, 8, 8, 8, 6, 6, 2, 5, 1, 6], [1, 4, 4, 8, 6, 1, 8, 2, 2, 8, 1, 6, 8, 2, 3, 1], [8, 4, 4, 6, 7, 1, 5, 5, 5, 5, 1, 7, 6, 6, 9, 8], [9, 6, 6, 1, 1, 1, 5, 5, 5, 5, 1, 1, 1, 6, 6, 9], [6, 6, 8, 8, 5, 5, 9, 5, 5, 9, 5, 5, 8, 8, 6, 6], [6, 1, 8, 2, 5, 5, 5, 8, 8, 5, 5, 5, 2, 8, 1, 6], [6, 1, 8, 2, 5, 5, 5, 8, 8, 5, 5, 4, 4, 4, 1, 6], [6, 6, 8, 8, 5, 5, 9, 5, 5, 9, 5, 4, 4, 4, 6, 6], [9, 6, 6, 1, 1, 1, 5, 5, 5, 5, 1, 1, 1, 6, 6, 9], [8, 9, 6, 6, 7, 1, 5, 5, 5, 5, 1, 7, 6, 6, 9, 8], [1, 3, 2, 8, 6, 1, 8, 2, 2, 8, 1, 6, 8, 2, 3, 1], [6, 1, 5, 2, 6, 6, 8, 8, 8, 8, 6, 6, 2, 5, 1, 6], [9, 6, 1, 3, 9, 6, 6, 1, 1, 6, 6, 9, 3, 1, 6, 9], [9, 9, 6, 1, 8, 9, 6, 6, 6, 6, 9, 8, 1, 6, 9, 9]], "output": [[9, 9, 6, 1, 8, 9, 6, 6, 6, 6, 9, 8, 1, 6, 9, 9], [9, 6, 1, 3, 9, 6, 6, 1, 1, 6, 6, 9, 3, 1, 6, 9], [6, 1, 5, 2, 6, 6, 8, 8, 8, 8, 6, 6, 2, 5, 1, 6], [1, 3, 2, 8, 6, 1, 8, 2, 2, 8, 1, 6, 8, 2, 3, 1], [8, 9, 6, 6, 7, 1, 5, 5, 5, 5, 1, 7, 6, 6, 9, 8], [9, 6, 6, 1, 1, 1, 5, 5, 5, 5, 1, 1, 1, 6, 6, 9], [6, 6, 8, 8, 5, 5, 9, 5, 5, 9, 5, 5, 8, 8, 6, 6], [6, 1, 8, 2, 5, 5, 5, 8, 8, 5, 5, 5, 2, 8, 1, 6], [6, 1, 8, 2, 5, 5, 5, 8, 8, 5, 5, 5, 2, 8, 1, 6], [6, 6, 8, 8, 5, 5, 9, 5, 5, 9, 5, 5, 8, 8, 6, 6], [9, 6, 6, 1, 1, 1, 5, 5, 5, 5, 1, 1, 1, 6, 6, 9], [8, 9, 6, 6, 7, 1, 5, 5, 5, 5, 1, 7, 6, 6, 9, 8], [1, 3, 2, 8, 6, 1, 8, 2, 2, 8, 1, 6, 8, 2, 3, 1], [6, 1, 5, 2, 6, 6, 8, 8, 8, 8, 6, 6, 2, 5, 1, 6], [9, 6, 1, 3, 9, 6, 6, 1, 1, 6, 6, 9, 3, 1, 6, 9], [9, 9, 6, 1, 8, 9, 6, 6, 6, 6, 9, 8, 1, 6, 9, 9]]}, {"input": [[2, 2, 7, 6, 8, 9, 9, 1, 1, 9, 9, 8, 6, 7, 2, 2], [2, 1, 6, 2, 9, 5, 1, 1, 1, 1, 4, 4, 4, 4, 1, 2], [7, 6, 3, 3, 9, 1, 6, 6, 6, 6, 4, 4, 4, 4, 6, 7], [6, 2, 3, 8, 1, 1, 6, 6, 6, 6, 4, 4, 4, 4, 2, 6], [8, 9, 9, 1, 1, 7, 1, 1, 1, 1, 7, 1, 1, 9, 9, 8], [9, 5, 1, 1, 7, 7, 1, 3, 3, 1, 7, 7, 1, 1, 5, 9], [9, 1, 6, 6, 1, 1, 3, 3, 3, 3, 1, 1, 6, 6, 1, 9], [1, 1, 6, 6, 1, 3, 3, 2, 2, 3, 3, 1, 6, 6, 1, 1], [1, 1, 6, 4, 4, 3, 3, 2, 2, 3, 3, 1, 6, 6, 1, 1], [9, 1, 6, 4, 4, 1, 3, 3, 3, 3, 1, 1, 6, 6, 1, 9], [9, 5, 1, 4, 4, 7, 1, 3, 3, 1, 7, 7, 1, 1, 5, 9], [8, 9, 9, 1, 1, 7, 1, 1, 1, 1, 7, 1, 1, 9, 9, 8], [6, 2, 3, 8, 1, 1, 6, 6, 6, 6, 1, 1, 8, 3, 2, 6], [7, 6, 3, 3, 9, 1, 6, 6, 6, 6, 1, 9, 3, 3, 6, 7], [2, 1, 6, 2, 9, 5, 1, 1, 1, 1, 5, 9, 2, 6, 1, 2], [2, 2, 7, 6, 8, 9, 9, 1, 1, 9, 9, 8, 6, 7, 2, 2]], "output": [[2, 2, 7, 6, 8, 9, 9, 1, 1, 9, 9, 8, 6, 7, 2, 2], [2, 1, 6, 2, 9, 5, 1, 1, 1, 1, 5, 9, 2, 6, 1, 2], [7, 6, 3, 3, 9, 1, 6, 6, 6, 6, 1, 9, 3, 3, 6, 7], [6, 2, 3, 8, 1, 1, 6, 6, 6, 6, 1, 1, 8, 3, 2, 6], [8, 9, 9, 1, 1, 7, 1, 1, 1, 1, 7, 1, 1, 9, 9, 8], [9, 5, 1, 1, 7, 7, 1, 3, 3, 1, 7, 7, 1, 1, 5, 9], [9, 1, 6, 6, 1, 1, 3, 3, 3, 3, 1, 1, 6, 6, 1, 9], [1, 1, 6, 6, 1, 3, 3, 2, 2, 3, 3, 1, 6, 6, 1, 1], [1, 1, 6, 6, 1, 3, 3, 2, 2, 3, 3, 1, 6, 6, 1, 1], [9, 1, 6, 6, 1, 1, 3, 3, 3, 3, 1, 1, 6, 6, 1, 9], [9, 5, 1, 1, 7, 7, 1, 3, 3, 1, 7, 7, 1, 1, 5, 9], [8, 9, 9, 1, 1, 7, 1, 1, 1, 1, 7, 1, 1, 9, 9, 8], [6, 2, 3, 8, 1, 1, 6, 6, 6, 6, 1, 1, 8, 3, 2, 6], [7, 6, 3, 3, 9, 1, 6, 6, 6, 6, 1, 9, 3, 3, 6, 7], [2, 1, 6, 2, 9, 5, 1, 1, 1, 1, 5, 9, 2, 6, 1, 2], [2, 2, 7, 6, 8, 9, 9, 1, 1, 9, 9, 8, 6, 7, 2, 2]]}, {"input": [[9, 9, 6, 5, 9, 6, 7, 7, 7, 7, 6, 9, 5, 6, 9, 9], [9, 1, 5, 5, 6, 1, 7, 9, 9, 7, 1, 6, 5, 5, 1, 9], [6, 5, 1, 9, 7, 7, 3, 3, 3, 3, 7, 7, 9, 1, 5, 6], [5, 5, 9, 3, 7, 9, 3, 3, 3, 3, 9, 7, 3, 9, 5, 5], [9, 6, 7, 7, 3, 8, 9, 1, 1, 9, 8, 3, 7, 7, 6, 9], [6, 1, 7, 9, 8, 3, 1, 1, 1, 1, 4, 4, 4, 4, 1, 6], [7, 7, 3, 3, 9, 1, 6, 6, 6, 6, 4, 4, 4, 4, 7, 7], [7, 9, 3, 3, 1, 1, 6, 1, 1, 6, 4, 4, 4, 4, 9, 7], [7, 9, 3, 3, 1, 1, 6, 1, 1, 6, 1, 1, 3, 3, 9, 7], [7, 7, 3, 3, 9, 1, 6, 6, 6, 6, 1, 9, 3, 3, 7, 7], [6, 1, 7, 9, 8, 3, 1, 1, 1, 1, 4, 4, 4, 7, 1, 6], [9, 6, 7, 7, 3, 8, 9, 1, 1, 9, 4, 4, 4, 7, 6, 9], [5, 5, 9, 3, 7, 9, 3, 3, 3, 3, 4, 4, 4, 9, 5, 5], [6, 5, 1, 9, 7, 7, 3, 3, 3, 3, 4, 4, 4, 1, 5, 6], [9, 1, 5, 5, 6, 1, 7, 9, 9, 7, 1, 6, 5, 5, 1, 9], [9, 9, 6, 5, 9, 6, 7, 7, 7, 7, 6, 9, 5, 6, 9, 9]], "output": [[9, 9, 6, 5, 9, 6, 7, 7, 7, 7, 6, 9, 5, 6, 9, 9], [9, 1, 5, 5, 6, 1, 7, 9, 9, 7, 1, 6, 5, 5, 1, 9], [6, 5, 1, 9, 7, 7, 3, 3, 3, 3, 7, 7, 9, 1, 5, 6], [5, 5, 9, 3, 7, 9, 3, 3, 3, 3, 9, 7, 3, 9, 5, 5], [9, 6, 7, 7, 3, 8, 9, 1, 1, 9, 8, 3, 7, 7, 6, 9], [6, 1, 7, 9, 8, 3, 1, 1, 1, 1, 3, 8, 9, 7, 1, 6], [7, 7, 3, 3, 9, 1, 6, 6, 6, 6, 1, 9, 3, 3, 7, 7], [7, 9, 3, 3, 1, 1, 6, 1, 1, 6, 1, 1, 3, 3, 9, 7], [7, 9, 3, 3, 1, 1, 6, 1, 1, 6, 1, 1, 3, 3, 9, 7], [7, 7, 3, 3, 9, 1, 6, 6, 6, 6, 1, 9, 3, 3, 7, 7], [6, 1, 7, 9, 8, 3, 1, 1, 1, 1, 3, 8, 9, 7, 1, 6], [9, 6, 7, 7, 3, 8, 9, 1, 1, 9, 8, 3, 7, 7, 6, 9], [5, 5, 9, 3, 7, 9, 3, 3, 3, 3, 9, 7, 3, 9, 5, 5], [6, 5, 1, 9, 7, 7, 3, 3, 3, 3, 7, 7, 9, 1, 5, 6], [9, 1, 5, 5, 6, 1, 7, 9, 9, 7, 1, 6, 5, 5, 1, 9], [9, 9, 6, 5, 9, 6, 7, 7, 7, 7, 6, 9, 5, 6, 9, 9]]}], "test": [{"input": [[7, 7, 8, 1, 9, 8, 2, 6, 6, 2, 8, 9, 1, 8, 7, 7], [7, 1, 1, 8, 8, 8, 6, 6, 6, 6, 8, 8, 8, 1, 1, 7], [8, 1, 6, 9, 2, 6, 4, 4, 4, 6, 6, 2, 9, 6, 1, 8], [1, 8, 9, 1, 6, 6, 4, 4, 4, 1, 6, 6, 1, 9, 8, 1], [9, 8, 2, 6, 8, 7, 4, 4, 4, 6, 4, 4, 4, 4, 8, 9], [8, 8, 6, 6, 7, 7, 6, 5, 5, 6, 4, 4, 4, 4, 8, 8], [2, 6, 6, 1, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 6, 2], [6, 6, 1, 1, 6, 5, 5, 7, 7, 5, 4, 4, 4, 4, 6, 6], [6, 6, 1, 1, 6, 5, 5, 7, 7, 5, 5, 6, 1, 1, 6, 6], [2, 6, 6, 1, 6, 6, 5, 5, 5, 5, 6, 6, 1, 6, 6, 2], [8, 8, 6, 6, 7, 7, 6, 5, 5, 6, 7, 7, 6, 6, 8, 8], [9, 8, 2, 6, 8, 7, 6, 6, 6, 6, 7, 8, 6, 2, 8, 9], [1, 8, 9, 1, 6, 6, 1, 1, 1, 1, 6, 6, 1, 9, 8, 1], [8, 1, 6, 9, 2, 6, 6, 1, 1, 6, 6, 2, 9, 6, 1, 8], [7, 1, 1, 8, 8, 8, 6, 6, 6, 6, 8, 8, 8, 1, 1, 7], [7, 7, 8, 1, 9, 8, 2, 6, 6, 2, 8, 9, 1, 8, 7, 7]], "output": [[7, 7, 8, 1, 9, 8, 2, 6, 6, 2, 8, 9, 1, 8, 7, 7], [7, 1, 1, 8, 8, 8, 6, 6, 6, 6, 8, 8, 8, 1, 1, 7], [8, 1, 6, 9, 2, 6, 6, 1, 1, 6, 6, 2, 9, 6, 1, 8], [1, 8, 9, 1, 6, 6, 1, 1, 1, 1, 6, 6, 1, 9, 8, 1], [9, 8, 2, 6, 8, 7, 6, 6, 6, 6, 7, 8, 6, 2, 8, 9], [8, 8, 6, 6, 7, 7, 6, 5, 5, 6, 7, 7, 6, 6, 8, 8], [2, 6, 6, 1, 6, 6, 5, 5, 5, 5, 6, 6, 1, 6, 6, 2], [6, 6, 1, 1, 6, 5, 5, 7, 7, 5, 5, 6, 1, 1, 6, 6], [6, 6, 1, 1, 6, 5, 5, 7, 7, 5, 5, 6, 1, 1, 6, 6], [2, 6, 6, 1, 6, 6, 5, 5, 5, 5, 6, 6, 1, 6, 6, 2], [8, 8, 6, 6, 7, 7, 6, 5, 5, 6, 7, 7, 6, 6, 8, 8], [9, 8, 2, 6, 8, 7, 6, 6, 6, 6, 7, 8, 6, 2, 8, 9], [1, 8, 9, 1, 6, 6, 1, 1, 1, 1, 6, 6, 1, 9, 8, 1], [8, 1, 6, 9, 2, 6, 6, 1, 1, 6, 6, 2, 9, 6, 1, 8], [7, 1, 1, 8, 8, 8, 6, 6, 6, 6, 8, 8, 8, 1, 1, 7], [7, 7, 8, 1, 9, 8, 2, 6, 6, 2, 8, 9, 1, 8, 7, 7]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/symmetry_complete/e729b7be.json b/research/universal_core/holonomy_v2_2/tests/fixtures/symmetry_complete/e729b7be.json new file mode 100644 index 00000000..f4bea2c6 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/symmetry_complete/e729b7be.json @@ -0,0 +1 @@ +{"train": [{"input": [[7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 6, 6, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 6, 1, 1, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 6, 1, 1, 3, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 6, 1, 1, 3, 9, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 6, 1, 3, 9, 9, 4, 7, 7, 7, 7, 7, 7, 7, 7], [8, 7, 7, 6, 1, 3, 9, 9, 8, 7, 7, 7, 7, 7, 7, 7, 8], [7, 7, 6, 1, 1, 3, 9, 9, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 6, 1, 3, 9, 9, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 6, 1, 1, 3, 9, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 6, 1, 1, 3, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 6, 1, 6, 1, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 6, 7, 6, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7]], "output": [[7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 4, 6, 7, 6, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 6, 6, 4, 1, 6, 1, 6, 7, 7, 7, 7], [7, 7, 7, 7, 7, 6, 1, 1, 4, 3, 1, 1, 6, 7, 7, 7, 7], [7, 7, 7, 7, 6, 1, 1, 3, 4, 9, 3, 1, 1, 6, 7, 7, 7], [7, 7, 7, 6, 1, 1, 3, 9, 4, 9, 9, 3, 1, 6, 7, 7, 7], [7, 7, 7, 6, 1, 3, 9, 9, 4, 9, 9, 3, 1, 1, 6, 7, 7], [8, 7, 7, 6, 1, 3, 9, 9, 8, 9, 9, 3, 1, 6, 7, 7, 8], [7, 7, 6, 1, 1, 3, 9, 9, 4, 9, 9, 3, 1, 6, 7, 7, 7], [7, 7, 7, 6, 1, 3, 9, 9, 4, 9, 3, 1, 1, 6, 7, 7, 7], [7, 7, 7, 6, 1, 1, 3, 9, 4, 3, 1, 1, 6, 7, 7, 7, 7], [7, 7, 7, 7, 6, 1, 1, 3, 4, 1, 1, 6, 7, 7, 7, 7, 7], [7, 7, 7, 7, 6, 1, 6, 1, 4, 6, 6, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 6, 7, 6, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7]]}, {"input": [[7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 1, 1, 4, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 1, 2, 2, 4, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 1, 2, 9, 9, 4, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 1, 2, 9, 9, 4, 7, 7, 7, 7, 7, 7, 7], [7, 7, 1, 2, 9, 9, 9, 4, 7, 7, 7, 7, 7, 7, 7], [8, 7, 7, 1, 2, 9, 9, 8, 7, 7, 7, 7, 7, 7, 8], [7, 7, 7, 1, 2, 9, 9, 4, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 1, 2, 9, 9, 4, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 1, 2, 2, 4, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 1, 1, 4, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7]], "output": [[7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 1, 1, 4, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 1, 2, 2, 4, 1, 1, 7, 7, 7, 7, 7], [7, 7, 7, 1, 2, 9, 9, 4, 2, 2, 1, 7, 7, 7, 7], [7, 7, 7, 1, 2, 9, 9, 4, 9, 9, 2, 1, 7, 7, 7], [7, 7, 1, 2, 9, 9, 9, 4, 9, 9, 2, 1, 7, 7, 7], [8, 7, 7, 1, 2, 9, 9, 8, 9, 9, 2, 1, 7, 7, 8], [7, 7, 7, 1, 2, 9, 9, 4, 9, 9, 9, 2, 1, 7, 7], [7, 7, 7, 1, 2, 9, 9, 4, 9, 9, 2, 1, 7, 7, 7], [7, 7, 7, 7, 1, 2, 2, 4, 9, 9, 2, 1, 7, 7, 7], [7, 7, 7, 7, 7, 1, 1, 4, 2, 2, 1, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 4, 1, 1, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7]]}, {"input": [[7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 5, 5, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 5, 5, 9, 9, 4, 7, 7, 7, 7, 7, 7, 7, 7], [8, 7, 5, 5, 9, 9, 1, 1, 8, 7, 7, 7, 7, 7, 7, 7, 8], [7, 7, 5, 9, 1, 1, 1, 1, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 5, 9, 1, 1, 1, 1, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 5, 9, 6, 6, 6, 6, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 5, 9, 6, 6, 6, 6, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 5, 5, 9, 9, 6, 6, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 5, 5, 9, 9, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 5, 5, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7]], "output": [[7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 4, 5, 5, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 4, 9, 9, 5, 5, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 4, 6, 6, 9, 9, 5, 5, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 4, 6, 6, 6, 6, 9, 5, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 4, 6, 6, 6, 6, 9, 5, 7, 7], [7, 7, 7, 7, 7, 7, 5, 5, 4, 1, 1, 1, 1, 9, 5, 7, 7], [7, 7, 7, 7, 5, 5, 9, 9, 4, 1, 1, 1, 1, 9, 5, 7, 7], [8, 7, 5, 5, 9, 9, 1, 1, 8, 1, 1, 9, 9, 5, 5, 7, 8], [7, 7, 5, 9, 1, 1, 1, 1, 4, 9, 9, 5, 5, 7, 7, 7, 7], [7, 7, 5, 9, 1, 1, 1, 1, 4, 5, 5, 7, 7, 7, 7, 7, 7], [7, 7, 5, 9, 6, 6, 6, 6, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 5, 9, 6, 6, 6, 6, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 5, 5, 9, 9, 6, 6, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 5, 5, 9, 9, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 5, 5, 4, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7]]}], "test": [{"input": [[7, 7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 3, 3, 4, 7, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 3, 3, 3, 9, 4, 7, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 3, 0, 0, 9, 4, 7, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 3, 3, 0, 0, 9, 4, 7, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 3, 3, 2, 2, 2, 9, 4, 7, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 3, 3, 2, 2, 2, 2, 9, 4, 7, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 3, 2, 2, 2, 2, 2, 9, 4, 7, 7, 7, 7, 7, 7, 7, 7, 7], [8, 7, 3, 5, 5, 5, 5, 5, 9, 8, 7, 7, 7, 7, 7, 7, 7, 7, 8], [7, 7, 3, 5, 5, 5, 5, 5, 9, 4, 7, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 3, 1, 1, 1, 1, 1, 9, 4, 7, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 3, 1, 1, 1, 1, 1, 9, 4, 7, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 3, 3, 1, 1, 1, 1, 9, 4, 7, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 3, 3, 6, 6, 6, 9, 4, 7, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 3, 3, 3, 7, 9, 4, 7, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 3, 3, 3, 4, 7, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7, 7]], "output": [[7, 7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 3, 3, 4, 3, 3, 3, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 3, 3, 3, 9, 4, 9, 7, 3, 3, 3, 7, 7, 7, 7], [7, 7, 7, 7, 7, 3, 0, 0, 9, 4, 9, 6, 6, 6, 3, 3, 7, 7, 7], [7, 7, 7, 7, 3, 3, 0, 0, 9, 4, 9, 1, 1, 1, 1, 3, 3, 7, 7], [7, 7, 7, 3, 3, 2, 2, 2, 9, 4, 9, 1, 1, 1, 1, 1, 3, 7, 7], [7, 7, 3, 3, 2, 2, 2, 2, 9, 4, 9, 1, 1, 1, 1, 1, 3, 7, 7], [7, 7, 3, 2, 2, 2, 2, 2, 9, 4, 9, 5, 5, 5, 5, 5, 3, 7, 7], [8, 7, 3, 5, 5, 5, 5, 5, 9, 8, 9, 5, 5, 5, 5, 5, 3, 7, 8], [7, 7, 3, 5, 5, 5, 5, 5, 9, 4, 9, 2, 2, 2, 2, 2, 3, 7, 7], [7, 7, 3, 1, 1, 1, 1, 1, 9, 4, 9, 2, 2, 2, 2, 3, 3, 7, 7], [7, 7, 3, 1, 1, 1, 1, 1, 9, 4, 9, 2, 2, 2, 3, 3, 7, 7, 7], [7, 7, 3, 3, 1, 1, 1, 1, 9, 4, 9, 0, 0, 3, 3, 7, 7, 7, 7], [7, 7, 7, 3, 3, 6, 6, 6, 9, 4, 9, 0, 0, 3, 7, 7, 7, 7, 7], [7, 7, 7, 7, 3, 3, 3, 7, 9, 4, 9, 3, 3, 3, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 3, 3, 3, 4, 3, 3, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7, 7], [7, 7, 7, 7, 7, 7, 7, 7, 7, 4, 7, 7, 7, 7, 7, 7, 7, 7, 7]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/fixtures/symmetry_complete/f25ffba3.json b/research/universal_core/holonomy_v2_2/tests/fixtures/symmetry_complete/f25ffba3.json new file mode 100644 index 00000000..e488c910 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/fixtures/symmetry_complete/f25ffba3.json @@ -0,0 +1 @@ +{"train": [{"input": [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 2], [0, 0, 0, 2], [0, 8, 0, 2], [0, 3, 8, 2], [3, 3, 8, 2]], "output": [[3, 3, 8, 2], [0, 3, 8, 2], [0, 8, 0, 2], [0, 0, 0, 2], [0, 0, 0, 2], [0, 0, 0, 2], [0, 0, 0, 2], [0, 8, 0, 2], [0, 3, 8, 2], [3, 3, 8, 2]]}, {"input": [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 9], [0, 0, 3, 9], [0, 0, 3, 9], [2, 0, 3, 9], [2, 4, 3, 9]], "output": [[2, 4, 3, 9], [2, 0, 3, 9], [0, 0, 3, 9], [0, 0, 3, 9], [0, 0, 0, 9], [0, 0, 0, 9], [0, 0, 3, 9], [0, 0, 3, 9], [2, 0, 3, 9], [2, 4, 3, 9]]}], "test": [{"input": [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 1, 0, 0], [7, 1, 0, 0], [7, 1, 3, 0], [7, 1, 3, 3], [7, 1, 4, 4]], "output": [[7, 1, 4, 4], [7, 1, 3, 3], [7, 1, 3, 0], [7, 1, 0, 0], [0, 1, 0, 0], [0, 1, 0, 0], [7, 1, 0, 0], [7, 1, 3, 0], [7, 1, 3, 3], [7, 1, 4, 4]]}]} \ No newline at end of file diff --git a/research/universal_core/holonomy_v2_2/tests/test_axis_collapse.py b/research/universal_core/holonomy_v2_2/tests/test_axis_collapse.py new file mode 100644 index 00000000..e61db773 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_axis_collapse.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from research.universal_core.holonomy_v2_2 import grid_induction +from research.universal_core.holonomy_v2_2.axis_collapse import ( + apply_axis_collapse, + axis_collapse_programs, +) + + +_FIXTURES = Path(__file__).parent / "fixtures" / "axis_collapse" +_TASK_IDS = ( + "2dee498d", + "746b3537", + "7b7f7511", + "ce8d95cc", + "e1baa8a4", + "eb5a1d5d", +) + + +def load_payload(task_id: str) -> dict[str, Any]: + return json.loads((_FIXTURES / f"{task_id}.json").read_text(encoding="utf-8")) + + +def public_task(task_id: str) -> dict[str, Any]: + payload = load_payload(task_id) + return { + "task_id": task_id, + "instructions": "Infer the exact grid mapping.", + "demonstrations": payload["train"], + "hidden_inputs": [item["input"] for item in payload["test"]], + } + + +def expected_predictions(task_id: str) -> list[dict[str, Any]]: + return [ + {"status": "ACCEPTED", "prediction": item["output"]} + for item in load_payload(task_id)["test"] + ] + + +def test_adjacent_run_collapse_rows_columns_and_both() -> None: + source = [ + [1, 1, 2], + [1, 1, 2], + [3, 3, 4], + ] + + assert apply_axis_collapse({ + "kind": "axis_duplicate_collapse", + "scope": "adjacent_runs", + "axes": "rows", + }, source) == [ + [1, 1, 2], + [3, 3, 4], + ] + assert apply_axis_collapse({ + "kind": "axis_duplicate_collapse", + "scope": "adjacent_runs", + "axes": "columns", + }, source) == [ + [1, 2], + [1, 2], + [3, 4], + ] + assert apply_axis_collapse({ + "kind": "axis_duplicate_collapse", + "scope": "adjacent_runs", + "axes": "both", + }, source) == [ + [1, 2], + [3, 4], + ] + + +def test_global_first_collapse_preserves_first_occurrence_order() -> None: + source = [ + [1, 2, 1], + [3, 4, 3], + [1, 2, 1], + ] + + assert apply_axis_collapse({ + "kind": "axis_duplicate_collapse", + "scope": "global_first", + "axes": "both", + }, source) == [ + [1, 2], + [3, 4], + ] + + +def test_row_and_column_collapse_commute() -> None: + source = [ + [1, 1, 2, 2], + [1, 1, 2, 2], + [3, 3, 4, 4], + ] + rows_then_columns = apply_axis_collapse({ + "kind": "axis_duplicate_collapse", + "scope": "adjacent_runs", + "axes": "both", + }, source) + columns_only = apply_axis_collapse({ + "kind": "axis_duplicate_collapse", + "scope": "adjacent_runs", + "axes": "columns", + }, source) + columns_then_rows = apply_axis_collapse({ + "kind": "axis_duplicate_collapse", + "scope": "adjacent_runs", + "axes": "rows", + }, columns_only) + + assert rows_then_columns == columns_then_rows == [[1, 2], [3, 4]] + + +def test_program_derivation_skips_when_no_axis_reduction_occurs() -> None: + programs, reason = axis_collapse_programs(( + ([[1, 2], [3, 4]], [[1, 2], [3, 4]]), + )) + + assert programs == () + assert reason == "NO_DUPLICATE_AXIS_REDUCTION" + + +def test_six_training_gaps_become_exact_trace_explained_acceptances() -> None: + for task_id in _TASK_IDS: + result = grid_induction.run_public_task_v2_2( + public_task(task_id), + engine=None, + mechanistic=True, + ) + + assert result["predictions"] == expected_predictions(task_id) + trace = result["mechanistic_trace"] + final = [item for item in trace["events"] if item["event"] == "final_decision"][0] + successes = [ + item + for item in trace["events"] + if item.get("event") == "hidden_execution" + and item.get("candidate_kind") == "axis_duplicate_collapse" + and item.get("outcome") == "SUCCEEDED" + ] + assert trace["terminal_status"] == "ACCEPTED" + assert final["selected_candidate_id"].startswith("axis_collapse:") + assert successes diff --git a/research/universal_core/holonomy_v2_2/tests/test_axis_collapse_fixtures.py b/research/universal_core/holonomy_v2_2/tests/test_axis_collapse_fixtures.py new file mode 100644 index 00000000..0ce932c5 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_axis_collapse_fixtures.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +import hashlib +from pathlib import Path + + +_FIXTURES = Path(__file__).parent / "fixtures" / "axis_collapse" +_EXPECTED_SHA256 = { + "2dee498d.json": "38e27a5c17bf593547182b8914cf044052b158ebe4c5e802d14fb6af11458388", + "746b3537.json": "2fa42394d0d48c4c9534aac1ce000ab97ff804f9d69a34d16126f6a17a757686", + "7b7f7511.json": "28480a33bbe9ff487d845a62a7c21f3d5bc2d1de493a5f91e0c19b57b1aa1d60", + "ce8d95cc.json": "728025ba05aacc3b678ecd7a75d605bf8d9477c2bcd2b667e031c32406233c41", + "e1baa8a4.json": "16411bc668a55e51401716d4fc48424413443a8b21f97a670ce14df9ba325f9d", + "eb5a1d5d.json": "96a1772a93e9cecf1050532b031ad3605390cc5c907cb4b919d4e095eee2f54d", +} + + +def test_axis_collapse_fixtures_match_pinned_training_payloads() -> None: + actual = { + path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in sorted(_FIXTURES.glob("*.json")) + } + + assert actual == _EXPECTED_SHA256 diff --git a/research/universal_core/holonomy_v2_2/tests/test_bbox_complete.py b/research/universal_core/holonomy_v2_2/tests/test_bbox_complete.py new file mode 100644 index 00000000..613943f1 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_bbox_complete.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from research.universal_core.holonomy_v2_2 import grid_induction +from research.universal_core.holonomy_v2_2.bbox_complete import ( + apply_bbox_complete, + bbox_complete_programs, +) + + +_FIXTURES = Path(__file__).parent / "fixtures" / "bbox" + + +def load_payload(task_id: str) -> dict[str, Any]: + return json.loads((_FIXTURES / f"{task_id}.json").read_text(encoding="utf-8")) + + +def public_task(task_id: str) -> dict[str, Any]: + payload = load_payload(task_id) + return { + "task_id": task_id, + "instructions": "Infer the exact grid mapping.", + "demonstrations": payload["train"], + "hidden_inputs": [item["input"] for item in payload["test"]], + } + + +def expected_predictions(task_id: str) -> list[dict[str, Any]]: + payload = load_payload(task_id) + return [ + {"status": "ACCEPTED", "prediction": item["output"]} + for item in payload["test"] + ] + + +def test_global_fill_preserves_existing_cells() -> None: + program = { + "kind": "bounding_box_complete", + "background": 0, + "fill_color": 2, + "mode": "fill", + "scope": "global", + } + + assert apply_bbox_complete(program, [ + [0, 8, 0], + [0, 0, 0], + [0, 0, 8], + ]) == [ + [0, 8, 2], + [0, 2, 2], + [0, 2, 8], + ] + + +def test_global_outline_only_completes_border() -> None: + program = { + "kind": "bounding_box_complete", + "background": 0, + "fill_color": 1, + "mode": "outline", + "scope": "global", + } + + assert apply_bbox_complete(program, [ + [8, 0, 0, 8], + [0, 0, 0, 0], + [8, 0, 0, 8], + ]) == [ + [8, 1, 1, 8], + [1, 0, 0, 1], + [8, 1, 1, 8], + ] + + +def test_component_fill_completes_each_object_independently() -> None: + program = { + "kind": "bounding_box_complete", + "background": 0, + "fill_color": 1, + "mode": "fill", + "scope": "components", + "connectivity": 4, + } + + assert apply_bbox_complete(program, [ + [8, 8, 0, 0, 8], + [8, 0, 0, 0, 8], + [0, 0, 0, 0, 8], + ]) == [ + [8, 8, 0, 0, 8], + [8, 1, 0, 0, 8], + [0, 0, 0, 0, 8], + ] + + +def test_program_enumeration_exposes_fill_outline_global_and_component_modes() -> None: + demos = (( + [[8, 0], [0, 8]], + [[8, 1], [1, 8]], + ),) + + programs, reason = bbox_complete_programs(demos) + + assert reason is None + assert any(item["scope"] == "global" and item["mode"] == "fill" for item in programs) + assert any(item["scope"] == "global" and item["mode"] == "outline" for item in programs) + assert any(item["scope"] == "components" and item["connectivity"] == 4 for item in programs) + assert any(item["scope"] == "components" and item["connectivity"] == 8 for item in programs) + + +def test_three_training_defects_become_exact_trace_explained_acceptances() -> None: + for task_id in ("3aa6fb7a", "6d75e8bb", "e7639916"): + result = grid_induction.run_public_task_v2_2( + public_task(task_id), + engine=None, + mechanistic=True, + ) + + assert result["predictions"] == expected_predictions(task_id) + trace = result["mechanistic_trace"] + final = [item for item in trace["events"] if item["event"] == "final_decision"] + assert trace["terminal_status"] == "ACCEPTED" + assert final[0]["status"] == "ACCEPTED" + assert final[0]["selected_candidate_id"].startswith("bbox_complete:") + assert any( + item.get("event") == "hidden_execution" + and item.get("candidate_kind") == "bounding_box_complete" + and item.get("outcome") == "SUCCEEDED" + for item in trace["events"] + ) + + +def test_connectivity_disagreement_remains_ambiguous() -> None: + result = grid_induction.run_public_task_v2_2( + public_task("60b61512"), + engine=None, + mechanistic=True, + ) + trace = result["mechanistic_trace"] + classes = [item for item in trace["events"] if item["event"] == "equivalence_class"] + bbox_members = [ + candidate_id + for item in classes + for candidate_id in item["candidate_ids"] + if candidate_id.startswith("bbox_complete:") + ] + + assert result["predictions"] == [ + {"status": "AMBIGUOUS_PROGRAM", "prediction": None} + ] + assert trace["terminal_status"] == "AMBIGUOUS_PROGRAM" + assert len(classes) > 1 + assert bbox_members diff --git a/research/universal_core/holonomy_v2_2/tests/test_block_reduce.py b/research/universal_core/holonomy_v2_2/tests/test_block_reduce.py new file mode 100644 index 00000000..60f5bc89 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_block_reduce.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from research.universal_core.holonomy_v2_2 import grid_induction +from research.universal_core.holonomy_v2_2.block_reduce import ( + apply_block_reduce, + block_reduce_programs, +) + + +_FIXTURES = Path(__file__).parent / "fixtures" / "block_reduce" +_TASK_IDS = ("5614dbcf", "68b67ca3") + + +def load_payload(task_id: str) -> dict[str, Any]: + return json.loads((_FIXTURES / f"{task_id}.json").read_text(encoding="utf-8")) + + +def public_task(task_id: str) -> dict[str, Any]: + payload = load_payload(task_id) + return { + "task_id": task_id, + "instructions": "Infer the exact grid mapping.", + "demonstrations": payload["train"], + "hidden_inputs": [item["input"] for item in payload["test"]], + } + + +def expected_predictions(task_id: str) -> list[dict[str, Any]]: + return [ + {"status": "ACCEPTED", "prediction": item["output"]} + for item in load_payload(task_id)["test"] + ] + + +def test_strict_mode_reduces_each_fixed_block() -> None: + program = { + "kind": "fixed_block_reduce", + "row_factor": 2, + "col_factor": 2, + "reducer": "strict_mode", + } + source = [ + [1, 1, 2, 2], + [1, 0, 2, 2], + [3, 3, 4, 4], + [3, 3, 4, 0], + ] + + assert apply_block_reduce(program, source) == [[1, 2], [3, 4]] + + +def test_strict_mode_rejects_tied_block() -> None: + program = { + "kind": "fixed_block_reduce", + "row_factor": 2, + "col_factor": 2, + "reducer": "strict_mode", + } + + with pytest.raises(ValueError, match="unique mode"): + apply_block_reduce(program, [[1, 2], [2, 1]]) + + +def test_unique_non_background_reduces_sparse_blocks() -> None: + program = { + "kind": "fixed_block_reduce", + "row_factor": 2, + "col_factor": 2, + "reducer": "unique_non_background", + "background": 0, + } + source = [ + [3, 0, 0, 4], + [0, 0, 0, 0], + [0, 5, 0, 0], + [0, 0, 0, 0], + ] + + assert apply_block_reduce(program, source) == [[3, 4], [5, 0]] + + +def test_unique_non_background_rejects_conflicting_block() -> None: + program = { + "kind": "fixed_block_reduce", + "row_factor": 2, + "col_factor": 2, + "reducer": "unique_non_background", + "background": 0, + } + + with pytest.raises(ValueError, match="one non-background color"): + apply_block_reduce(program, [[1, 2], [0, 0]]) + + +def test_program_derivation_exposes_stable_block_factors() -> None: + demos = ( + ( + [[1, 1, 2, 2], [1, 0, 2, 2], [3, 3, 4, 4], [3, 3, 4, 0]], + [[1, 2], [3, 4]], + ), + ( + [[5, 5, 6, 6], [5, 0, 6, 6], [7, 7, 8, 8], [7, 7, 8, 0]], + [[5, 6], [7, 8]], + ), + ) + + programs, reason = block_reduce_programs(demos) + + assert reason is None + assert { + (item["row_factor"], item["col_factor"], item["reducer"]) + for item in programs + } >= {(2, 2, "strict_mode")} + + +def test_program_derivation_rejects_unstable_block_factors() -> None: + demos = ( + ([[1, 1], [1, 1]], [[1]]), + ([[2, 2, 2]], [[2]]), + ) + + programs, reason = block_reduce_programs(demos) + + assert programs == () + assert reason == "NO_STABLE_BLOCK_FACTORS" + + +def test_two_training_gaps_become_exact_trace_explained_acceptances() -> None: + for task_id in _TASK_IDS: + result = grid_induction.run_public_task_v2_2( + public_task(task_id), + engine=None, + mechanistic=True, + ) + + assert result["predictions"] == expected_predictions(task_id) + trace = result["mechanistic_trace"] + final = [item for item in trace["events"] if item["event"] == "final_decision"][0] + successes = [ + item + for item in trace["events"] + if item.get("event") == "hidden_execution" + and item.get("candidate_kind") == "fixed_block_reduce" + and item.get("outcome") == "SUCCEEDED" + ] + assert trace["terminal_status"] == "ACCEPTED" + assert final["selected_candidate_id"].startswith("block_reduce:") + assert successes diff --git a/research/universal_core/holonomy_v2_2/tests/test_block_reduce_fixtures.py b/research/universal_core/holonomy_v2_2/tests/test_block_reduce_fixtures.py new file mode 100644 index 00000000..87c46355 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_block_reduce_fixtures.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +import hashlib +from pathlib import Path + + +_FIXTURES = Path(__file__).parent / "fixtures" / "block_reduce" +_EXPECTED_SHA256 = { + "5614dbcf.json": "0751d867504f022964f0d4ef909ae9e68c928f499841f6a3e8b7143791fca53e", + "68b67ca3.json": "767e0baf6a71a7f288ab5a08f13a4ecc65ac5a43144586dec0c3ccd2eff5edb4", +} + + +def test_block_reduce_fixtures_match_pinned_training_payloads() -> None: + actual = { + path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in sorted(_FIXTURES.glob("*.json")) + } + assert actual == _EXPECTED_SHA256 diff --git a/research/universal_core/holonomy_v2_2/tests/test_component_outlier.py b/research/universal_core/holonomy_v2_2/tests/test_component_outlier.py new file mode 100644 index 00000000..7d11350d --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_component_outlier.py @@ -0,0 +1,245 @@ +from __future__ import annotations + +from typing import Any + +import pytest + +from research.universal_core.holonomy_v2_2 import grid_induction +from research.universal_core.holonomy_v2_2.component_outlier import ( + apply_component_outlier, + component_outlier_programs, +) + + +def test_outlier_programs_infer_unique_minimum_and_uniform_peers() -> None: + demos = ( + ( + [[8, 8, 0, 8, 8, 8, 0, 8, 8, 8]], + [[2, 2, 0, 1, 1, 1, 0, 1, 1, 1]], + ), + ( + [[8, 0, 8, 8, 0, 8, 8]], + [[2, 0, 1, 1, 0, 1, 1]], + ), + ) + + programs, reason = component_outlier_programs(demos) + + assert reason is None + assert programs == ( + { + "kind": "component_area_outlier_color", + "background": 0, + "connectivity": 4, + "selector": "minimum", + "selected_color": 2, + "other_color": 1, + }, + { + "kind": "component_area_outlier_color", + "background": 0, + "connectivity": 8, + "selector": "minimum", + "selected_color": 2, + "other_color": 1, + }, + ) + + +def test_hidden_topology_rejects_four_connectivity_and_accepts_eight() -> None: + hidden = [ + [0, 8, 8, 0, 0, 0, 8, 8, 8, 0], + [0, 0, 8, 0, 0, 0, 0, 0, 8, 0], + [0, 8, 0, 0, 0, 0, 0, 8, 0, 0], + [0, 8, 8, 8, 0, 0, 8, 8, 8, 8], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 8, 8, 8, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 8, 0, 0, 0, 0], + [0, 0, 0, 0, 8, 0, 0, 0, 0, 0], + [0, 0, 0, 8, 8, 8, 8, 0, 0, 0], + ] + base = { + "kind": "component_area_outlier_color", + "background": 0, + "selector": "minimum", + "selected_color": 2, + "other_color": 1, + } + + with pytest.raises(ValueError, match="component peer areas must be uniform"): + apply_component_outlier({**base, "connectivity": 4}, hidden) + + assert apply_component_outlier({**base, "connectivity": 8}, hidden) == [ + [0, 2, 2, 0, 0, 0, 1, 1, 1, 0], + [0, 0, 2, 0, 0, 0, 0, 0, 1, 0], + [0, 2, 0, 0, 0, 0, 0, 1, 0, 0], + [0, 2, 2, 2, 0, 0, 1, 1, 1, 1], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 1, 1, 1, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 1, 0, 0, 0, 0], + [0, 0, 0, 0, 1, 0, 0, 0, 0, 0], + [0, 0, 0, 1, 1, 1, 1, 0, 0, 0], + ] + + +def test_outlier_programs_report_missing_common_background() -> None: + programs, reason = component_outlier_programs(( + ([[0, 8, 8]], [[0, 2, 2]]), + ([[3, 8, 8]], [[3, 2, 2]]), + )) + + assert programs == () + assert reason == "NO_BACKGROUND_CANDIDATE" + + +def b230c067_task() -> dict[str, Any]: + return { + "task_id": "b230c067", + "instructions": "Infer the exact grid mapping.", + "demonstrations": [ + { + "input": [ + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 8, 8, 8, 8], + [0, 8, 8, 8, 8, 0, 0, 0, 8, 8], + [0, 0, 0, 8, 8, 0, 0, 0, 8, 8], + [0, 0, 0, 8, 8, 8, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 8, 8, 8, 8, 0, 0, 0], + [0, 0, 0, 0, 0, 8, 8, 0, 0, 0], + [0, 0, 0, 0, 0, 8, 8, 8, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + ], + "output": [ + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 2, 2, 2, 2], + [0, 1, 1, 1, 1, 0, 0, 0, 2, 2], + [0, 0, 0, 1, 1, 0, 0, 0, 2, 2], + [0, 0, 0, 1, 1, 1, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 1, 1, 1, 1, 0, 0, 0], + [0, 0, 0, 0, 0, 1, 1, 0, 0, 0], + [0, 0, 0, 0, 0, 1, 1, 1, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + ], + }, + { + "input": [ + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 8, 8, 8], + [0, 8, 8, 8, 8, 0, 0, 8, 0, 8], + [0, 8, 0, 0, 8, 0, 0, 8, 8, 8], + [0, 8, 8, 8, 8, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 8, 8, 8, 8, 0], + [0, 0, 0, 0, 0, 8, 0, 0, 8, 0], + [0, 0, 0, 0, 0, 8, 8, 8, 8, 0], + ], + "output": [ + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 2, 2, 2], + [0, 1, 1, 1, 1, 0, 0, 2, 0, 2], + [0, 1, 0, 0, 1, 0, 0, 2, 2, 2], + [0, 1, 1, 1, 1, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 1, 1, 1, 1, 0], + [0, 0, 0, 0, 0, 1, 0, 0, 1, 0], + [0, 0, 0, 0, 0, 1, 1, 1, 1, 0], + ], + }, + { + "input": [ + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 8, 8, 8, 0, 0, 0, 0, 0, 0], + [0, 8, 0, 8, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [8, 8, 0, 0, 0, 0, 0, 0, 0, 0], + [8, 8, 0, 8, 8, 8, 0, 0, 0, 0], + [0, 0, 0, 8, 0, 8, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + ], + "output": [ + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 1, 1, 1, 0, 0, 0, 0, 0, 0], + [0, 1, 0, 1, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [2, 2, 0, 0, 0, 0, 0, 0, 0, 0], + [2, 2, 0, 1, 1, 1, 0, 0, 0, 0], + [0, 0, 0, 1, 0, 1, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + ], + }, + ], + "hidden_inputs": [[ + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 8, 8, 0, 0, 0, 8, 8, 8, 0], + [0, 0, 8, 0, 0, 0, 0, 0, 8, 0], + [0, 8, 0, 0, 0, 0, 0, 8, 0, 0], + [0, 8, 8, 8, 0, 0, 8, 8, 8, 8], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 8, 8, 8, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 8, 0, 0, 0, 0], + [0, 0, 0, 0, 8, 0, 0, 0, 0, 0], + [0, 0, 0, 8, 8, 8, 8, 0, 0, 0], + ]], + } + + +def test_exact_rank_failure_becomes_trace_explained_acceptance() -> None: + result = grid_induction.run_public_task_v2_2( + b230c067_task(), + engine=None, + mechanistic=True, + ) + expected = [ + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 2, 2, 0, 0, 0, 1, 1, 1, 0], + [0, 0, 2, 0, 0, 0, 0, 0, 1, 0], + [0, 2, 0, 0, 0, 0, 0, 1, 0, 0], + [0, 2, 2, 2, 0, 0, 1, 1, 1, 1], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 1, 1, 1, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 1, 0, 0, 0, 0], + [0, 0, 0, 0, 1, 0, 0, 0, 0, 0], + [0, 0, 0, 1, 1, 1, 1, 0, 0, 0], + ] + + assert result["predictions"] == [{"status": "ACCEPTED", "prediction": expected}] + events = result["mechanistic_trace"]["events"] + rank_failures = [ + event + for event in events + if event.get("event") == "hidden_execution" + and event.get("candidate_kind") == "component_rank_color" + and event.get("outcome") == "FAILED" + and event.get("exception_type") == "KeyError" + and "unmapped component rank" in event.get("exception_message", "") + ] + assert len(rank_failures) == 2 + outlier_failures = [ + event + for event in events + if event.get("event") == "hidden_execution" + and event.get("candidate_kind") == "component_area_outlier_color" + and event.get("outcome") == "FAILED" + and event.get("exception_type") == "ValueError" + and "component peer areas must be uniform" in event.get("exception_message", "") + ] + outlier_successes = [ + event + for event in events + if event.get("event") == "hidden_execution" + and event.get("candidate_kind") == "component_area_outlier_color" + and event.get("outcome") == "SUCCEEDED" + ] + assert len(outlier_failures) == 1 + assert len(outlier_successes) == 1 + final = [event for event in events if event.get("event") == "final_decision"] + assert final[0]["status"] == "ACCEPTED" + assert final[0]["selected_candidate_id"].startswith("component_outlier:") diff --git a/research/universal_core/holonomy_v2_2/tests/test_component_rank.py b/research/universal_core/holonomy_v2_2/tests/test_component_rank.py new file mode 100644 index 00000000..cd9c3aa0 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_component_rank.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from research.universal_core.holonomy_v2_2.grid_induction import run_public_task_v2_2 + + +def make_task(demos: list[dict[str, object]], hidden: list[object]) -> dict[str, object]: + return { + "task_id": "component-rank", + "instructions": "Infer the exact grid mapping.", + "demonstrations": demos, + "hidden_inputs": hidden, + } + + +def test_components_use_ascending_size_rank_colors() -> None: + value = make_task( + [{ + "input": [[5, 0, 5, 0, 5], [0, 0, 5, 0, 5], [0, 0, 0, 0, 5]], + "output": [[1, 0, 2, 0, 3], [0, 0, 2, 0, 3], [0, 0, 0, 0, 3]], + }], + [[[5, 0, 5, 0, 5], [5, 0, 5, 0, 0], [5, 0, 0, 0, 0]]], + ) + + result = run_public_task_v2_2(value, engine=None) + + assert result["predictions"] == [{ + "status": "ACCEPTED", + "prediction": [[3, 0, 2, 0, 1], [3, 0, 2, 0, 0], [3, 0, 0, 0, 0]], + }] + assert "component_rank_color" in result["evidence"]["candidate_kinds"] + + +def test_middle_size_class_can_map_to_background() -> None: + value = make_task( + [{ + "input": [[4, 0, 4, 0, 4], [0, 0, 4, 0, 4], [0, 0, 0, 0, 4]], + "output": [[1, 0, 0, 0, 2], [0, 0, 0, 0, 2], [0, 0, 0, 0, 2]], + }], + [[[4, 0, 4, 4, 0], [0, 0, 0, 0, 0], [0, 0, 4, 4, 4]]], + ) + + result = run_public_task_v2_2(value, engine=None) + + assert result["predictions"][0] == { + "status": "ACCEPTED", + "prediction": [[1, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 2, 2, 2]], + } + + +def test_unseen_size_rank_returns_execution_failure() -> None: + value = make_task( + [{ + "input": [[5, 0, 5], [0, 0, 5]], + "output": [[1, 0, 2], [0, 0, 2]], + }], + [[[5, 0, 5, 0, 5], [0, 0, 5, 0, 5], [0, 0, 0, 0, 5]]], + ) + + result = run_public_task_v2_2(value, engine=None) + + assert result["predictions"] == [{"status": "EXECUTION_FAILED", "prediction": None}] diff --git a/research/universal_core/holonomy_v2_2/tests/test_component_select.py b/research/universal_core/holonomy_v2_2/tests/test_component_select.py new file mode 100644 index 00000000..4747fada --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_component_select.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +from research.universal_core.holonomy_v2_2.grid_induction import run_public_task_v2_2 + + +def make_task(demos: list[dict[str, object]], hidden: list[object]) -> dict[str, object]: + return { + "task_id": "component-select", + "instructions": "Infer the exact grid mapping.", + "demonstrations": demos, + "hidden_inputs": hidden, + } + + +def test_unique_largest_component_is_cropped() -> None: + value = make_task( + [{ + "input": [ + [0, 2, 0, 3, 3, 0], + [0, 2, 0, 3, 3, 0], + [0, 0, 0, 3, 0, 0], + ], + "output": [[3, 3], [3, 3], [3, 0]], + }], + [[ + [0, 4, 4, 0, 5, 5, 5], + [0, 4, 0, 0, 5, 5, 5], + ]], + ) + + result = run_public_task_v2_2(value, engine=None) + + assert result["predictions"] == [{ + "status": "ACCEPTED", + "prediction": [[5, 5, 5], [5, 5, 5]], + }] + assert "component_area_select" in result["evidence"]["candidate_kinds"] + + +def test_unique_smallest_component_is_cropped() -> None: + value = make_task( + [{ + "input": [ + [0, 2, 2, 2, 0, 7, 7], + [0, 2, 2, 2, 0, 7, 7], + [0, 2, 2, 2, 0, 0, 0], + ], + "output": [[7, 7], [7, 7]], + }], + [[ + [6, 6, 6, 0, 8], + [6, 6, 6, 0, 8], + [6, 6, 6, 0, 0], + ]], + ) + + result = run_public_task_v2_2(value, engine=None) + + assert result["predictions"] == [{ + "status": "ACCEPTED", + "prediction": [[8], [8]], + }] + + +def test_selected_component_crop_preserves_background_holes() -> None: + value = make_task( + [{ + "input": [ + [9, 4, 4, 4, 9, 2], + [9, 4, 9, 4, 9, 2], + [9, 4, 4, 4, 9, 9], + ], + "output": [[4, 4, 4], [4, 9, 4], [4, 4, 4]], + }], + [[ + [9, 3, 3, 3, 9, 6, 6], + [9, 3, 9, 3, 9, 6, 9], + [9, 3, 3, 3, 9, 9, 9], + ]], + ) + + result = run_public_task_v2_2(value, engine=None) + + assert result["predictions"] == [{ + "status": "ACCEPTED", + "prediction": [[3, 3, 3], [3, 9, 3], [3, 3, 3]], + }] + + +def test_hidden_area_tie_returns_execution_failure() -> None: + value = make_task( + [{ + "input": [ + [0, 2, 0, 0, 0, 3, 3, 0, 0], + [0, 0, 0, 0, 0, 3, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0], + ], + "output": [[3, 3], [3, 0]], + }], + [[ + [0, 4, 4, 0, 0, 5, 5, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0], + ]], + ) + + result = run_public_task_v2_2(value, engine=None) + + assert result["predictions"] == [{"status": "EXECUTION_FAILED", "prediction": None}] + + +def test_minimum_and_maximum_selectors_that_disagree_abstain() -> None: + value = make_task( + [{ + "input": [[0, 2, 2], [0, 2, 2]], + "output": [[2, 2], [2, 2]], + }], + [[[3, 0, 4, 4], [0, 0, 4, 4]]], + ) + + result = run_public_task_v2_2(value, engine=None) + + assert result["predictions"] == [{"status": "AMBIGUOUS_PROGRAM", "prediction": None}] diff --git a/research/universal_core/holonomy_v2_2/tests/test_gravity_compact.py b/research/universal_core/holonomy_v2_2/tests/test_gravity_compact.py new file mode 100644 index 00000000..63de75a9 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_gravity_compact.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path +from typing import Any + +import pytest + +from research.universal_core.holonomy_v2_2 import grid_induction +from research.universal_core.holonomy_v2_2.gravity_compact import ( + apply_gravity_compact, + gravity_compact_programs, +) + + +_FIXTURES = Path(__file__).parent / "fixtures" / "gravity_compact" +_TASK_IDS = ("1e0a9b12", "3906de3d") + + +def load_payload(task_id: str) -> dict[str, Any]: + return json.loads((_FIXTURES / f"{task_id}.json").read_text(encoding="utf-8")) + + +def public_task(task_id: str) -> dict[str, Any]: + payload = load_payload(task_id) + return { + "task_id": task_id, + "instructions": "Infer the exact grid mapping.", + "demonstrations": payload["train"], + "hidden_inputs": [item["input"] for item in payload["test"]], + } + + +def expected_predictions(task_id: str) -> list[dict[str, Any]]: + return [ + {"status": "ACCEPTED", "prediction": item["output"]} + for item in load_payload(task_id)["test"] + ] + + +def test_vertical_gravity_compacts_up_and_down() -> None: + source = [[1, 0], [0, 2], [3, 0]] + + assert apply_gravity_compact( + {"kind": "axis_gravity_compact", "background": 0, "direction": "up"}, + source, + ) == [[1, 2], [3, 0], [0, 0]] + assert apply_gravity_compact( + {"kind": "axis_gravity_compact", "background": 0, "direction": "down"}, + source, + ) == [[0, 0], [1, 0], [3, 2]] + + +def test_horizontal_gravity_compacts_left_and_right() -> None: + source = [[1, 0, 2, 0], [0, 3, 0, 4]] + + assert apply_gravity_compact( + {"kind": "axis_gravity_compact", "background": 0, "direction": "left"}, + source, + ) == [[1, 2, 0, 0], [3, 4, 0, 0]] + assert apply_gravity_compact( + {"kind": "axis_gravity_compact", "background": 0, "direction": "right"}, + source, + ) == [[0, 0, 1, 2], [0, 0, 3, 4]] + + +def test_gravity_preserves_shape_counts_and_lane_order() -> None: + source = [[4, 0, 7], [0, 8, 0], [4, 0, 9]] + output = apply_gravity_compact( + {"kind": "axis_gravity_compact", "background": 0, "direction": "down"}, + source, + ) + + assert (len(output), len(output[0])) == (len(source), len(source[0])) + assert Counter(cell for row in output for cell in row) == Counter( + cell for row in source for cell in row + ) + for col in range(len(source[0])): + before = [row[col] for row in source if row[col] != 0] + after = [row[col] for row in output if row[col] != 0] + assert after == before + + +def test_gravity_rejects_unknown_direction() -> None: + with pytest.raises(ValueError, match="unknown gravity direction"): + apply_gravity_compact( + {"kind": "axis_gravity_compact", "background": 0, "direction": "diagonal"}, + [[1, 0], [0, 2]], + ) + + +def test_program_derivation_returns_unique_demonstrated_candidate() -> None: + demos = (( + [[1, 0], [0, 2], [3, 0]], + [[0, 0], [1, 0], [3, 2]], + ),) + + programs, reason = gravity_compact_programs(demos) + + assert reason is None + assert programs == ({ + "kind": "axis_gravity_compact", + "background": 0, + "direction": "down", + },) + + +def test_program_derivation_rejects_shape_change() -> None: + programs, reason = gravity_compact_programs(( + ([[1, 0], [0, 2]], [[1, 2]]), + )) + + assert programs == () + assert reason == "NO_SHAPE_PRESERVING_GRAVITY" + + +def test_program_derivation_skips_noop_demonstration() -> None: + programs, reason = gravity_compact_programs(( + ([[1, 2], [0, 0]], [[1, 2], [0, 0]]), + )) + + assert programs == () + assert reason == "NO_GRAVITY_COMPACTION" + + +def test_two_training_gaps_become_exact_trace_explained_acceptances() -> None: + for task_id in _TASK_IDS: + result = grid_induction.run_public_task_v2_2( + public_task(task_id), + engine=None, + mechanistic=True, + ) + + assert result["predictions"] == expected_predictions(task_id) + trace = result["mechanistic_trace"] + final = [item for item in trace["events"] if item["event"] == "final_decision"][0] + successes = [ + item + for item in trace["events"] + if item.get("event") == "hidden_execution" + and item.get("candidate_kind") == "axis_gravity_compact" + and item.get("outcome") == "SUCCEEDED" + ] + assert trace["terminal_status"] == "ACCEPTED" + assert final["selected_candidate_id"].startswith("gravity_compact:") + assert successes diff --git a/research/universal_core/holonomy_v2_2/tests/test_gravity_compact_fixtures.py b/research/universal_core/holonomy_v2_2/tests/test_gravity_compact_fixtures.py new file mode 100644 index 00000000..df4f7408 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_gravity_compact_fixtures.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import hashlib +from pathlib import Path + + +_FIXTURES = Path(__file__).parent / "fixtures" / "gravity_compact" +_EXPECTED_SHA256 = { + "1e0a9b12.json": "6a71dbedc8403bd28fa76bd8b472597a8ecaab3af31151a2a3b72baeadd5e617", + "3906de3d.json": "b4c20d433380e00840a75c6976ad0d0fabd80e052ab77b47422d64b1cae5f6dc", +} + + +def test_gravity_compact_fixtures_match_pinned_training_payloads() -> None: + actual = { + path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in sorted(_FIXTURES.glob("*.json")) + } + + assert actual == _EXPECTED_SHA256 diff --git a/research/universal_core/holonomy_v2_2/tests/test_grid_induction.py b/research/universal_core/holonomy_v2_2/tests/test_grid_induction.py new file mode 100644 index 00000000..02a32da6 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_grid_induction.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +from typing import Any + +from research.universal_core.holonomy_v2_2 import grid_induction + + +def task(demos: list[dict[str, object]], hidden: list[object], task_id: str = "grid-task") -> dict[str, object]: + return { + "task_id": task_id, + "instructions": "Infer the exact grid mapping.", + "demonstrations": demos, + "hidden_inputs": hidden, + } + + +def test_color_map_is_inferred_across_all_demonstrations() -> None: + value = task( + [ + {"input": [[0, 1], [1, 0]], "output": [[2, 3], [3, 2]]}, + {"input": [[1, 0]], "output": [[3, 2]]}, + ], + [[[0, 1, 0]]], + ) + + result = grid_induction.run_public_task_v2_2(value, engine=None) + + assert result["predictions"] == [{"status": "ACCEPTED", "prediction": [[2, 3, 2]]}] + assert result["program_digest"] is not None + + +def test_rotation_and_reflection_candidates_predict_hidden_grids() -> None: + rotation = task( + [{"input": [[1, 2], [3, 4]], "output": [[3, 1], [4, 2]]}], + [[[5, 6], [7, 8]]], + "rotate", + ) + reflection = task( + [{"input": [[1, 2, 3], [4, 5, 6]], "output": [[3, 2, 1], [6, 5, 4]]}], + [[[7, 8], [9, 0]]], + "reflect", + ) + + rotated = grid_induction.run_public_task_v2_2(rotation, engine=None) + reflected = grid_induction.run_public_task_v2_2(reflection, engine=None) + + assert rotated["predictions"][0] == {"status": "ACCEPTED", "prediction": [[7, 5], [8, 6]]} + assert reflected["predictions"][0] == {"status": "ACCEPTED", "prediction": [[8, 7], [0, 9]]} + + +def test_cell_scale_and_whole_grid_tile_are_distinct() -> None: + scale = task( + [{"input": [[1, 2]], "output": [[1, 1, 2, 2], [1, 1, 2, 2]]}], + [[[3], [4]]], + "scale", + ) + tile = task( + [{"input": [[1, 2]], "output": [[1, 2, 1, 2], [1, 2, 1, 2]]}], + [[[3], [4]]], + "tile", + ) + + scaled = grid_induction.run_public_task_v2_2(scale, engine=None) + tiled = grid_induction.run_public_task_v2_2(tile, engine=None) + + assert scaled["predictions"][0]["prediction"] == [[3, 3], [3, 3], [4, 4], [4, 4]] + assert tiled["predictions"][0]["prediction"] == [[3, 3], [4, 4], [3, 3], [4, 4]] + + +def test_crop_uses_a_demonstrated_background_value() -> None: + value = task( + [ + { + "input": [[0, 0, 0, 0], [0, 4, 4, 0], [0, 0, 0, 0]], + "output": [[4, 4]], + }, + { + "input": [[0, 5, 0], [0, 5, 0], [0, 0, 0]], + "output": [[5], [5]], + }, + ], + [[[0, 0, 0], [6, 6, 0], [0, 0, 0]]], + ) + + result = grid_induction.run_public_task_v2_2(value, engine=None) + + assert result["predictions"] == [{"status": "ACCEPTED", "prediction": [[6, 6]]}] + + +def test_distinct_hidden_outputs_from_fitting_programs_abstain() -> None: + value = task( + [{"input": [[1, 1], [1, 1]], "output": [[1, 1], [1, 1]]}], + [[[1, 2], [3, 4]]], + ) + + result = grid_induction.run_public_task_v2_2(value, engine=None) + + assert result["program_digest"] is None + assert result["predictions"] == [{"status": "AMBIGUOUS_PROGRAM", "prediction": None}] + + +def test_no_fitting_grid_program_returns_grammar_exhausted() -> None: + value = task( + [{"input": [[1, 2], [3, 4]], "output": [[9, 9, 9], [9, 9, 9], [9, 9, 9]]}], + [[[5]]], + ) + + result = grid_induction.run_public_task_v2_2(value, engine=None) + + assert result["program_digest"] is None + assert result["predictions"] == [{"status": "GRAMMAR_EXHAUSTED", "prediction": None}] + + +def test_program_identity_does_not_depend_on_task_id() -> None: + demos = [{"input": [[1, 2]], "output": [[2, 1]]}] + first = grid_induction.run_public_task_v2_2(task(demos, [[[3, 4]]], "first"), engine=None) + second = grid_induction.run_public_task_v2_2(task(demos, [[[3, 4]]], "second"), engine=None) + + assert first["program_digest"] == second["program_digest"] + assert first["freeze_digest"] == second["freeze_digest"] + + +def test_non_grid_task_delegates_to_frozen_v2(monkeypatch: Any) -> None: + expected = {"task_id": "text", "predictions": [{"status": "ACCEPTED", "prediction": "B"}]} + calls: list[tuple[object, object]] = [] + + def fake_delegate(value: object, engine: object) -> dict[str, object]: + calls.append((value, engine)) + return expected + + monkeypatch.setattr(grid_induction, "run_public_task_v2", fake_delegate) + value = task([{"input": "A", "output": "B"}], ["A"], "text") + marker = object() + + assert grid_induction.run_public_task_v2_2(value, marker) is expected + assert calls == [(value, marker)] diff --git a/research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py b/research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py new file mode 100644 index 00000000..7cd995e1 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_grid_mechanistic.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +import json +from typing import Any + +from research.universal_core.holonomy_v2_2 import grid_induction +from research.universal_core.holonomy_v2_2.mechanistic_trace import validate_trace + + +def task( + demos: list[dict[str, object]], + hidden: list[object], + task_id: str = "mechanistic-grid", +) -> dict[str, object]: + return { + "task_id": task_id, + "instructions": "Infer the exact grid mapping.", + "demonstrations": demos, + "hidden_inputs": hidden, + } + + +def without_trace(result: dict[str, Any]) -> dict[str, Any]: + copied = dict(result) + copied.pop("mechanistic_trace", None) + return copied + + +def events(trace: dict[str, Any], name: str) -> list[dict[str, Any]]: + return [item for item in trace["events"] if item["event"] == name] + + +def test_mechanistic_mode_has_exact_compact_parity() -> None: + value = task( + [{"input": [[0, 1]], "output": [[2, 3]]}], + [[[1, 0]]], + "parity", + ) + + compact = grid_induction.run_public_task_v2_2(value, engine=None) + traced = grid_induction.run_public_task_v2_2(value, engine=None, mechanistic=True) + + assert "mechanistic_trace" not in compact + assert without_trace(traced) == compact + validate_trace(traced["mechanistic_trace"]) + + +def test_trace_records_every_candidate_demonstration_and_terminal_fit() -> None: + value = task( + [{"input": [[1, 2]], "output": [[2, 1]]}], + [[[3, 4]]], + "candidate-ledger", + ) + + trace = grid_induction.run_public_task_v2_2( + value, + engine=None, + mechanistic=True, + )["mechanistic_trace"] + proposed = events(trace, "candidate_proposed") + executions = events(trace, "demo_execution") + terminal = events(trace, "candidate_fit_decision") + + assert proposed + assert len(proposed) == len(terminal) + assert len(executions) == len(proposed) + assert {item["decision"] for item in terminal} >= { + "FITS_ALL_DEMOS", + "DEMO_MISMATCH", + } + assert all("generator" in item and "program_ref" in item for item in proposed) + validate_trace(trace) + + +def test_hidden_component_rank_key_error_is_fully_recorded() -> None: + value = task( + [{ + "input": [[5, 0, 5], [0, 0, 5]], + "output": [[1, 0, 2], [0, 0, 2]], + }], + [[[5, 0, 5, 0, 5], [0, 0, 5, 0, 5], [0, 0, 0, 0, 5]]], + "unseen-rank", + ) + + result = grid_induction.run_public_task_v2_2(value, None, mechanistic=True) + rank_failures = [ + item + for item in events(result["mechanistic_trace"], "hidden_execution") + if item["outcome"] == "FAILED" + and item["candidate_kind"] == "component_rank_color" + ] + + assert result["predictions"] == [{"status": "EXECUTION_FAILED", "prediction": None}] + assert rank_failures + assert all(item["exception_type"] == "KeyError" for item in rank_failures) + assert all("unmapped component rank" in item["exception_message"] for item in rank_failures) + + +def test_trace_replay_is_deterministic_and_task_id_independent() -> None: + demos = [{"input": [[1, 2]], "output": [[2, 1]]}] + first = grid_induction.run_public_task_v2_2( + task(demos, [[[3, 4]]], "first"), + None, + mechanistic=True, + )["mechanistic_trace"] + replay = grid_induction.run_public_task_v2_2( + task(demos, [[[3, 4]]], "first"), + None, + mechanistic=True, + )["mechanistic_trace"] + renamed = grid_induction.run_public_task_v2_2( + task(demos, [[[3, 4]]], "second"), + None, + mechanistic=True, + )["mechanistic_trace"] + + assert json.dumps(first, sort_keys=True) == json.dumps(replay, sort_keys=True) + assert first["trace_digest"] == renamed["trace_digest"] + assert first["envelope_digest"] != renamed["envelope_digest"] + + +def test_ambiguous_program_trace_lists_every_output_class() -> None: + value = task( + [{"input": [[1, 1], [1, 1]], "output": [[1, 1], [1, 1]]}], + [[[1, 2], [3, 4]]], + "ambiguous", + ) + + result = grid_induction.run_public_task_v2_2(value, None, mechanistic=True) + trace = result["mechanistic_trace"] + classes = events(trace, "equivalence_class") + final = events(trace, "final_decision") + + assert result["predictions"] == [{"status": "AMBIGUOUS_PROGRAM", "prediction": None}] + assert len(classes) > 1 + assert final == [ + { + **final[0], + "status": "AMBIGUOUS_PROGRAM", + } + ] + assert set(final[0]["candidate_ids"]) == { + candidate_id + for item in classes + for candidate_id in item["candidate_ids"] + } + + +def test_grammar_exhaustion_trace_has_no_fitting_candidate() -> None: + value = task( + [{"input": [[1, 2], [3, 4]], "output": [[9, 9, 9], [9, 9, 9], [9, 9, 9]]}], + [[[5]]], + "exhausted", + ) + + result = grid_induction.run_public_task_v2_2(value, None, mechanistic=True) + trace = result["mechanistic_trace"] + + assert result["predictions"] == [{"status": "GRAMMAR_EXHAUSTED", "prediction": None}] + assert not [ + item + for item in events(trace, "candidate_fit_decision") + if item["decision"] == "FITS_ALL_DEMOS" + ] + assert events(trace, "final_decision")[0]["candidate_ids"] == [] + + +def test_non_grid_mechanistic_mode_marks_frozen_delegate_as_opaque(monkeypatch: Any) -> None: + expected = {"task_id": "text", "predictions": [{"status": "ACCEPTED", "prediction": "B"}]} + + def fake_delegate(value: object, engine: object) -> dict[str, object]: + return expected + + monkeypatch.setattr(grid_induction, "run_public_task_v2", fake_delegate) + value = task([{"input": "A", "output": "B"}], ["A"], "text") + + result = grid_induction.run_public_task_v2_2(value, object(), mechanistic=True) + trace = result["mechanistic_trace"] + boundary = events(trace, "delegate_boundary") + + assert without_trace(result) == expected + assert boundary == [{ + **boundary[0], + "visibility": "OPAQUE_FROZEN_CORE", + "frozen_core_commit": "1d7c489bcdbff755d638b35ad47ce62bd4fe829d", + }] + assert events(trace, "final_decision")[0]["status"] == "DELEGATED" + validate_trace(trace) diff --git a/research/universal_core/holonomy_v2_2/tests/test_line_connect.py b/research/universal_core/holonomy_v2_2/tests/test_line_connect.py new file mode 100644 index 00000000..fd0bf7e3 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_line_connect.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from research.universal_core.holonomy_v2_2 import grid_induction +from research.universal_core.holonomy_v2_2.line_connect import ( + apply_line_connect, + line_connect_programs, +) + + +_FIXTURES = Path(__file__).parent / "fixtures" / "line_connect" +_TASK_IDS = ("1f876c06", "22168020", "22eb0ac0", "ded97339") + + +def load_payload(task_id: str) -> dict[str, Any]: + return json.loads((_FIXTURES / f"{task_id}.json").read_text(encoding="utf-8")) + + +def public_task(task_id: str) -> dict[str, Any]: + payload = load_payload(task_id) + return { + "task_id": task_id, + "instructions": "Infer the exact grid mapping.", + "demonstrations": payload["train"], + "hidden_inputs": [item["input"] for item in payload["test"]], + } + + +def expected_predictions(task_id: str) -> list[dict[str, Any]]: + return [ + {"status": "ACCEPTED", "prediction": item["output"]} + for item in load_payload(task_id)["test"] + ] + + +def test_horizontal_connection_fills_between_same_color_anchors() -> None: + assert apply_line_connect( + {"kind": "same_color_line_connect", "background": 0, "mode": "horizontal"}, + [[1, 0, 0, 1], [0, 0, 0, 0]], + ) == [[1, 1, 1, 1], [0, 0, 0, 0]] + + +def test_vertical_connection_fills_between_same_color_anchors() -> None: + assert apply_line_connect( + {"kind": "same_color_line_connect", "background": 0, "mode": "vertical"}, + [[2, 0], [0, 0], [2, 0]], + ) == [[2, 0], [2, 0], [2, 0]] + + +def test_diagonal_connection_fills_both_diagonal_families() -> None: + source = [ + [3, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 3, 0, 4], + [0, 0, 0, 0, 0], + [0, 0, 4, 0, 0], + ] + + assert apply_line_connect( + {"kind": "same_color_line_connect", "background": 0, "mode": "diagonal"}, + source, + ) == [ + [3, 0, 0, 0, 0], + [0, 3, 0, 0, 0], + [0, 0, 3, 0, 4], + [0, 0, 0, 4, 0], + [0, 0, 4, 0, 0], + ] + + +def test_newly_filled_cells_do_not_become_anchors() -> None: + source = [ + [5, 0, 5], + [0, 0, 0], + [0, 5, 0], + ] + + assert apply_line_connect( + {"kind": "same_color_line_connect", "background": 0, "mode": "orthogonal"}, + source, + ) == [ + [5, 5, 5], + [0, 0, 0], + [0, 5, 0], + ] + + +def test_connection_rejects_cross_color_overwrite() -> None: + with pytest.raises(ValueError, match="line connection color conflict"): + apply_line_connect( + {"kind": "same_color_line_connect", "background": 0, "mode": "horizontal"}, + [[1, 2, 1]], + ) + + +def test_program_derivation_returns_unique_horizontal_candidate() -> None: + source = [ + [1, 0, 0, 1], + [2, 0, 0, 0], + [0, 0, 0, 0], + [2, 0, 0, 0], + ] + target = [ + [1, 1, 1, 1], + [2, 0, 0, 0], + [0, 0, 0, 0], + [2, 0, 0, 0], + ] + + programs, reason = line_connect_programs(((source, target),)) + + assert reason is None + assert programs == ({ + "kind": "same_color_line_connect", + "background": 0, + "mode": "horizontal", + },) + + +def test_program_derivation_rejects_shape_change() -> None: + programs, reason = line_connect_programs(( + ([[1, 0, 1]], [[1, 1]]), + )) + + assert programs == () + assert reason == "NO_SHAPE_PRESERVING_LINE_CONNECT" + + +def test_program_derivation_skips_noop_demonstration() -> None: + programs, reason = line_connect_programs(( + ([[1, 1, 1]], [[1, 1, 1]]), + )) + + assert programs == () + assert reason == "NO_LINE_CONNECTION" + + +def test_four_training_gaps_become_exact_trace_explained_acceptances() -> None: + for task_id in _TASK_IDS: + result = grid_induction.run_public_task_v2_2( + public_task(task_id), + engine=None, + mechanistic=True, + ) + + assert result["predictions"] == expected_predictions(task_id) + trace = result["mechanistic_trace"] + final = [item for item in trace["events"] if item["event"] == "final_decision"][0] + successes = [ + item + for item in trace["events"] + if item.get("event") == "hidden_execution" + and item.get("candidate_kind") == "same_color_line_connect" + and item.get("outcome") == "SUCCEEDED" + ] + assert trace["terminal_status"] == "ACCEPTED" + assert final["selected_candidate_id"].startswith("line_connect:") + assert successes diff --git a/research/universal_core/holonomy_v2_2/tests/test_line_connect_fixtures.py b/research/universal_core/holonomy_v2_2/tests/test_line_connect_fixtures.py new file mode 100644 index 00000000..a36aec67 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_line_connect_fixtures.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +import hashlib +from pathlib import Path + + +_FIXTURES = Path(__file__).parent / "fixtures" / "line_connect" +_EXPECTED_SHA256 = { + "1f876c06.json": "b53a1a25685895400323e813a373f0a66fc9512f7dbbb326d52a93695fca8143", + "22168020.json": "d067b4f83f84120300310199e3a95c65258edcc53d8b0f425288d7ddb71a962e", + "22eb0ac0.json": "cf37cc34e7b0aa123c2806eca77a7d9916b50afea4e6e2db71d5149fe1f1af0d", + "ded97339.json": "fd14b4ee37bb1539c26a79abed743626b37f5935794ce82cbd0b3332c207906e", +} + + +def test_line_connect_fixtures_match_pinned_training_payloads() -> None: + actual = { + path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in sorted(_FIXTURES.glob("*.json")) + } + + assert actual == _EXPECTED_SHA256 diff --git a/research/universal_core/holonomy_v2_2/tests/test_marker_recolor.py b/research/universal_core/holonomy_v2_2/tests/test_marker_recolor.py new file mode 100644 index 00000000..072d86f0 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_marker_recolor.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +from typing import Any + +import pytest + +from research.universal_core.holonomy_v2_2 import grid_induction +from research.universal_core.holonomy_v2_2.marker_recolor import ( + apply_marker_recolor, + marker_recolor_programs, +) + + +def test_marker_programs_infer_dynamic_marker_recoloring() -> None: + demos = ( + ( + [[0, 3, 3], [0, 3, 0], [6, 0, 0]], + [[0, 6, 6], [0, 6, 0], [0, 0, 0]], + ), + ( + [[0, 2, 2], [0, 0, 2], [4, 0, 0]], + [[0, 4, 4], [0, 0, 4], [0, 0, 0]], + ), + ) + + programs, reason = marker_recolor_programs(demos) + + assert reason is None + assert programs == ( + {"kind": "marker_recolor", "background": 0, "connectivity": 4}, + {"kind": "marker_recolor", "background": 0, "connectivity": 8}, + ) + hidden = [[0, 8, 8], [0, 8, 0], [2, 0, 0]] + expected = [[0, 2, 2], [0, 2, 0], [0, 0, 0]] + assert {tuple(map(tuple, apply_marker_recolor(program, hidden))) for program in programs} == { + tuple(map(tuple, expected)) + } + + +def test_marker_apply_rejects_multiple_singleton_components() -> None: + program = {"kind": "marker_recolor", "background": 0, "connectivity": 4} + + with pytest.raises(ValueError, match="exactly one singleton marker"): + apply_marker_recolor(program, [[6, 0, 7], [0, 3, 3]]) + + +def test_marker_programs_report_missing_common_background() -> None: + programs, reason = marker_recolor_programs(( + ([[0, 1]], [[0, 1]]), + ([[2, 3]], [[2, 3]]), + )) + + assert programs == () + assert reason == "NO_BACKGROUND_CANDIDATE" + + +def aabf363d_task() -> dict[str, Any]: + return { + "task_id": "aabf363d", + "instructions": "Infer the exact grid mapping.", + "demonstrations": [ + { + "input": [ + [0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 3, 0, 0, 0], + [0, 0, 3, 3, 3, 0, 0], + [0, 3, 3, 3, 3, 0, 0], + [0, 3, 3, 0, 0, 0, 0], + [0, 0, 3, 3, 0, 0, 0], + [6, 0, 0, 0, 0, 0, 0], + ], + "output": [ + [0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 6, 0, 0, 0], + [0, 0, 6, 6, 6, 0, 0], + [0, 6, 6, 6, 6, 0, 0], + [0, 6, 6, 0, 0, 0, 0], + [0, 0, 6, 6, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0], + ], + }, + { + "input": [ + [0, 0, 0, 0, 0, 0, 0], + [0, 2, 2, 2, 0, 0, 0], + [0, 0, 2, 0, 0, 0, 0], + [0, 2, 2, 2, 2, 0, 0], + [0, 0, 2, 2, 2, 0, 0], + [0, 0, 0, 2, 0, 0, 0], + [4, 0, 0, 0, 0, 0, 0], + ], + "output": [ + [0, 0, 0, 0, 0, 0, 0], + [0, 4, 4, 4, 0, 0, 0], + [0, 0, 4, 0, 0, 0, 0], + [0, 4, 4, 4, 4, 0, 0], + [0, 0, 4, 4, 4, 0, 0], + [0, 0, 0, 4, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0], + ], + }, + ], + "hidden_inputs": [[ + [0, 0, 0, 0, 0, 0, 0], + [0, 8, 8, 8, 0, 0, 0], + [0, 8, 8, 8, 8, 8, 0], + [0, 0, 0, 8, 8, 0, 0], + [0, 0, 8, 8, 0, 0, 0], + [0, 0, 8, 8, 8, 0, 0], + [2, 0, 0, 0, 0, 0, 0], + ]], + } + + +def test_exact_execution_failure_becomes_trace_explained_acceptance() -> None: + result = grid_induction.run_public_task_v2_2( + aabf363d_task(), + engine=None, + mechanistic=True, + ) + expected = [ + [0, 0, 0, 0, 0, 0, 0], + [0, 2, 2, 2, 0, 0, 0], + [0, 2, 2, 2, 2, 2, 0], + [0, 0, 0, 2, 2, 0, 0], + [0, 0, 2, 2, 0, 0, 0], + [0, 0, 2, 2, 2, 0, 0], + [0, 0, 0, 0, 0, 0, 0], + ] + + assert result["predictions"] == [{"status": "ACCEPTED", "prediction": expected}] + events = result["mechanistic_trace"]["events"] + assert any( + event.get("event") == "hidden_execution" + and event.get("candidate_kind") == "color_map" + and event.get("outcome") == "FAILED" + and event.get("exception_type") == "KeyError" + and "unmapped grid cell" in event.get("exception_message", "") + for event in events + ) + marker_successes = [ + event + for event in events + if event.get("event") == "hidden_execution" + and event.get("candidate_kind") == "marker_recolor" + and event.get("outcome") == "SUCCEEDED" + ] + assert len(marker_successes) == 2 + final = [event for event in events if event.get("event") == "final_decision"] + assert final[0]["status"] == "ACCEPTED" + assert final[0]["selected_candidate_id"].startswith("marker_recolor:") diff --git a/research/universal_core/holonomy_v2_2/tests/test_mechanistic_trace.py b/research/universal_core/holonomy_v2_2/tests/test_mechanistic_trace.py new file mode 100644 index 00000000..b49d44fe --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_mechanistic_trace.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from research.universal_core.holonomy_v2_2.mechanistic_trace import ( + TraceRecorder, + validate_trace, + write_trace, +) + + +def complete_trace(task_id: str = "alpha") -> dict[str, object]: + recorder = TraceRecorder(task_id) + program_ref = recorder.store({"kind": "identity"}) + grid_ref = recorder.store([[1]]) + batch_ref = recorder.store([[[1]]]) + recorder.emit( + "candidate_proposed", + candidate_id="fixed:0000", + generator="fixed", + program_digest=program_ref, + program_ref=program_ref, + ) + recorder.emit( + "demo_execution", + candidate_id="fixed:0000", + demo_index=0, + outcome="MATCH", + output_ref=grid_ref, + target_ref=grid_ref, + ) + recorder.emit( + "candidate_fit_decision", + candidate_id="fixed:0000", + decision="FITS_ALL_DEMOS", + ) + recorder.emit( + "hidden_execution", + candidate_id="fixed:0000", + hidden_index=0, + outcome="SUCCEEDED", + output_ref=grid_ref, + ) + recorder.emit( + "hidden_batch_decision", + candidate_id="fixed:0000", + decision="COMPLETED", + batch_ref=batch_ref, + ) + recorder.emit( + "equivalence_class", + batch_ref=batch_ref, + candidate_ids=["fixed:0000"], + ) + recorder.emit( + "final_decision", + status="ACCEPTED", + candidate_ids=["fixed:0000"], + ) + return recorder.seal("ACCEPTED") + + +def test_trace_is_deterministic_and_task_id_independent() -> None: + first = complete_trace("alpha") + second = complete_trace("beta") + + assert first["trace_digest"] == second["trace_digest"] + assert first["envelope_digest"] != second["envelope_digest"] + assert json.dumps(first, sort_keys=True) == json.dumps(complete_trace("alpha"), sort_keys=True) + + +def test_validator_rejects_missing_candidate_terminal_event() -> None: + recorder = TraceRecorder("bad") + program_ref = recorder.store({"kind": "identity"}) + recorder.emit( + "candidate_proposed", + candidate_id="fixed:0000", + generator="fixed", + program_digest=program_ref, + program_ref=program_ref, + ) + recorder.emit("final_decision", status="GRAMMAR_EXHAUSTED", candidate_ids=[]) + + with pytest.raises(ValueError, match="terminal demonstration decision"): + recorder.seal("GRAMMAR_EXHAUSTED") + + +def test_validator_rejects_missing_hidden_terminal_event() -> None: + recorder = TraceRecorder("bad-hidden") + program_ref = recorder.store({"kind": "identity"}) + recorder.emit( + "candidate_proposed", + candidate_id="fixed:0000", + generator="fixed", + program_digest=program_ref, + program_ref=program_ref, + ) + recorder.emit( + "candidate_fit_decision", + candidate_id="fixed:0000", + decision="FITS_ALL_DEMOS", + ) + recorder.emit("final_decision", status="EXECUTION_FAILED", candidate_ids=["fixed:0000"]) + + with pytest.raises(ValueError, match="terminal hidden-batch decision"): + recorder.seal("EXECUTION_FAILED") + + +def test_validator_rejects_unknown_object_reference() -> None: + trace = complete_trace() + trace["events"][1]["output_ref"] = "0" * 64 + + with pytest.raises(ValueError, match="unknown object reference"): + validate_trace(trace) + + +def test_store_rejects_non_json_values() -> None: + recorder = TraceRecorder("unsafe") + + with pytest.raises(TypeError, match="JSON-safe"): + recorder.store({"bad": object()}) + + +def test_write_trace_emits_canonical_json(tmp_path: Path) -> None: + trace = complete_trace() + path = write_trace(trace, tmp_path / "trace.json") + + assert path.read_text(encoding="utf-8").endswith("\n") + validate_trace(json.loads(path.read_text(encoding="utf-8"))) diff --git a/research/universal_core/holonomy_v2_2/tests/test_mirror_concat.py b/research/universal_core/holonomy_v2_2/tests/test_mirror_concat.py new file mode 100644 index 00000000..7e70d5f6 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_mirror_concat.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from research.universal_core.holonomy_v2_2 import grid_induction +from research.universal_core.holonomy_v2_2.mirror_concat import ( + apply_mirror_concat, + mirror_concat_programs, +) + + +_FIXTURES = Path(__file__).parent / "fixtures" / "mirror_concat" +_TASK_IDS = ("4c4377d9", "6d0aefbc", "6fa7a44f", "8be77c9e", "c9e6f938") + + +def load_payload(task_id: str) -> dict[str, Any]: + return json.loads((_FIXTURES / f"{task_id}.json").read_text(encoding="utf-8")) + + +def public_task(task_id: str) -> dict[str, Any]: + payload = load_payload(task_id) + return { + "task_id": task_id, + "instructions": "Infer the exact grid mapping.", + "demonstrations": payload["train"], + "hidden_inputs": [item["input"] for item in payload["test"]], + } + + +def expected_predictions(task_id: str) -> list[dict[str, Any]]: + payload = load_payload(task_id) + return [ + {"status": "ACCEPTED", "prediction": item["output"]} + for item in payload["test"] + ] + + +def test_row_source_first_appends_vertical_reflection() -> None: + program = { + "kind": "mirror_concatenate", + "axis": "rows", + "order": "source_first", + } + + assert apply_mirror_concat(program, [[1, 2], [3, 4]]) == [ + [1, 2], + [3, 4], + [3, 4], + [1, 2], + ] + + +def test_row_reflection_first_prepends_vertical_reflection() -> None: + program = { + "kind": "mirror_concatenate", + "axis": "rows", + "order": "reflection_first", + } + + assert apply_mirror_concat(program, [[1, 2], [3, 4]]) == [ + [3, 4], + [1, 2], + [1, 2], + [3, 4], + ] + + +def test_column_orders_append_or_prepend_horizontal_reflection() -> None: + source = [[1, 2], [3, 4]] + + assert apply_mirror_concat({ + "kind": "mirror_concatenate", + "axis": "columns", + "order": "source_first", + }, source) == [ + [1, 2, 2, 1], + [3, 4, 4, 3], + ] + assert apply_mirror_concat({ + "kind": "mirror_concatenate", + "axis": "columns", + "order": "reflection_first", + }, source) == [ + [2, 1, 1, 2], + [4, 3, 3, 4], + ] + + +def test_program_enumeration_exposes_all_four_structural_hypotheses() -> None: + programs, reason = mirror_concat_programs((([[1]], [[1], [1]]),)) + + assert reason is None + assert programs == ( + {"kind": "mirror_concatenate", "axis": "rows", "order": "source_first"}, + {"kind": "mirror_concatenate", "axis": "rows", "order": "reflection_first"}, + {"kind": "mirror_concatenate", "axis": "columns", "order": "source_first"}, + {"kind": "mirror_concatenate", "axis": "columns", "order": "reflection_first"}, + ) + + +def test_five_training_gaps_become_exact_trace_explained_acceptances() -> None: + for task_id in _TASK_IDS: + result = grid_induction.run_public_task_v2_2( + public_task(task_id), + engine=None, + mechanistic=True, + ) + + assert result["predictions"] == expected_predictions(task_id) + trace = result["mechanistic_trace"] + final = [item for item in trace["events"] if item["event"] == "final_decision"] + successes = [ + item + for item in trace["events"] + if item.get("event") == "hidden_execution" + and item.get("candidate_kind") == "mirror_concatenate" + and item.get("outcome") == "SUCCEEDED" + ] + assert trace["terminal_status"] == "ACCEPTED" + assert final[0]["status"] == "ACCEPTED" + assert final[0]["selected_candidate_id"].startswith("mirror_concat:") + assert successes diff --git a/research/universal_core/holonomy_v2_2/tests/test_palette_repeat.py b/research/universal_core/holonomy_v2_2/tests/test_palette_repeat.py new file mode 100644 index 00000000..0af5790f --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_palette_repeat.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from research.universal_core.holonomy_v2_2 import grid_induction +from research.universal_core.holonomy_v2_2.palette_repeat import ( + apply_palette_repeat, + palette_repeat_programs, +) + + +_FIXTURES = Path(__file__).parent / "fixtures" / "palette_repeat" +_TASK_IDS = ("a59b95c0", "ac0a08a4", "b91ae062", "d4b1c2b1") + + +def load_payload(task_id: str) -> dict[str, Any]: + return json.loads((_FIXTURES / f"{task_id}.json").read_text(encoding="utf-8")) + + +def public_task(task_id: str) -> dict[str, Any]: + payload = load_payload(task_id) + return { + "task_id": task_id, + "instructions": "Infer the exact grid mapping.", + "demonstrations": payload["train"], + "hidden_inputs": [item["input"] for item in payload["test"]], + } + + +def expected_predictions(task_id: str) -> list[dict[str, Any]]: + return [ + {"status": "ACCEPTED", "prediction": item["output"]} + for item in load_payload(task_id)["test"] + ] + + +def test_apply_scales_by_total_palette_size() -> None: + program = { + "kind": "palette_cardinality_repeat", + "operation": "scale", + "statistic": "distinct_all", + } + + assert apply_palette_repeat(program, [[1, 2], [2, 1]]) == [ + [1, 1, 2, 2], + [1, 1, 2, 2], + [2, 2, 1, 1], + [2, 2, 1, 1], + ] + + +def test_apply_tiles_by_total_palette_size() -> None: + program = { + "kind": "palette_cardinality_repeat", + "operation": "tile", + "statistic": "distinct_all", + } + + assert apply_palette_repeat(program, [[1, 2]]) == [ + [1, 2, 1, 2], + [1, 2, 1, 2], + ] + + +def test_apply_scales_by_non_background_palette_size() -> None: + program = { + "kind": "palette_cardinality_repeat", + "operation": "scale", + "statistic": "distinct_non_background", + "background": 0, + } + + assert apply_palette_repeat(program, [[0, 1, 2]]) == [ + [0, 0, 1, 1, 2, 2], + [0, 0, 1, 1, 2, 2], + ] + + +def test_program_derivation_requires_demonstrated_factor_variation() -> None: + demos = ( + ( + [[1, 2]], + [[1, 1, 2, 2], [1, 1, 2, 2]], + ), + ( + [[3, 4, 5]], + [ + [3, 3, 3, 4, 4, 4, 5, 5, 5], + [3, 3, 3, 4, 4, 4, 5, 5, 5], + [3, 3, 3, 4, 4, 4, 5, 5, 5], + ], + ), + ) + + programs, reason = palette_repeat_programs(demos) + + assert reason is None + assert programs == ({ + "kind": "palette_cardinality_repeat", + "operation": "scale", + "statistic": "distinct_all", + },) + + +def test_program_derivation_defers_constant_factor_to_fixed_grammar() -> None: + demos = ( + ( + [[1, 2]], + [[1, 1, 2, 2], [1, 1, 2, 2]], + ), + ( + [[3, 4]], + [[3, 3, 4, 4], [3, 3, 4, 4]], + ), + ) + + programs, reason = palette_repeat_programs(demos) + + assert programs == () + assert reason == "NO_FACTOR_VARIATION" + + +def test_apply_rejects_output_beyond_arc_bounds() -> None: + program = { + "kind": "palette_cardinality_repeat", + "operation": "scale", + "statistic": "distinct_all", + } + source = [[value for value in range(6)] for _ in range(6)] + + with pytest.raises(ValueError, match="exceeds ARC bounds"): + apply_palette_repeat(program, source) + + +def test_four_training_gaps_become_exact_trace_explained_acceptances() -> None: + for task_id in _TASK_IDS: + result = grid_induction.run_public_task_v2_2( + public_task(task_id), + engine=None, + mechanistic=True, + ) + + assert result["predictions"] == expected_predictions(task_id) + trace = result["mechanistic_trace"] + final = [item for item in trace["events"] if item["event"] == "final_decision"][0] + successes = [ + item + for item in trace["events"] + if item.get("event") == "hidden_execution" + and item.get("candidate_kind") == "palette_cardinality_repeat" + and item.get("outcome") == "SUCCEEDED" + ] + assert trace["terminal_status"] == "ACCEPTED" + assert final["selected_candidate_id"].startswith("palette_repeat:") + assert successes diff --git a/research/universal_core/holonomy_v2_2/tests/test_palette_repeat_fixtures.py b/research/universal_core/holonomy_v2_2/tests/test_palette_repeat_fixtures.py new file mode 100644 index 00000000..64ef340e --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_palette_repeat_fixtures.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +import hashlib +from pathlib import Path + + +_FIXTURES = Path(__file__).parent / "fixtures" / "palette_repeat" +_EXPECTED_SHA256 = { + "a59b95c0.json": "138f5638ba814834e4f222d6509665ea091c36be6bfc9305fdf41fecd72c41a1", + "ac0a08a4.json": "5e68feb78f2f5405be84063528e4beab2b66b8b8cbdac211e305d75e8ac8703d", + "b91ae062.json": "dfb1b3e1b6d275cb1830c1d42320358a1ddc45a925504904ff07fa0dac8a2a53", + "d4b1c2b1.json": "46b0f9fec811ac831b916e44ef8269e45e2b054d59354a4a8f8a1a299716d753", +} + + +def test_palette_repeat_fixtures_match_pinned_training_payloads() -> None: + actual = { + path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in sorted(_FIXTURES.glob("*.json")) + } + + assert actual == _EXPECTED_SHA256 diff --git a/research/universal_core/holonomy_v2_2/tests/test_panel_combine.py b/research/universal_core/holonomy_v2_2/tests/test_panel_combine.py new file mode 100644 index 00000000..b7b920f7 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_panel_combine.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +from research.universal_core.holonomy_v2_2.grid_induction import run_public_task_v2_2 + + +def make_task(demos: list[dict[str, object]], hidden: list[object]) -> dict[str, object]: + return { + "task_id": "panel-combine", + "instructions": "Infer the exact grid mapping.", + "demonstrations": demos, + "hidden_inputs": hidden, + } + + +def test_vertical_panels_with_separator_use_intersection() -> None: + value = make_task( + [{ + "input": [[1, 0, 9, 2, 0], [1, 1, 9, 0, 2]], + "output": [[4, 0], [0, 4]], + }], + [[[3, 3, 9, 5, 0], [0, 3, 9, 0, 5]]], + ) + + result = run_public_task_v2_2(value, engine=None) + + assert result["predictions"] == [{ + "status": "ACCEPTED", + "prediction": [[4, 0], [0, 4]], + }] + assert "panel_boolean_combine" in result["evidence"]["candidate_kinds"] + + +def test_horizontal_panels_without_gap_use_exclusive_or() -> None: + value = make_task( + [{ + "input": [[1, 0], [0, 1], [0, 2], [2, 1]], + "output": [[6, 6], [6, 0]], + }], + [[[3, 0], [3, 3], [0, 4], [4, 0]]], + ) + + result = run_public_task_v2_2(value, engine=None) + + assert result["predictions"] == [{ + "status": "ACCEPTED", + "prediction": [[6, 6], [0, 6]], + }] + + +def test_vertical_panels_can_select_left_only_cells() -> None: + value = make_task( + [{ + "input": [[1, 1, 0, 2], [0, 1, 2, 0]], + "output": [[7, 0], [0, 7]], + }], + [[[3, 0, 0, 4], [3, 3, 4, 0]]], + ) + + result = run_public_task_v2_2(value, engine=None) + + assert result["predictions"] == [{ + "status": "ACCEPTED", + "prediction": [[7, 0], [0, 7]], + }] + + +def test_disagreeing_panel_rules_abstain() -> None: + value = make_task( + [{ + "input": [[1, 0, 0, 9, 0, 2, 0]], + "output": [[5, 5, 0]], + }], + [[[1, 0, 0, 9, 2, 0, 0]]], + ) + + result = run_public_task_v2_2(value, engine=None) + + assert result["predictions"] == [{"status": "AMBIGUOUS_PROGRAM", "prediction": None}] diff --git a/research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py b/research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py new file mode 100644 index 00000000..1ee7d349 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from research.universal_core.holonomy_v2_2 import grid_induction +from research.universal_core.holonomy_v2_2.quadrant_mosaic import ( + apply_quadrant_mosaic, + quadrant_mosaic_programs, +) + + +_FIXTURES = Path(__file__).parent / "fixtures" / "quadrant_mosaic" +_TASK_IDS = ( + "0c786b71", + "3af2c5a8", + "46442a0e", + "62c24649", + "67e8384a", + "7953d61e", + "7fe24cdd", + "833dafe3", + "ed98d772", +) + + +def load_payload(task_id: str) -> dict[str, Any]: + return json.loads((_FIXTURES / f"{task_id}.json").read_text(encoding="utf-8")) + + +def public_task(task_id: str) -> dict[str, Any]: + payload = load_payload(task_id) + return { + "task_id": task_id, + "instructions": "Infer the exact grid mapping.", + "demonstrations": payload["train"], + "hidden_inputs": [item["input"] for item in payload["test"]], + } + + +def expected_predictions(task_id: str) -> list[dict[str, Any]]: + payload = load_payload(task_id) + return [ + {"status": "ACCEPTED", "prediction": item["output"]} + for item in payload["test"] + ] + + +def test_apply_standard_reflected_quadrant_mosaic() -> None: + program = { + "kind": "dihedral_quadrant_mosaic", + "top_left": "identity", + "top_right": "reflect_columns", + "bottom_left": "reflect_rows", + "bottom_right": "rotate_180", + } + + assert apply_quadrant_mosaic(program, [[1, 2], [3, 4]]) == [ + [1, 2, 2, 1], + [3, 4, 4, 3], + [3, 4, 4, 3], + [1, 2, 2, 1], + ] + + +def test_apply_rejects_output_beyond_arc_bounds() -> None: + source = [[0] * 16 for _ in range(16)] + program = { + "kind": "dihedral_quadrant_mosaic", + "top_left": "identity", + "top_right": "identity", + "bottom_left": "identity", + "bottom_right": "identity", + } + + with pytest.raises(ValueError, match="exceeds ARC bounds"): + apply_quadrant_mosaic(program, source) + + +def test_program_derivation_returns_one_canonical_program() -> None: + source = [[1, 2], [3, 4]] + target = [ + [1, 2, 2, 1], + [3, 4, 4, 3], + [3, 4, 4, 3], + [1, 2, 2, 1], + ] + + programs, reason = quadrant_mosaic_programs(((source, target),)) + + assert reason is None + assert programs == ({ + "kind": "dihedral_quadrant_mosaic", + "top_left": "identity", + "top_right": "reflect_columns", + "bottom_left": "reflect_rows", + "bottom_right": "rotate_180", + },) + + +def test_program_derivation_reports_non_double_output_shape() -> None: + programs, reason = quadrant_mosaic_programs(( + ([[1, 2], [3, 4]], [[1, 2], [3, 4]]), + )) + + assert programs == () + assert reason == "OUTPUT_NOT_DOUBLE_SOURCE_SHAPE" + + +def test_program_derivation_defers_uniform_quadrant_roles_to_tile() -> None: + programs, reason = quadrant_mosaic_programs(( + ( + [[1, 2]], + [ + [1, 2, 1, 2], + [1, 2, 1, 2], + ], + ), + )) + + assert programs == () + assert reason == "NO_DISTINCT_QUADRANT_ROLES" + + +def test_nine_training_gaps_become_exact_trace_explained_acceptances() -> None: + for task_id in _TASK_IDS: + result = grid_induction.run_public_task_v2_2( + public_task(task_id), + engine=None, + mechanistic=True, + ) + + assert result["predictions"] == expected_predictions(task_id) + trace = result["mechanistic_trace"] + final = [ + item for item in trace["events"] + if item["event"] == "final_decision" + ] + successes = [ + item + for item in trace["events"] + if item.get("event") == "hidden_execution" + and item.get("candidate_kind") == "dihedral_quadrant_mosaic" + and item.get("outcome") == "SUCCEEDED" + ] + assert trace["terminal_status"] == "ACCEPTED" + assert final[0]["status"] == "ACCEPTED" + assert final[0]["selected_candidate_id"].startswith("quadrant_mosaic:") + assert successes + + +def test_symmetric_demonstration_preserves_hidden_transform_ambiguity() -> None: + task = { + "task_id": "quadrant-mosaic-ambiguity", + "instructions": "Infer the exact grid mapping.", + "demonstrations": [{ + "input": [[1, 1], [2, 2]], + "output": [ + [1, 1, 1, 1], + [2, 2, 2, 2], + [2, 2, 2, 2], + [1, 1, 1, 1], + ], + }], + "hidden_inputs": [[[1, 2], [3, 4]]], + } + + result = grid_induction.run_public_task_v2_2( + task, + engine=None, + mechanistic=True, + ) + + assert result["predictions"] == [ + {"status": "AMBIGUOUS_PROGRAM", "prediction": None} + ] + classes = [ + item + for item in result["mechanistic_trace"]["events"] + if item["event"] == "equivalence_class" + and any( + str(candidate_id).startswith("quadrant_mosaic:") + for candidate_id in item["candidate_ids"] + ) + ] + assert len(classes) >= 2 diff --git a/research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic_fixtures.py b/research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic_fixtures.py new file mode 100644 index 00000000..786f7c33 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_quadrant_mosaic_fixtures.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +import hashlib +from pathlib import Path + + +_FIXTURES = Path(__file__).parent / "fixtures" / "quadrant_mosaic" +_EXPECTED_SHA256 = { + "0c786b71.json": "df8a0f7356a4b200d7c9a4d8f58d40662f01563ebb25c85f67c718bdb487a3f2", + "3af2c5a8.json": "969747e767a1af9f23689cc7e262abaadccf1e02ff8c2a446f738daaa4a24a48", + "46442a0e.json": "cd58b40fb6c991567f666e2d42a98a8a5ba9de6708011d3ffb36245487aba5ba", + "62c24649.json": "bd4124704f8b1b02038f80c7cb1187224d1daf4dd528cea940486bb294b48dc4", + "67e8384a.json": "3ad82285fa52a72081cfb705f80a224cf4f984232a1e002421d12b38b83a012d", + "7953d61e.json": "789c244eae24dc51956ed3beea9b530b9af8d895692b384bd6fa3eb04e3e6f12", + "7fe24cdd.json": "6deb4aa52a4cc3d63cf8b8b99846e50297837910a1b9c6d60f7e028cee8c68d7", + "833dafe3.json": "c3c9b31eb8def554991acfc0be966e49cb128568a5f948452be5b9bda1ce9dd2", + "ed98d772.json": "f7f2569ee2b94b6e4d98ff72e3f36ff83441a802f3bc7c19ea8c04c81108b18b", +} + + +def test_quadrant_mosaic_fixtures_match_pinned_training_payloads() -> None: + actual = { + path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in sorted(_FIXTURES.glob("*.json")) + } + + assert actual == _EXPECTED_SHA256 diff --git a/research/universal_core/holonomy_v2_2/tests/test_ray_extend.py b/research/universal_core/holonomy_v2_2/tests/test_ray_extend.py new file mode 100644 index 00000000..5c19d8b2 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_ray_extend.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from research.universal_core.holonomy_v2_2 import grid_induction +from research.universal_core.holonomy_v2_2.ray_extend import ( + apply_ray_extend, + ray_extend_programs, +) + + +_FIXTURES = Path(__file__).parent / "fixtures" / "ray_extend" +_TASK_IDS = ("623ea044", "d037b0a7") + + +def load_payload(task_id: str) -> dict[str, Any]: + return json.loads((_FIXTURES / f"{task_id}.json").read_text(encoding="utf-8")) + + +def public_task(task_id: str) -> dict[str, Any]: + payload = load_payload(task_id) + return { + "task_id": task_id, + "instructions": "Infer the exact grid mapping.", + "demonstrations": payload["train"], + "hidden_inputs": [item["input"] for item in payload["test"]], + } + + +def expected_predictions(task_id: str) -> list[dict[str, Any]]: + return [ + {"status": "ACCEPTED", "prediction": item["output"]} + for item in load_payload(task_id)["test"] + ] + + +def test_down_ray_extends_each_anchor_to_bottom_border() -> None: + assert apply_ray_extend( + {"kind": "anchor_ray_extend", "background": 0, "mode": "down"}, + [[1, 0], [0, 2], [0, 0]], + ) == [[1, 0], [1, 2], [1, 2]] + + +def test_diagonal_ray_extends_anchor_to_four_diagonal_borders() -> None: + assert apply_ray_extend( + {"kind": "anchor_ray_extend", "background": 0, "mode": "diagonal"}, + [[0, 0, 0], [0, 3, 0], [0, 0, 0]], + ) == [[3, 0, 3], [0, 3, 0], [3, 0, 3]] + + +def test_orthogonal_ray_extends_anchor_in_four_cardinal_directions() -> None: + assert apply_ray_extend( + {"kind": "anchor_ray_extend", "background": 0, "mode": "orthogonal"}, + [[0, 0, 0], [0, 4, 0], [0, 0, 0]], + ) == [[0, 4, 0], [4, 4, 4], [0, 4, 0]] + + +def test_ray_extension_preserves_original_anchor_cells() -> None: + source = [[5, 0, 0], [0, 0, 0]] + output = apply_ray_extend( + {"kind": "anchor_ray_extend", "background": 0, "mode": "right"}, + source, + ) + + assert source == [[5, 0, 0], [0, 0, 0]] + assert output == [[5, 5, 5], [0, 0, 0]] + + +def test_ray_extension_rejects_cross_color_conflict() -> None: + with pytest.raises(ValueError, match="ray extension color conflict"): + apply_ray_extend( + {"kind": "anchor_ray_extend", "background": 0, "mode": "right"}, + [[1, 0, 2]], + ) + + +def test_ray_extension_rejects_unknown_mode() -> None: + with pytest.raises(ValueError, match="unknown ray extension mode"): + apply_ray_extend( + {"kind": "anchor_ray_extend", "background": 0, "mode": "spiral"}, + [[1, 0], [0, 0]], + ) + + +def test_program_derivation_returns_unique_down_candidate() -> None: + source = [[1, 0], [0, 2], [0, 0]] + target = [[1, 0], [1, 2], [1, 2]] + + programs, reason = ray_extend_programs(((source, target),)) + + assert reason is None + assert programs == ({ + "kind": "anchor_ray_extend", + "background": 0, + "mode": "down", + },) + + +def test_program_derivation_rejects_shape_change() -> None: + programs, reason = ray_extend_programs(( + ([[1, 0], [0, 0]], [[1, 1]]), + )) + + assert programs == () + assert reason == "NO_SHAPE_PRESERVING_RAY_EXTENSION" + + +def test_program_derivation_skips_noop_demonstration() -> None: + programs, reason = ray_extend_programs(( + ([[1, 1], [1, 1]], [[1, 1], [1, 1]]), + )) + + assert programs == () + assert reason == "NO_RAY_EXTENSION" + + +def test_two_training_gaps_become_exact_trace_explained_acceptances() -> None: + for task_id in _TASK_IDS: + result = grid_induction.run_public_task_v2_2( + public_task(task_id), + engine=None, + mechanistic=True, + ) + + assert result["predictions"] == expected_predictions(task_id) + trace = result["mechanistic_trace"] + final = [item for item in trace["events"] if item["event"] == "final_decision"][0] + successes = [ + item + for item in trace["events"] + if item.get("event") == "hidden_execution" + and item.get("candidate_kind") == "anchor_ray_extend" + and item.get("outcome") == "SUCCEEDED" + ] + assert trace["terminal_status"] == "ACCEPTED" + assert final["selected_candidate_id"].startswith("ray_extend:") + assert successes diff --git a/research/universal_core/holonomy_v2_2/tests/test_ray_extend_fixtures.py b/research/universal_core/holonomy_v2_2/tests/test_ray_extend_fixtures.py new file mode 100644 index 00000000..8a695ca0 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_ray_extend_fixtures.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import hashlib +from pathlib import Path + + +_FIXTURES = Path(__file__).parent / "fixtures" / "ray_extend" +_EXPECTED_SHA256 = { + "623ea044.json": "578ab7e47a2d67489f1103715615f719ef81e9faf773be85586803d6da44cbdd", + "d037b0a7.json": "9e39a4acdb3e3bff7dd54bed5f4ab77973f4027d48beb62a9c740d8a7a0b0aee", +} + + +def test_ray_extend_fixtures_match_pinned_training_payloads() -> None: + actual = { + path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in sorted(_FIXTURES.glob("*.json")) + } + + assert actual == _EXPECTED_SHA256 diff --git a/research/universal_core/holonomy_v2_2/tests/test_region_fill.py b/research/universal_core/holonomy_v2_2/tests/test_region_fill.py new file mode 100644 index 00000000..971dbefe --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_region_fill.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +from research.universal_core.holonomy_v2_2.grid_induction import run_public_task_v2_2 + + +def make_task(demos: list[dict[str, object]], hidden: list[object]) -> dict[str, object]: + return { + "task_id": "region-fill", + "instructions": "Infer the exact grid mapping.", + "demonstrations": demos, + "hidden_inputs": hidden, + } + + +def ring(color: int) -> list[list[int]]: + return [ + [0, 0, 0, 0, 0], + [0, color, color, color, 0], + [0, color, 0, color, 0], + [0, color, color, color, 0], + [0, 0, 0, 0, 0], + ] + + +def test_enclosed_background_cells_are_filled_on_a_copy() -> None: + value = make_task( + [ + { + "input": ring(2), + "output": [ + [0, 0, 0, 0, 0], + [0, 2, 2, 2, 0], + [0, 2, 4, 2, 0], + [0, 2, 2, 2, 0], + [0, 0, 0, 0, 0], + ], + }, + { + "input": ring(8), + "output": [ + [0, 0, 0, 0, 0], + [0, 8, 8, 8, 0], + [0, 8, 4, 8, 0], + [0, 8, 8, 8, 0], + [0, 0, 0, 0, 0], + ], + }, + ], + [[[3, 3, 3], [3, 0, 3], [3, 3, 3]]], + ) + + result = run_public_task_v2_2(value, engine=None) + + assert result["predictions"] == [{ + "status": "ACCEPTED", + "prediction": [[3, 3, 3], [3, 4, 3], [3, 3, 3]], + }] + assert "enclosed_region_fill" in result["evidence"]["candidate_kinds"] + + +def test_only_enclosed_cells_can_be_highlighted() -> None: + blank = [[0, 0, 0, 0, 0] for _ in range(5)] + first = [row[:] for row in blank] + second = [row[:] for row in blank] + first[2][2] = 7 + second[2][2] = 7 + value = make_task( + [ + {"input": ring(2), "output": first}, + {"input": ring(8), "output": second}, + ], + [[[5, 5, 5], [5, 0, 5], [5, 5, 5]]], + ) + + result = run_public_task_v2_2(value, engine=None) + + assert result["predictions"] == [{ + "status": "ACCEPTED", + "prediction": [[0, 0, 0], [0, 7, 0], [0, 0, 0]], + }] + + +def test_closed_regions_can_be_solidified_to_one_color() -> None: + solid = [ + [0, 0, 0, 0, 0], + [0, 6, 6, 6, 0], + [0, 6, 6, 6, 0], + [0, 6, 6, 6, 0], + [0, 0, 0, 0, 0], + ] + value = make_task( + [ + {"input": ring(2), "output": solid}, + {"input": ring(8), "output": solid}, + ], + [ring(3)], + ) + + result = run_public_task_v2_2(value, engine=None) + + assert result["predictions"] == [{ + "status": "ACCEPTED", + "prediction": solid, + }] + + +def test_background_connected_to_border_is_not_enclosed() -> None: + value = make_task( + [{ + "input": [[2, 2, 2], [2, 0, 0], [2, 2, 2]], + "output": [[2, 2, 2], [2, 0, 0], [2, 2, 2]], + }], + [[[3, 3, 3], [3, 0, 3], [3, 3, 3]]], + ) + + result = run_public_task_v2_2(value, engine=None) + + assert result["predictions"] == [{"status": "AMBIGUOUS_PROGRAM", "prediction": None}] diff --git a/research/universal_core/holonomy_v2_2/tests/test_structural_enumeration.py b/research/universal_core/holonomy_v2_2/tests/test_structural_enumeration.py new file mode 100644 index 00000000..53d62142 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_structural_enumeration.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from research.universal_core.holonomy_v2_2 import ( + component_select, + panel_combine, + region_fill, +) + + +def test_panel_programs_expose_hypotheses_before_fit() -> None: + demos = (([[1, 0, 9, 2, 0]], [[4, 0]]),) + + programs, reason = panel_combine.panel_programs(demos) + + assert reason is None + assert len(programs) > len(panel_combine.panel_candidates(demos)) + assert any(item["predicate"] == "xor" for item in programs) + + +def test_panel_programs_report_target_color_skip() -> None: + programs, reason = panel_combine.panel_programs((([[1, 0]], [[7, 7]]),)) + + assert programs == () + assert reason == "TARGET_COLOR_CARDINALITY_NOT_TWO" + + +def test_region_programs_expose_hypotheses_before_fit() -> None: + source = [ + [0, 2, 2], + [2, 0, 2], + [2, 2, 2], + ] + demos = ((source, source),) + + programs, reason = region_fill.region_programs(demos) + + assert reason is None + assert len(programs) > len(region_fill.region_candidates(demos)) + assert {item["mode"] for item in programs} == {"fill_copy", "highlight", "solidify"} + + +def test_region_programs_report_missing_background() -> None: + demos = ( + ([[1, 1], [1, 2]], [[1, 1], [1, 2]]), + ([[2, 2], [2, 1]], [[2, 2], [2, 1]]), + ) + + programs, reason = region_fill.region_programs(demos) + + assert programs == () + assert reason == "NO_BACKGROUND_CANDIDATE" + + +def test_component_select_programs_expose_hypotheses_before_fit() -> None: + demos = (( + [ + [0, 2, 0, 3, 3, 0], + [0, 2, 0, 3, 3, 0], + [0, 0, 0, 3, 0, 0], + ], + [[3, 3], [3, 3], [3, 0]], + ),) + + programs, reason = component_select.component_select_programs(demos) + + assert reason is None + assert len(programs) > len(component_select.component_select_candidates(demos)) + assert {item["selector"] for item in programs} == {"minimum", "maximum"} + + +def test_component_select_programs_report_missing_background() -> None: + demos = ( + ([[1, 1], [1, 2]], [[1]]), + ([[2, 2], [2, 1]], [[2]]), + ) + + programs, reason = component_select.component_select_programs(demos) + + assert programs == () + assert reason == "NO_BACKGROUND_CANDIDATE" diff --git a/research/universal_core/holonomy_v2_2/tests/test_symmetry_complete.py b/research/universal_core/holonomy_v2_2/tests/test_symmetry_complete.py new file mode 100644 index 00000000..c6b953cf --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_symmetry_complete.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from research.universal_core.holonomy_v2_2 import grid_induction +from research.universal_core.holonomy_v2_2.symmetry_complete import ( + apply_symmetry_complete, + symmetry_complete_programs, +) + + +_FIXTURES = Path(__file__).parent / "fixtures" / "symmetry_complete" +_EXACT_TASK_IDS = ("496994bd", "e729b7be", "f25ffba3") + + +def load_payload(task_id: str) -> dict[str, Any]: + return json.loads((_FIXTURES / f"{task_id}.json").read_text(encoding="utf-8")) + + +def public_task(task_id: str) -> dict[str, Any]: + payload = load_payload(task_id) + return { + "task_id": task_id, + "instructions": "Infer the exact grid mapping.", + "demonstrations": payload["train"], + "hidden_inputs": [item["input"] for item in payload["test"]], + } + + +def expected_predictions(task_id: str) -> list[dict[str, Any]]: + return [ + {"status": "ACCEPTED", "prediction": item["output"]} + for item in load_payload(task_id)["test"] + ] + + +def test_vertical_completion_reflects_columns() -> None: + assert apply_symmetry_complete( + {"kind": "reflective_symmetry_complete", "background": 0, "transform": "vertical"}, + [[1, 0, 0], [0, 2, 0]], + ) == [[1, 0, 1], [0, 2, 0]] + + +def test_horizontal_completion_reflects_rows() -> None: + assert apply_symmetry_complete( + {"kind": "reflective_symmetry_complete", "background": 0, "transform": "horizontal"}, + [[1, 0, 0], [0, 2, 0]], + ) == [[1, 2, 0], [1, 2, 0]] + + +def test_rotate_180_completion_preserves_original_cells() -> None: + assert apply_symmetry_complete( + {"kind": "reflective_symmetry_complete", "background": 0, "transform": "rotate_180"}, + [[1, 0, 0], [0, 0, 0]], + ) == [[1, 0, 0], [0, 0, 1]] + + +def test_diagonal_completion_uses_original_anchors_only() -> None: + source = [[3, 0, 0], [0, 0, 4], [0, 0, 0]] + assert apply_symmetry_complete( + {"kind": "reflective_symmetry_complete", "background": 0, "transform": "main_diagonal"}, + source, + ) == [[3, 0, 0], [0, 0, 4], [0, 4, 0]] + + +def test_symmetry_completion_rejects_cross_color_overwrite() -> None: + with pytest.raises(ValueError, match="symmetry completion color conflict"): + apply_symmetry_complete( + {"kind": "reflective_symmetry_complete", "background": 0, "transform": "vertical"}, + [[1, 2]], + ) + + +def test_diagonal_completion_rejects_non_square_grid() -> None: + with pytest.raises(ValueError, match="diagonal symmetry requires a square grid"): + apply_symmetry_complete( + {"kind": "reflective_symmetry_complete", "background": 0, "transform": "anti_diagonal"}, + [[1, 0, 0], [0, 0, 0]], + ) + + +def test_program_derivation_returns_unique_horizontal_candidate() -> None: + source = [[1, 0, 0], [0, 2, 0]] + target = [[1, 2, 0], [1, 2, 0]] + + programs, reason = symmetry_complete_programs(((source, target),)) + + assert reason is None + assert programs == ({ + "kind": "reflective_symmetry_complete", + "background": 0, + "transform": "horizontal", + },) + + +def test_program_derivation_rejects_shape_change() -> None: + programs, reason = symmetry_complete_programs(( + ([[1, 0], [0, 0]], [[1, 1]]), + )) + + assert programs == () + assert reason == "NO_SHAPE_PRESERVING_SYMMETRY" + + +def test_program_derivation_skips_noop_demonstration() -> None: + programs, reason = symmetry_complete_programs(( + ([[1, 1], [1, 1]], [[1, 1], [1, 1]]), + )) + + assert programs == () + assert reason == "NO_REFLECTIVE_SYMMETRY_COMPLETION" + + +def test_three_training_gaps_become_exact_trace_explained_acceptances() -> None: + for task_id in _EXACT_TASK_IDS: + result = grid_induction.run_public_task_v2_2( + public_task(task_id), + engine=None, + mechanistic=True, + ) + + assert result["predictions"] == expected_predictions(task_id) + trace = result["mechanistic_trace"] + final = [item for item in trace["events"] if item["event"] == "final_decision"][0] + successes = [ + item + for item in trace["events"] + if item.get("event") == "hidden_execution" + and item.get("candidate_kind") == "reflective_symmetry_complete" + and item.get("outcome") == "SUCCEEDED" + ] + assert trace["terminal_status"] == "ACCEPTED" + assert final["selected_candidate_id"].startswith("symmetry_complete:") + assert successes + + +def test_b8825c91_preserves_hidden_symmetry_ambiguity() -> None: + result = grid_induction.run_public_task_v2_2( + public_task("b8825c91"), + engine=None, + mechanistic=True, + ) + + assert result["predictions"] == [ + {"status": "AMBIGUOUS_PROGRAM", "prediction": None} + ] + trace = result["mechanistic_trace"] + classes = [ + item + for item in trace["events"] + if item["event"] == "equivalence_class" + and any( + str(candidate_id).startswith("symmetry_complete:") + for candidate_id in item["candidate_ids"] + ) + ] + assert trace["terminal_status"] == "AMBIGUOUS_PROGRAM" + assert len(classes) >= 2 diff --git a/research/universal_core/holonomy_v2_2/tests/test_symmetry_complete_fixtures.py b/research/universal_core/holonomy_v2_2/tests/test_symmetry_complete_fixtures.py new file mode 100644 index 00000000..f6b27356 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_symmetry_complete_fixtures.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +import hashlib +from pathlib import Path + + +_FIXTURES = Path(__file__).parent / "fixtures" / "symmetry_complete" +_EXPECTED_SHA256 = { + "496994bd.json": "ecc40bb943911e33cdfc1e668a5bb49b22ad55a662aa37d68ba57e884897dc25", + "b8825c91.json": "ccf892a7ba8797ab4c69b241d80dd389cde0b9c7b8c0667a8b3a273873c4d49d", + "e729b7be.json": "d2565e3c8e6482a1a3cec2ccf604032c223ee6d6f579dfee98a72b4cbdc12c99", + "f25ffba3.json": "98bd5da81220ac72cf0a2d8ae093ed602066e14cb28ffce6db3110d91382067c", +} + + +def test_symmetry_complete_fixtures_match_pinned_training_payloads() -> None: + actual = { + path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in sorted(_FIXTURES.glob("*.json")) + } + + assert actual == _EXPECTED_SHA256 diff --git a/research/universal_core/holonomy_v2_2/tests/test_trace_diagnosis.py b/research/universal_core/holonomy_v2_2/tests/test_trace_diagnosis.py new file mode 100644 index 00000000..600aa71f --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_trace_diagnosis.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +from research.universal_core.holonomy_v2_2.mechanistic_trace import TraceRecorder +from research.universal_core.holonomy_v2_2.trace_diagnosis import build_trace_diagnosis + + +def accepted_trace(task_id: str) -> dict[str, object]: + recorder = TraceRecorder(task_id) + program = {"kind": "identity"} + program_ref = recorder.store(program) + grid_ref = recorder.store([[1]]) + batch_ref = recorder.store([[[1]]]) + recorder.emit("run_start", version="test", demo_count=1, hidden_count=1, mode="GRID") + recorder.emit( + "candidate_proposed", + candidate_id="fixed:0000", + generator="fixed", + generator_ordinal=0, + candidate_kind="identity", + program_digest=program_ref, + program_ref=program_ref, + ) + recorder.emit( + "demo_execution", + candidate_id="fixed:0000", + candidate_kind="identity", + demo_index=0, + outcome="MATCH", + source_ref=grid_ref, + target_ref=grid_ref, + output_ref=grid_ref, + ) + recorder.emit( + "candidate_fit_decision", + candidate_id="fixed:0000", + candidate_kind="identity", + decision="FITS_ALL_DEMOS", + ) + recorder.emit( + "hidden_execution", + candidate_id="fixed:0000", + candidate_kind="identity", + hidden_index=0, + outcome="SUCCEEDED", + input_ref=grid_ref, + output_ref=grid_ref, + ) + recorder.emit( + "hidden_batch_decision", + candidate_id="fixed:0000", + candidate_kind="identity", + decision="COMPLETED", + batch_ref=batch_ref, + ) + recorder.emit( + "equivalence_class", + batch_ref=batch_ref, + candidate_ids=["fixed:0000"], + candidate_kinds=["identity"], + ) + recorder.emit( + "final_decision", + status="ACCEPTED", + candidate_ids=["fixed:0000"], + selected_candidate_id="fixed:0000", + ) + return recorder.seal("ACCEPTED") + + +def failed_trace(task_id: str) -> dict[str, object]: + recorder = TraceRecorder(task_id) + program = {"kind": "color_map", "mapping": {"0": 1}} + program_ref = recorder.store(program) + grid_ref = recorder.store([[0]]) + recorder.emit("run_start", version="test", demo_count=1, hidden_count=1, mode="GRID") + recorder.emit( + "candidate_proposed", + candidate_id="color_map:0000", + generator="color_map", + generator_ordinal=0, + candidate_kind="color_map", + program_digest=program_ref, + program_ref=program_ref, + ) + recorder.emit( + "demo_execution", + candidate_id="color_map:0000", + candidate_kind="color_map", + demo_index=0, + outcome="MATCH", + source_ref=grid_ref, + target_ref=grid_ref, + output_ref=grid_ref, + ) + recorder.emit( + "candidate_fit_decision", + candidate_id="color_map:0000", + candidate_kind="color_map", + decision="FITS_ALL_DEMOS", + ) + recorder.emit( + "hidden_execution", + candidate_id="color_map:0000", + candidate_kind="color_map", + hidden_index=0, + outcome="FAILED", + input_ref=grid_ref, + exception_type="KeyError", + exception_message="'unmapped grid cell'", + ) + recorder.emit( + "hidden_batch_decision", + candidate_id="color_map:0000", + candidate_kind="color_map", + decision="FAILED", + ) + recorder.emit( + "final_decision", + status="EXECUTION_FAILED", + candidate_ids=["color_map:0000"], + ) + return recorder.seal("EXECUTION_FAILED") + + +def test_diagnosis_counts_terminal_decisions_selected_kinds_and_hidden_failures() -> None: + diagnosis = build_trace_diagnosis([ + accepted_trace("accepted"), + failed_trace("failed"), + ]) + + assert diagnosis["trace_count"] == 2 + assert diagnosis["terminal_status_counts"] == { + "ACCEPTED": 1, + "EXECUTION_FAILED": 1, + } + assert diagnosis["accepted_selected_kind_counts"] == {"identity": 1} + assert diagnosis["hidden_failure_counts"] == { + "color_map": {"KeyError": 1}, + } + assert diagnosis["candidate_fit_counts"]["color_map"] == { + "FITS_ALL_DEMOS": 1, + } + assert len(diagnosis["diagnosis_digest"]) == 64 + + +def test_diagnosis_is_deterministic_across_input_order() -> None: + first = build_trace_diagnosis([accepted_trace("a"), failed_trace("b")]) + second = build_trace_diagnosis([failed_trace("b"), accepted_trace("a")]) + + assert first == second diff --git a/research/universal_core/holonomy_v2_2/tests/test_training_eval.py b/research/universal_core/holonomy_v2_2/tests/test_training_eval.py new file mode 100644 index 00000000..f58f0974 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_training_eval.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from research.universal_core.holonomy_v2_2 import training_eval + + +def write_task(path: Path, target: int) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps( + { + "train": [{"input": [[0]], "output": [[target]]}], + "test": [{"input": [[0]], "output": [[target]]}], + } + ), + encoding="utf-8", + ) + + +def build_vendor(tmp_path: Path) -> Path: + vendor = tmp_path / "vendor" + write_task(vendor / "arc_agi_2" / "data" / "training" / "b.json", 2) + write_task(vendor / "arc_agi_2" / "data" / "training" / "a.json", 1) + return vendor + + +def test_training_run_is_ordered_and_keeps_test_targets_outside_core(tmp_path: Path, monkeypatch: Any) -> None: + vendor = build_vendor(tmp_path) + captured: list[dict[str, object]] = [] + + def fake_run(task: dict[str, object], engine: object) -> dict[str, object]: + captured.append(task) + expected = 1 if task["task_id"] == "a" else 2 + return { + "task_id": task["task_id"], + "predictions": [{"status": "ACCEPTED", "prediction": [[expected]]}], + } + + monkeypatch.setattr(training_eval, "run_public_task_v2_2", fake_run) + + report = training_eval.run_training_corpus( + vendor, + engine=object(), + run_label="training-0001", + candidate_commit="candidate-sha", + ) + + assert [item["task_id"] for item in report["reports"]] == ["a", "b"] + assert report["metrics"]["correct"] == 2 + assert report["metrics"]["coverage"] == 1.0 + assert report["status_counts"] == {"ACCEPTED": 2} + assert report["identity"]["candidate_commit"] == "candidate-sha" + assert len(report["report_digest"]) == 64 + assert [item["hidden_inputs"] for item in captured] == [([[0]],), ([[0]],)] + assert all("output" not in item for item in captured) + + +def test_training_run_counts_core_exception_as_failed_row(tmp_path: Path, monkeypatch: Any) -> None: + vendor = build_vendor(tmp_path) + + def fail(task: object, engine: object) -> dict[str, object]: + raise RuntimeError("boom") + + monkeypatch.setattr(training_eval, "run_public_task_v2_2", fail) + + report = training_eval.run_training_corpus(vendor, engine=object(), run_label="training-failure") + + assert report["metrics"]["rows"] == 2 + assert report["metrics"]["failed"] == 2 + assert report["status_counts"] == {"RUNNER_FAILED": 2} + assert all(item["error_type"] == "RuntimeError" for item in report["reports"]) + + +def test_write_report_emits_canonical_json(tmp_path: Path) -> None: + report = {"report_digest": "a" * 64, "metrics": {"rows": 0}} + path = training_eval.write_report(report, tmp_path / "nested" / "report.json") + + assert path.read_text(encoding="utf-8").endswith("\n") + assert json.loads(path.read_text(encoding="utf-8")) == report diff --git a/research/universal_core/holonomy_v2_2/tests/test_training_mechanistic.py b/research/universal_core/holonomy_v2_2/tests/test_training_mechanistic.py new file mode 100644 index 00000000..19e754a5 --- /dev/null +++ b/research/universal_core/holonomy_v2_2/tests/test_training_mechanistic.py @@ -0,0 +1,233 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from research.universal_core.holonomy_v2_2 import training_eval +from research.universal_core.holonomy_v2_2.mechanistic_trace import ( + TraceRecorder, + validate_trace, +) + + +def write_task(path: Path, target: int) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps({ + "train": [{"input": [[0]], "output": [[target]]}], + "test": [{"input": [[0]], "output": [[target]]}], + }), + encoding="utf-8", + ) + + +def build_vendor(tmp_path: Path) -> Path: + vendor = tmp_path / "vendor" + write_task(vendor / "arc_agi_2" / "data" / "training" / "b.json", 2) + write_task(vendor / "arc_agi_2" / "data" / "training" / "a.json", 1) + return vendor + + +def accepted_trace(task_id: str, prediction: list[list[int]]) -> dict[str, object]: + recorder = TraceRecorder(task_id) + program = {"kind": "color_map", "mapping": {"0": prediction[0][0]}} + program_ref = recorder.store(program) + source_ref = recorder.store([[0]]) + target_ref = recorder.store(prediction) + batch_ref = recorder.store([prediction]) + recorder.emit("run_start", version="test", demo_count=1, hidden_count=1, mode="GRID") + recorder.emit( + "candidate_proposed", + candidate_id="color_map:0000", + generator="color_map", + generator_ordinal=0, + candidate_kind="color_map", + program_digest=program_ref, + program_ref=program_ref, + ) + recorder.emit( + "demo_execution", + candidate_id="color_map:0000", + candidate_kind="color_map", + demo_index=0, + outcome="MATCH", + source_ref=source_ref, + target_ref=target_ref, + output_ref=target_ref, + ) + recorder.emit( + "candidate_fit_decision", + candidate_id="color_map:0000", + candidate_kind="color_map", + decision="FITS_ALL_DEMOS", + ) + recorder.emit( + "hidden_execution", + candidate_id="color_map:0000", + candidate_kind="color_map", + hidden_index=0, + outcome="SUCCEEDED", + input_ref=source_ref, + output_ref=target_ref, + ) + recorder.emit( + "hidden_batch_decision", + candidate_id="color_map:0000", + candidate_kind="color_map", + decision="COMPLETED", + batch_ref=batch_ref, + ) + recorder.emit( + "equivalence_class", + batch_ref=batch_ref, + candidate_ids=["color_map:0000"], + candidate_kinds=["color_map"], + ) + recorder.emit( + "final_decision", + status="ACCEPTED", + candidate_ids=["color_map:0000"], + selected_candidate_id="color_map:0000", + ) + return recorder.seal("ACCEPTED") + + +def runner_result(task_id: str, prediction: list[list[int]]) -> dict[str, object]: + return { + "task_id": task_id, + "predictions": [{"status": "ACCEPTED", "prediction": prediction}], + } + + +def test_training_run_writes_ordered_trace_files_and_digest_only_references( + tmp_path: Path, + monkeypatch: Any, +) -> None: + vendor = build_vendor(tmp_path) + trace_dir = tmp_path / "traces" + calls: list[tuple[str, bool]] = [] + + def fake_run( + task: dict[str, object], + engine: object, + *, + mechanistic: bool = False, + ) -> dict[str, object]: + task_id = str(task["task_id"]) + calls.append((task_id, mechanistic)) + prediction = [[1 if task_id == "a" else 2]] + result = runner_result(task_id, prediction) + if mechanistic: + result["mechanistic_trace"] = accepted_trace(task_id, prediction) + return result + + monkeypatch.setattr(training_eval, "run_public_task_v2_2", fake_run) + + report = training_eval.run_training_corpus( + vendor, + engine=object(), + run_label="training-mechanistic", + candidate_commit="candidate-sha", + mechanistic_dir=trace_dir, + verify_parity=True, + ) + + assert calls == [("a", False), ("a", True), ("b", False), ("b", True)] + assert [item["task_id"] for item in report["reports"]] == ["a", "b"] + assert sorted(path.name for path in trace_dir.glob("*.json")) == [ + "a.json", + "b.json", + "diagnosis.json", + ] + assert report["mechanistic"]["trace_count"] == 2 + assert report["mechanistic"]["parity_count"] == 2 + assert report["mechanistic"]["diagnosis_path"] == "diagnosis.json" + for item in report["reports"]: + reference = item["mechanistic_trace"] + assert set(reference) == { + "path", + "trace_digest", + "envelope_digest", + "event_count", + "terminal_status", + } + assert "mechanistic_trace" not in item["raw_result"] + trace = json.loads((trace_dir / reference["path"]).read_text(encoding="utf-8")) + validate_trace(trace) + assert trace["trace_digest"] == reference["trace_digest"] + assert trace["envelope_digest"] == reference["envelope_digest"] + diagnosis = json.loads((trace_dir / "diagnosis.json").read_text(encoding="utf-8")) + assert diagnosis["trace_count"] == 2 + assert report["mechanistic"]["diagnosis_digest"] == diagnosis["diagnosis_digest"] + + +def test_training_run_fails_closed_on_mechanistic_parity_mismatch( + tmp_path: Path, + monkeypatch: Any, +) -> None: + vendor = build_vendor(tmp_path) + + def fake_run( + task: dict[str, object], + engine: object, + *, + mechanistic: bool = False, + ) -> dict[str, object]: + task_id = str(task["task_id"]) + compact_prediction = [[1 if task_id == "a" else 2]] + prediction = [[9]] if mechanistic else compact_prediction + result = runner_result(task_id, prediction) + if mechanistic: + result["mechanistic_trace"] = accepted_trace(task_id, prediction) + return result + + monkeypatch.setattr(training_eval, "run_public_task_v2_2", fake_run) + + with pytest.raises(RuntimeError, match="mechanistic parity failure for task a"): + training_eval.run_training_corpus( + vendor, + engine=object(), + run_label="parity-failure", + mechanistic_dir=tmp_path / "traces", + verify_parity=True, + ) + + +def test_trace_files_do_not_embed_the_aggregate_report_or_scoring_metrics( + tmp_path: Path, + monkeypatch: Any, +) -> None: + vendor = build_vendor(tmp_path) + trace_dir = tmp_path / "traces" + + def fake_run( + task: dict[str, object], + engine: object, + *, + mechanistic: bool = False, + ) -> dict[str, object]: + task_id = str(task["task_id"]) + prediction = [[1 if task_id == "a" else 2]] + result = runner_result(task_id, prediction) + if mechanistic: + result["mechanistic_trace"] = accepted_trace(task_id, prediction) + return result + + monkeypatch.setattr(training_eval, "run_public_task_v2_2", fake_run) + training_eval.run_training_corpus( + vendor, + engine=object(), + run_label="no-score-leak", + mechanistic_dir=trace_dir, + verify_parity=True, + ) + + for path in (trace_dir / "a.json", trace_dir / "b.json"): + text = path.read_text(encoding="utf-8") + assert '"metrics"' not in text + assert '"correct"' not in text + assert '"raw_accuracy"' not in text + assert '"report_digest"' not in text diff --git a/research/universal_core/holonomy_v2_2/trace_diagnosis.py b/research/universal_core/holonomy_v2_2/trace_diagnosis.py new file mode 100644 index 00000000..f37898ec --- /dev/null +++ b/research/universal_core/holonomy_v2_2/trace_diagnosis.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any, Mapping, Sequence + +from .mechanistic_trace import validate_trace + + +def _canonical(value: Any) -> bytes: + return ( + json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ) + + "\n" + ).encode("utf-8") + + +def _digest(value: Any) -> str: + return hashlib.sha256(_canonical(value)).hexdigest() + + +def _increment(counts: dict[str, int], key: str) -> None: + counts[key] = counts.get(key, 0) + 1 + + +def _increment_nested( + counts: dict[str, dict[str, int]], + outer: str, + inner: str, +) -> None: + bucket = counts.setdefault(outer, {}) + bucket[inner] = bucket.get(inner, 0) + 1 + + +def _sorted_counts(counts: Mapping[str, int]) -> dict[str, int]: + return {key: int(counts[key]) for key in sorted(counts)} + + +def _sorted_nested( + counts: Mapping[str, Mapping[str, int]], +) -> dict[str, dict[str, int]]: + return { + outer: _sorted_counts(counts[outer]) + for outer in sorted(counts) + } + + +def build_trace_diagnosis( + traces: Sequence[Mapping[str, Any]], +) -> dict[str, Any]: + ordered = sorted(traces, key=lambda item: str(item.get("task_id", ""))) + terminal_status_counts: dict[str, int] = {} + generator_skip_counts: dict[str, dict[str, int]] = {} + candidate_fit_counts: dict[str, dict[str, int]] = {} + hidden_failure_counts: dict[str, dict[str, int]] = {} + hidden_failure_message_counts: dict[str, dict[str, int]] = {} + accepted_selected_kind_counts: dict[str, int] = {} + ambiguous_kind_set_counts: dict[str, int] = {} + total_event_count = 0 + + for trace in ordered: + validate_trace(trace) + events_value = trace["events"] + events = [dict(item) for item in events_value] + total_event_count += len(events) + candidate_meta: dict[str, dict[str, str]] = {} + for event in events: + if event.get("event") != "candidate_proposed": + continue + candidate_id = str(event["candidate_id"]) + candidate_meta[candidate_id] = { + "generator": str(event.get("generator", "UNKNOWN")), + "kind": str(event.get("candidate_kind", "UNKNOWN")), + } + + terminal_status = str(trace["terminal_status"]) + _increment(terminal_status_counts, terminal_status) + + for event in events: + name = event.get("event") + if name == "generator_skip": + _increment_nested( + generator_skip_counts, + str(event.get("generator", "UNKNOWN")), + str(event.get("reason", "UNKNOWN")), + ) + elif name == "candidate_fit_decision": + candidate_id = str(event.get("candidate_id", "")) + generator = candidate_meta.get(candidate_id, {}).get("generator", "UNKNOWN") + _increment_nested( + candidate_fit_counts, + generator, + str(event.get("decision", "UNKNOWN")), + ) + elif name == "hidden_execution" and event.get("outcome") == "FAILED": + candidate_id = str(event.get("candidate_id", "")) + kind = str( + event.get("candidate_kind") + or candidate_meta.get(candidate_id, {}).get("kind", "UNKNOWN") + ) + exception_type = str(event.get("exception_type", "UNKNOWN")) + exception_message = str(event.get("exception_message", "")) + _increment_nested(hidden_failure_counts, kind, exception_type) + _increment_nested( + hidden_failure_message_counts, + kind, + f"{exception_type}:{exception_message}", + ) + + final_events = [ + event for event in events if event.get("event") == "final_decision" + ] + final = final_events[0] + if terminal_status == "ACCEPTED": + selected = str(final.get("selected_candidate_id", "")) + kind = candidate_meta.get(selected, {}).get("kind", "UNKNOWN") + _increment(accepted_selected_kind_counts, kind) + elif terminal_status == "AMBIGUOUS_PROGRAM": + kinds = sorted({ + candidate_meta.get(str(candidate_id), {}).get("kind", "UNKNOWN") + for candidate_id in final.get("candidate_ids", []) + }) + _increment(ambiguous_kind_set_counts, "+".join(kinds) or "NONE") + + body: dict[str, Any] = { + "schema_version": 1, + "trace_count": len(ordered), + "total_event_count": total_event_count, + "terminal_status_counts": _sorted_counts(terminal_status_counts), + "generator_skip_counts": _sorted_nested(generator_skip_counts), + "candidate_fit_counts": _sorted_nested(candidate_fit_counts), + "hidden_failure_counts": _sorted_nested(hidden_failure_counts), + "hidden_failure_message_counts": _sorted_nested(hidden_failure_message_counts), + "accepted_selected_kind_counts": _sorted_counts(accepted_selected_kind_counts), + "ambiguous_kind_set_counts": _sorted_counts(ambiguous_kind_set_counts), + } + body["diagnosis_digest"] = _digest(body) + return body + + +def write_diagnosis( + diagnosis: Mapping[str, Any], + path: Path | str, +) -> Path: + expected = dict(diagnosis) + digest = expected.pop("diagnosis_digest", None) + if not isinstance(digest, str) or digest != _digest(expected): + raise ValueError("diagnosis digest mismatch") + output = Path(path) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(_canonical(diagnosis)) + return output diff --git a/research/universal_core/holonomy_v2_2/training_eval.py b/research/universal_core/holonomy_v2_2/training_eval.py new file mode 100644 index 00000000..64e7106d --- /dev/null +++ b/research/universal_core/holonomy_v2_2/training_eval.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Mapping + +from research.universal_core.public_benchmarks import suite +from research.universal_core.public_benchmarks.adapters.arc_agi_2 import ( + arc_task_from_payload, + score_result, +) + +from .grid_induction import run_public_task_v2_2 +from .mechanistic_trace import validate_trace, write_trace +from .trace_diagnosis import build_trace_diagnosis, write_diagnosis + + +FROZEN_CORE_COMMIT = "1d7c489bcdbff755d638b35ad47ce62bd4fe829d" + + +def _failed_metrics(task_id: str, rows: int) -> dict[str, Any]: + return { + "task_id": task_id, + "rows": rows, + "correct": 0, + "accepted": 0, + "abstained": 0, + "failed": rows, + "incorrect_attempts": 0, + "raw_accuracy": 0.0, + "coverage": 0.0, + "attempted_accuracy": 0.0, + } + + +def _failure_report(task_id: str, payload: Mapping[str, Any], exc: Exception) -> dict[str, Any]: + test_rows = payload.get("test") + rows = len(test_rows) if isinstance(test_rows, (list, tuple)) else 0 + return { + "task_id": task_id, + "raw_result": { + "task_id": task_id, + "program_digest": None, + "freeze_digest": None, + "prediction_digest": None, + "predictions": [ + {"status": "RUNNER_FAILED", "prediction": None} + for _ in range(rows) + ], + }, + "metrics": _failed_metrics(task_id, rows), + "error_type": type(exc).__name__, + } + + +def _status_counts(reports: list[Mapping[str, Any]]) -> dict[str, int]: + counts: dict[str, int] = {} + for report in reports: + raw = report.get("raw_result") + predictions = raw.get("predictions") if isinstance(raw, Mapping) else None + if not isinstance(predictions, (list, tuple)): + continue + for item in predictions: + status = item.get("status") if isinstance(item, Mapping) else "RUNNER_FAILED" + name = str(status) + counts[name] = counts.get(name, 0) + 1 + return dict(sorted(counts.items())) + + +def _without_mechanistic_trace(result: Mapping[str, Any]) -> dict[str, Any]: + compact = dict(result) + compact.pop("mechanistic_trace", None) + return compact + + +def _trace_reference(trace: Mapping[str, Any], filename: str) -> dict[str, Any]: + return { + "path": filename, + "trace_digest": str(trace["trace_digest"]), + "envelope_digest": str(trace["envelope_digest"]), + "event_count": int(trace["event_count"]), + "terminal_status": str(trace["terminal_status"]), + } + + +def run_training_corpus( + vendor: Path | str, + engine: Any, + *, + run_label: str, + candidate_commit: str | None = None, + mechanistic_dir: Path | str | None = None, + verify_parity: bool = False, +) -> dict[str, Any]: + if not isinstance(run_label, str) or not run_label: + raise ValueError("run label must be a non-empty string") + if verify_parity and mechanistic_dir is None: + raise ValueError("parity verification requires a mechanistic directory") + + trace_root = Path(mechanistic_dir) if mechanistic_dir is not None else None + if trace_root is not None: + trace_root.mkdir(parents=True, exist_ok=True) + + tasks = suite.load_arc_tasks(Path(vendor), "training") + reports: list[dict[str, Any]] = [] + traces: list[Mapping[str, Any]] = [] + parity_count = 0 + + for task_id, payload in tasks: + public_task = arc_task_from_payload(task_id, payload) + try: + compact_result = run_public_task_v2_2(public_task, engine) + report = { + "task_id": task_id, + "raw_result": compact_result, + "metrics": score_result(task_id, payload, compact_result), + } + except Exception as exc: + reports.append(_failure_report(task_id, payload, exc)) + continue + + if trace_root is not None: + traced_result = run_public_task_v2_2( + public_task, + engine, + mechanistic=True, + ) + trace = traced_result.get("mechanistic_trace") + if not isinstance(trace, Mapping): + raise RuntimeError(f"missing mechanistic trace for task {task_id}") + traced_compact = _without_mechanistic_trace(traced_result) + if verify_parity: + if traced_compact != compact_result: + raise RuntimeError(f"mechanistic parity failure for task {task_id}") + parity_count += 1 + validate_trace(trace) + filename = f"{task_id}.json" + write_trace(trace, trace_root / filename) + report["mechanistic_trace"] = _trace_reference(trace, filename) + traces.append(trace) + + reports.append(report) + + body: dict[str, Any] = { + "schema_version": 1, + "run_label": run_label, + "split": "training", + "identity": { + "candidate_commit": candidate_commit, + "frozen_core_commit": FROZEN_CORE_COMMIT, + }, + "evaluated_tasks": len(reports), + "metrics": suite.aggregate_metrics(item["metrics"] for item in reports), + "status_counts": _status_counts(reports), + "reports": reports, + } + + if trace_root is not None: + diagnosis = build_trace_diagnosis(traces) + write_diagnosis(diagnosis, trace_root / "diagnosis.json") + schemas = sorted({str(trace["schema"]) for trace in traces}) + body["mechanistic"] = { + "trace_schema": schemas[0] if len(schemas) == 1 else schemas, + "trace_count": len(traces), + "parity_checked": verify_parity, + "parity_count": parity_count, + "diagnosis_path": "diagnosis.json", + "diagnosis_digest": diagnosis["diagnosis_digest"], + } + + return suite.seal_report(body) + + +def write_report(report: Mapping[str, Any], path: Path | str) -> Path: + output = Path(path) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + json.dumps(report, sort_keys=True, indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + return output