From a17869d957d3ee19f72b7b9626cf9f6a44a8fc35 Mon Sep 17 00:00:00 2001 From: Batuhan Date: Sun, 1 Nov 2026 19:16:08 +0300 Subject: [PATCH 1/2] fix: stabilize executable discovery and catalog identity --- .github/repository-metadata.json | 12 +--- .github/workflows/ci.yml | 4 ++ CHANGELOG.md | 8 +++ README.md | 33 +++++----- docs/ARCHITECTURE.md | 6 +- docs/DISCOVERABILITY.md | 2 +- docs/ROADMAP.md | 8 ++- docs/SPEC.md | 14 +++-- docs/STABILIZATION.md | 100 +++++++++++++++++++++++++++++++ docs/UPSTREAM-SYNC.md | 4 +- pyproject.toml | 6 +- scripts/generate_catalog.py | 5 +- src/rei_rea/data/tools.json | 4 +- src/rei_rea/discovery.py | 50 ++++++++++++++++ src/rei_rea/registry.py | 21 ++++--- src/rei_rea/webui/index.html | 4 +- tests/test_discovery.py | 65 ++++++++++++++++++++ tests/test_metadata.py | 2 +- 18 files changed, 293 insertions(+), 55 deletions(-) create mode 100644 docs/STABILIZATION.md create mode 100644 src/rei_rea/discovery.py create mode 100644 tests/test_discovery.py diff --git a/.github/repository-metadata.json b/.github/repository-metadata.json index 9de7266..8200d5d 100644 --- a/.github/repository-metadata.json +++ b/.github/repository-metadata.json @@ -1,6 +1,6 @@ { - "repository": "rootcastleco/Rei-Reverse-engineer-", - "description": "Evidence-first reverse engineering workbench: binary triage, 124-tool catalog, SHA-256 case ledger, local MCP server, CLI and 10 agent skills.", + "repository": "rootcastleco/Rei-Reverse-engineer", + "description": "Evidence-first reverse engineering workbench with bounded binary analysis, extensible static adapters, MCP and reproducible local investigations.", "homepage": "https://rootcastle.com", "topics": [ "reverse-engineering", @@ -11,15 +11,9 @@ "mcp-server", "model-context-protocol", "security-research", - "software-analysis", - "firmware-analysis", - "android-reverse-engineering", - "electron", - "iot-security", "artifact-analysis", "python", "developer-tools", - "evidence-management", - "reverse-engineering-tools" + "evidence-management" ] } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index faf4257..67ec00c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,3 +23,7 @@ jobs: run: python -m unittest discover -s tests -v - name: Validate built-in command run: python -m rei_rea --version + - name: Verify catalog regeneration + run: | + python scripts/generate_catalog.py + git diff --exit-code -- src/rei_rea/data/tools.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 81c4628..2c58b71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## Unreleased — stabilization + +- Unify catalog adapter flags with code-owned commands; expose exact adapter availability. +- Batch Windows discovery with a permission-aware, bounded parallel fallback. +- Repair canonical repository identity, current capability documentation and About metadata. +- Add discovery regression tests and a catalog regeneration CI gate. + + ## 0.1.0 — 2026-10-31 - Introduce independent Rootcastle REI REA CLI, bounded static inspector, central catalog of 124 external tools, 9 opt-in static parser adapters, case hash chain, signals and comparisons. diff --git a/README.md b/README.md index 0fe9f25..faa8920 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ ### Evidence-first reverse engineering. One workbench. Verifiable findings. -[![CI](https://github.com/rootcastleco/Rei-Reverse-engineer-/actions/workflows/ci.yml/badge.svg)](https://github.com/rootcastleco/Rei-Reverse-engineer-/actions/workflows/ci.yml) +[![CI](https://github.com/rootcastleco/Rei-Reverse-engineer/actions/workflows/ci.yml/badge.svg)](https://github.com/rootcastleco/Rei-Reverse-engineer/actions/workflows/ci.yml) [![Python 3.10+](https://img.shields.io/badge/Python-3.10%2B-0E3D8A?logo=python&logoColor=white)](https://www.python.org/) -[![MCP server](https://img.shields.io/badge/MCP-13%20tools-228B55)](#connect-an-ai-coding-agent) +[![MCP server](https://img.shields.io/badge/MCP-14%20tools-228B55)](#connect-an-ai-coding-agent) [![License: MIT](https://img.shields.io/badge/license-MIT-0E3D8A)](LICENSE) [![Status: Alpha](https://img.shields.io/badge/status-v0.2.0%20alpha-555555)](CHANGELOG.md) @@ -17,7 +17,7 @@ It brings together a **124-entry reverse-engineering tool catalog**, **read-only The objective is not to produce confident-looking pseudocode. It is to make each conclusion **traceable, testable, and reproducible**. > [!IMPORTANT] -> **Alpha scope, stated precisely.** REI REA is an original Rootcastle implementation, not an official fork or feature-parity replacement of [morluto/rea](https://github.com/morluto/rea). A catalog entry does **not** mean a tool is installed, integrated, or validated. Nine fixed static CLI adapters are implemented. External decompilers, debuggers, mobile instrumentation systems, and the upstream REA server are **not included**. See [Capabilities and limitations](#capabilities-and-limitations). +> **Alpha scope, stated precisely.** REI REA is an original Rootcastle implementation, not an official fork or feature-parity replacement of [morluto/rea](https://github.com/morluto/rea). A catalog entry does **not** mean a tool is installed, integrated, or validated. Eleven fixed static CLI adapters are implemented. External decompilers, debuggers, mobile instrumentation systems, and the upstream REA server are **not included**. See [Capabilities and limitations](#capabilities-and-limitations). **[Quick start](#quick-start)** · **[Capabilities](#capabilities-and-limitations)** · **[Architecture](#architecture)** · **[MCP integration](#connect-an-ai-coding-agent)** · **[Agent skills](#twelve-specialist-agent-skills)** · **[Security](#safety-and-trust-boundaries)** · **[Roadmap](docs/ROADMAP.md)** @@ -32,8 +32,8 @@ A reverse-engineering session usually involves multiple disconnected tools. Find | Which tool should I use? | Query one registry by artifact type and capability. | 124 catalog entries with tags, homepages, and PATH checks. | | What exactly did I inspect? | Identify the target before discussing behavior. | SHA-256 hashes, bounded format inspection, and archive metadata. | | Can I reproduce this finding? | Keep observations linked to one artifact identity. | Local hash-linked case events and integrity verification. | -| Can an AI agent use the same evidence? | Expose structured operations over a single interface. | JSON CLI plus thirteen local stdio MCP tools. | -| Can I add more analysis engines? | Use isolated provider contracts instead of rewriting the core. | Nine opt-in static adapters; advanced providers are planned. | +| Can an AI agent use the same evidence? | Expose structured operations over a single interface. | JSON CLI plus fourteen local stdio MCP tools. | +| Can I add more analysis engines? | Use isolated provider contracts instead of rewriting the core. | Eleven opt-in static adapters; advanced providers are planned. | | Is a model's explanation proof? | Distinguish what was seen from what was inferred. | `observed`, `inferred`, and `unknown` evidence states. | **Three distinct facts must never be conflated:** @@ -49,8 +49,8 @@ Even a supported operation is **not end-to-end verified** until its output is ch **Requirements:** Python 3.10 or later, Git, and a local file that you own or are permitted to inspect. The core has no third-party Python runtime dependencies. ```bash -git clone https://github.com/rootcastleco/Rei-Reverse-engineer-.git -cd Rei-Reverse-engineer- +git clone https://github.com/rootcastleco/Rei-Reverse-engineer.git +cd Rei-Reverse-engineer python -m pip install -e . rei-rea --version @@ -122,15 +122,15 @@ A new **SQLite casebook** explicitly indexes local case metadata for search by n | Bounded signals | **Implemented** | Entropy/signature probes and optional strings; scanning is capped. | | Version comparison | **Implemented** | File hashes and bounded byte differences. | | Local evidence cases | **Implemented** | SHA-256 identity, append-only hash-linked event records, verification, and Markdown reports. | -| CLI and agent transport | **Implemented** | JSON CLI and thirteen stdio MCP tools. | +| CLI and agent transport | **Implemented** | JSON CLI and fourteen stdio MCP tools. | | Dashboard | **Implemented** | Loopback-only catalog view; no target execution. | -| External static adapters | **Opt-in** | Nine fixed commands; only executable tools present on the machine can run. | +| External static adapters | **Opt-in** | Eleven fixed commands; only executable tools present on the machine can run. | | Ghidra / IDA / Rizin session control | **Planned** | Names may be cataloged; no live session provider is included. | | JADX / Frida / emulator automation | **Planned** | Skills document procedures, not active runtime integration. | | Upstream REA MCP compatibility | **Planned** | No API-parity claim or bundled upstream server. | | Firmware flashing or live network testing | **Not supported** | Excluded from this alpha gateway. | -The fixed external adapters are `file`, `readelf`, `objdump`, `llvm-objdump`, `llvm-readobj`, `nm`, `exiftool`, `otool`, and `aapt2` (catalog ID `android-aapt2`). Each requires explicit opt-in and runs without a shell or user-supplied arbitrary flags. +The fixed external adapters are `file`, `readelf`, `objdump`, `llvm-objdump`, `llvm-readobj`, `nm`, `exiftool`, `otool`, `aapt2` (catalog ID `android-aapt2`), `rabin2` and `rz-bin`. Each requires explicit opt-in and runs without a shell or user-supplied arbitrary flags. ```bash # Only if readelf is separately installed and the file is authorized: @@ -144,7 +144,7 @@ A failed or missing external parser does not disable the built-in inspection pat ```mermaid flowchart TD U["Analyst / coding agent"] --> CLI["JSON CLI"] - U --> MCP["Local stdio MCP · 8 tools"] + U --> MCP["Local stdio MCP · 14 tools"] U --> UI["Loopback dashboard"] CLI --> Core["REI REA control plane"] MCP --> Core @@ -153,7 +153,7 @@ flowchart TD Core --> Inspector["Static artifact inspector"] Core --> Planner["Format-aware investigation planner"] Core --> Evidence["SHA-256 evidence cases"] - Core --> Adapter["Nine fixed static adapters · explicit opt-in"] + Core --> Adapter["Eleven fixed static adapters · explicit opt-in"] Adapter -. "PATH-installed executable" .-> Local["External analysis tool"] Future["Future provider API: REA / Ghidra / JADX / Frida"] -. "not implemented" .-> Core ``` @@ -199,10 +199,11 @@ For MCP clients that use the `mcpServers` JSON format, a typical entry is: The `rei-rea` executable must be available in the environment used by your MCP client. Some clients require a full executable path and a restart after configuration. -**Thirteen exposed MCP tools:** +**Fourteen exposed MCP tools:** | Tool | Purpose | | --- | --- | +| `rei_rea_providers` | Discover provider profiles without execution. | | `rei_rea_tools` | Filter catalog entries. | | `rei_rea_doctor` | Inspect local PATH availability. | | `rei_rea_inspect` | Identify and hash an artifact. | @@ -217,7 +218,7 @@ The `rei-rea` executable must be available in the environment used by your MCP c | `rei_rea_case_search` | Search an existing local casebook index. | | `rei_rea_bundle_verify` | Check checksums of an exported evidence bundle. | -MCP registration gives access to **these thirteen operations only**. It does not install third-party engines, grant new filesystem permissions, or create a remote analysis service. Client-specific permission settings still apply. +MCP registration gives access to **these fourteen operations only**. It does not install third-party engines, grant new filesystem permissions, or create a remote analysis service. Client-specific permission settings still apply. ## Twelve specialist agent skills @@ -268,7 +269,7 @@ python -m unittest discover -s tests -v python -m rei_rea --version ``` -The v0.2 local baseline passed **46/46 automated tests on 1 November 2026**. These cover artifact formats, archive warnings, opt-in boundaries, catalog consistency, evidence-chain tampering, CLI behavior, stdio MCP handshake, and loopback HTTP behavior. GitHub Actions defines a separate Linux, Windows, and macOS matrix for Python 3.10 and 3.13. **A local pass does not prove CI passed**; consult the live badge and Actions history. +Verification is recorded per commit and environment in [the stabilization report](docs/STABILIZATION.md). GitHub Actions tests Python 3.10 and 3.13 on Linux, Windows and macOS. A local test pass does not establish remote CI or external-provider compatibility. Hardware behavior, mobile instrumentation, third-party engine accuracy, native decompilation parity, cloud deployment, and production performance are **not yet verified**. @@ -289,7 +290,7 @@ Track the scope and acceptance gates in [docs/ROADMAP.md](docs/ROADMAP.md). For
Does REI REA contain 124 working analysis integrations? -No. It has 124 **catalog records**. Nine fixed static tool adapters are implemented, and each adapter needs its executable installed locally. The other records are discovery-only. +No. It has 124 **catalog records**. Eleven fixed static tool adapters are implemented, and each adapter needs its executable installed locally. The other records are discovery-only.
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index ec91a6c..cb6bc83 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,4 +1,4 @@ -# Architecture · v0.1.0 +# Architecture · current Python core ## Layers and trust boundaries @@ -8,7 +8,7 @@ - **Signals / comparisons**: entropy/signature probes, strings opt-in and byte differences, with a 16 MiB scan budget. Full-file SHA-256 remains independent of scan limits. - **Planner**: maps observed target format to one or more specialist skills, tool capability tags and a testable evidence-first workflow. No model-generated code execution. - **Evidence**: local `case.json` and `events.jsonl` with seq, timestamp, predecessor SHA-256 and record SHA-256. Atomic case manifest creation and exclusive writer lock guard against simple concurrency conflicts. -- **Adapters**: fixed Python code map for nine static CLI commands with `shell=False`, no arbitrary flags, explicit `--allow-external`, process timeout and bounded capture. No dynamic methods, network probes or unrestricted `subprocess` API are exposed. +- **Adapters**: fixed Python code map for eleven static CLI commands with `shell=False`, no arbitrary flags, explicit `--allow-external`, process timeout and bounded capture. No dynamic methods, network probes or unrestricted `subprocess` API are exposed. ## Data flow @@ -25,7 +25,7 @@ - SHA-256 identity: hex string, full file contents; directory identity is hash of `relative_path\0size\0file_hash` records, bounded with completeness flag. - Observation states: `observed`, `inferred`, `unknown`. - Tool execution: catalog entry alone never grants execution permission. -- MCP output: JSON-RPC 2.0, stdio JSON lines, bounded inbound message size 1 MiB; eight structured tool contracts. +- MCP output: JSON-RPC 2.0, stdio JSON lines, bounded inbound message size 1 MiB; fourteen structured tool contracts. - Session state: no persistent MCP process handles, no target imports, no background jobs. - Local web: GET only for `/`, `/api/tools`, `/api/health`. Loopback only, conservative CSP, no cookies or tokens. diff --git a/docs/DISCOVERABILITY.md b/docs/DISCOVERABILITY.md index 05d0fff..0f18262 100644 --- a/docs/DISCOVERABILITY.md +++ b/docs/DISCOVERABILITY.md @@ -6,7 +6,7 @@ The GitHub About panel is **repository metadata**, not a field inferred from `RE **Using the GitHub UI (no token needed):** -1. Open `https://github.com/rootcastleco/Rei-Reverse-engineer-` as a repository administrator. +1. Open `https://github.com/rootcastleco/Rei-Reverse-engineer` as a repository administrator. 2. Click the gear icon beside **About** on the main repository page. 3. Paste the `description`, `homepage`, and all `topics` from `.github/repository-metadata.json`. 4. Save. Refresh and confirm the Topics chips are visible and link to corresponding GitHub topic pages. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index a0de5df..2b4f70c 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -14,12 +14,12 @@ - [x] 8-tool stdio MCP server and read-only loopback dashboard. - [x] Basic unit/integration suite, GitHub Actions test matrix definition. -## Next release — v0.2 (not implemented) +## Deferred work from the original v0.2 proposal - [ ] Upstream `morluto/rea` provider adapter with *actual* schema discovery and version-pinned MCP compatibility tests. - [ ] Semantically rich artifact inventory: PE import/relocation tables, ELF symbol/xref index, Mach-O load commands, DEX manifest and Java method indexes. - [ ] Configured sandbox runners (locked-down Windows VM, Linux user namespace, Android emulator); no automatic dynamic launch. -- [ ] Durable indexed case search, exportable report bundles and baseline diff. +- [x] Durable indexed case search, exportable report bundles and baseline diff (delivered in v0.2). - [ ] Cryptographic event-chain signing with external timestamp and key rotation. - [ ] Malformed binary and archive fuzzing, memory/CPU quotas, threat-model conformance tests. @@ -50,3 +50,7 @@ - [ ] Run coverage-guided parser fuzzing (AFL++/libFuzzer via wrapper) and establish memory/CPU budgets. - [ ] Optional signed, externally timestamped evidence receipts with documented key custody and rotation. - [ ] Portable dashboard for casebook browsing with explicit opt-in, authentication and local threat model. + +## Current execution plan + +The next vertical slices and their executable acceptance gates are tracked in [STABILIZATION.md](STABILIZATION.md#executable-next-milestones). Historical delivered counts above describe their release, not the current total. diff --git a/docs/SPEC.md b/docs/SPEC.md index 5c23f7b..a4a1fb9 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -1,4 +1,4 @@ -# REI REA v0.1 specification and verification map +# REI REA specification and verification map ## User stories @@ -9,8 +9,8 @@ - **REA-005** Analyst can launch only predefined static parsers with explicit CLI authorization and timeout. Tests: `test_registry`, `test_cli`. - **REA-006** Agent can use one stdio MCP interface to the same core capabilities. Test: end-to-end subprocess handshake in `test_mcp`. - **REA-007** Analyst can compare known file revisions and inspect bounded entropy/signatures with strings suppressed by default. Test: `test_signals`. -- **REA-008** Analyst can browse catalog over local HTTP with no write/execution endpoints. Test: web handler definition and planned localhost E2E. -- **REA-009** Ten standalone specialist skill entrypoints have valid frontmatter and task-specific acceptance. Test: external `quick_validate.py` for each and packaged root skill. +- **REA-008** Analyst can browse catalog over local HTTP with no write/execution endpoints. Test: `test_web` live loopback E2E. +- **REA-009** Twelve standalone specialist skill entrypoints have valid frontmatter and task-specific acceptance. Test: external `quick_validate.py` for each and packaged root skill. ## Explicit exclusions @@ -23,7 +23,7 @@ - New `run` accepts only a fixed code-reviewed adapter ID and explicit opt-in. - Every response reports bounded inspection and evidentiary limitations. - The case hash chain detects a one-field modification and optional target rehash detects changes. -- Ten validated skill folders and at least one valid single-skill ZIP. +- Twelve validated skill folders and at least one valid single-skill ZIP. - No claim of CI success until the remote GitHub Actions matrix executes. ## Rollback @@ -41,4 +41,8 @@ Features are additive in a new repository. To disable tool launching, never call ### v0.2 acceptance -44 existing/new local tests pass. Python 3.10 and 3.13 tests must pass on Linux, macOS and Windows before merging. Twelve skill folders validate. No migration of case manifests required. No database index is created by a read-only command. +All current regression tests must pass; see `STABILIZATION.md` for measured results. Python 3.10 and 3.13 tests must pass on Linux, macOS and Windows before merging. Twelve skill folders validate. No migration of case manifests required. No database index is created by a read-only command. + +## Current stabilization contract + +The provider inventory adds `rei_rea_providers` (14 MCP tools). `case_create` writes local metadata; the remaining tools are read-only. Discovery and adapter-availability contracts are specified in [STABILIZATION.md](STABILIZATION.md). diff --git a/docs/STABILIZATION.md b/docs/STABILIZATION.md new file mode 100644 index 0000000..d456841 --- /dev/null +++ b/docs/STABILIZATION.md @@ -0,0 +1,100 @@ +# Stabilization milestone — 2026-11-01 + +## Baseline and repository audit + +Canonical repository: https://github.com/rootcastleco/Rei-Reverse-engineer + +Baseline: `48c257c631e89ed704d7d497af2a8910f18b5313`, Python package 0.2.0. +Local environment: Windows 11 build 26300, Anaconda CPython 3.14.7, AMD64. +The unmodified suite ran 52 tests in 79.166 seconds: 50 passed, one +`plan` CLI timeout (10 seconds), one skipped because `file` was absent. +The environment restricts listing of several PATH directories. A reduced PATH +passed the failing test, identifying executable discovery as the bottleneck. + +Remote main CI succeeded at this exact baseline SHA: +https://github.com/rootcastleco/Rei-Reverse-engineer/actions/runs/41323430052 +No published releases were returned by the repository release listing. +Topics were empty; About metadata referenced older scope. + +Open changes reviewed: + +- PR #6, `5d24e76de03cd577bac5b835bace3c337099e172`: bounded electronics intake, + CLI/MCP endpoint and a specialist skill. CI succeeded at that SHA in run + 41323544503. This is signature/checksum intake, not PCB reconstruction. + Record validation accepts unsupported S-record types; follow-up acceptance + must explicitly separate checksums from full format parsing. Kept separate + from this stabilization PR so its intake contract can be tightened first. +- PR #4, `ce852cb5b0ae7e119ab4c461687e29256d398d03`: only the upstream gitlink + changes from `f72638f6` to `aa48d494`. CI run 41322739682 is `action_required`. + No upstream API compatibility has been established; no automatic merge. +- Other remote feature branches have corresponding historical changes; the + open-PR inventory returned only #4 and #6. + +## Observable requirements and verification + +| ID | Requirement | Verification | +| --- | --- | --- | +| STAB-01 | Canonical URLs in active package/docs/metadata | tracked-file search; metadata tests | +| STAB-02 | Catalog flags equal code-owned adapter IDs in both directions | `test_discovery` and regeneration CI gate | +| STAB-03 | Detecting r2 alone does not advertise rabin2 execution | frontend-only discovery regression | +| STAB-04 | No stale PATH results between requests | create/remove executable regression | +| STAB-05 | Windows discovery preserves `shutil.which` authority | denied-directory fallback and final resolver tests | +| STAB-06 | Original CLI `plan` passes its unchanged 10-second deadline | `test_cli.CliTests.test_inspect_and_plan_cli` | +| STAB-07 | Legacy CLI/MCP/evidence contracts continue working | full regression suite | + +Discovery snapshots enumerate Windows directories once when permitted and +resolve potential matches with `shutil.which`. Unlistable directories cause +a conservative fallback to the standard resolver with eight concurrent I/O +workers. No tool is executed, no result persists across requests, and there is +no ceiling on registry size. Network paths can still be slow; this does not +implement cancellation of blocked operating-system filesystem calls. + +`installed` and `resolved_executable` retain their historical PATH meaning. +Additive fields `adapter_available` and `adapter_executable` identify the exact +fixed command's executable. They do not establish versions, licensing, or E2E +verification. The dashboard and doctor use this more precise availability. + +## Scope and release gate + +This milestone does not add external engines or alter evidence files. The +14 MCP tools, 12 provider profiles and 12 skills remain available. Eleven fixed +adapters exist; their installation and actual parser accuracy are independent +checks. No full desktop application or hardware verification is claimed. + +Release gates: local regression suite, all skill validations, deterministic +catalog regeneration, wheel build/smoke test and the remote six-job OS/Python +matrix. Record measured results in the PR; do not infer a release from CI alone. + +## Executable next milestones + +1. **Electronics intake / PR #6:** add unsupported-record-type, termination, + count, truncated-UF2 and misleading-header fixtures. Define Recognized vs + Parsed separately. Merge only after exact-head CI and contract review. +2. **Provider contract v1:** normalize fixed-adapter outputs with tool version, + artifact hash, operation schema, error category and limits. Real rabin2, + rz-bin and LLVM tests must pass on pinned installations; otherwise label + E2E NOT VERIFIED. Preserve the existing `run` response fields. +3. **Firmware address maps:** Intel HEX/S-record/UF2 parsing with checksum, + address/overlap and termination checks; CLI and MCP share one parser. + Malformed fixtures, bounded reads and >=85% new-module coverage are gates. +4. **Artifact graph:** versioned local nodes/edges with artifact hash, source, + parser version and OBSERVED/INFERRED/VERIFIED status. Test contradictions, + stale identities and round trips before linking firmware to hardware. +5. **Engineering application:** evaluate local web, PySide and Tauri against + packaging cost and existing Python interfaces; implement one real case + timeline/import/report workflow, keyboard navigation and EN/TR strings. +6. **Distribution:** reproducible wheel/sdist, license inventory, SBOM and + attestations; verify each built platform artifact before publication. + +Upstream compatibility, physical measurements, proprietary providers and +cross-platform desktop binaries remain unverified dependencies, not completed +features. The existing daily upstream review workflow stays independent. + +## Local verification results + +- Full suite: 59 tests in 79.342 seconds, 58 passed and one skipped (`file` absent), no failures. No overall-suite speedup is claimed. +- New discovery module: coverage.py 7.11.0 reports 100% statement/branch coverage (29 statements, 10 branches); seven focused tests. This is not whole-project coverage. +- All 12 skills passed the installed skill validator. +- Python wheel built locally; publication and platform-specific executables remain separate gates. +- The original CLI timeout regression passes on the existing PATH without changing its 10-second deadline. +- External-engine E2E, peak memory and parser performance benchmarks were not measured in this milestone. diff --git a/docs/UPSTREAM-SYNC.md b/docs/UPSTREAM-SYNC.md index 2564df3..5816660 100644 --- a/docs/UPSTREAM-SYNC.md +++ b/docs/UPSTREAM-SYNC.md @@ -22,8 +22,8 @@ Rootcastle REI REA is a **separate Python workbench**. Its upstream reference is ## Manual setup ```bash -git clone https://github.com/rootcastleco/Rei-Reverse-engineer-.git -cd Rei-Reverse-engineer- +git clone https://github.com/rootcastleco/Rei-Reverse-engineer.git +cd Rei-Reverse-engineer git submodule update --init upstream/rea git -C upstream/rea rev-parse HEAD ``` diff --git a/pyproject.toml b/pyproject.toml index 3c435bd..4edb6a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,9 +14,9 @@ classifiers = ["Development Status :: 3 - Alpha", "Intended Audience :: Develope [project.urls] Homepage = "https://rootcastle.com" -Repository = "https://github.com/rootcastleco/Rei-Reverse-engineer-" -Documentation = "https://github.com/rootcastleco/Rei-Reverse-engineer-#readme" -Issues = "https://github.com/rootcastleco/Rei-Reverse-engineer-/issues" +Repository = "https://github.com/rootcastleco/Rei-Reverse-engineer" +Documentation = "https://github.com/rootcastleco/Rei-Reverse-engineer#readme" +Issues = "https://github.com/rootcastleco/Rei-Reverse-engineer/issues" [project.scripts] rei-rea = "rei_rea.cli:main" diff --git a/scripts/generate_catalog.py b/scripts/generate_catalog.py index 537c660..f0e0072 100644 --- a/scripts/generate_catalog.py +++ b/scripts/generate_catalog.py @@ -5,6 +5,9 @@ """ from pathlib import Path import json +import sys +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) +from rei_rea.registry import STATIC_ADAPTERS # id | name | category | executable alternatives | tags | official website ROWS = r''' @@ -134,7 +137,7 @@ git|Git|data|git|all,source|https://git-scm.com/ '''.strip() -ADAPTERS = {"file", "readelf", "objdump", "llvm-objdump", "llvm-readobj", "nm", "exiftool", "otool", "android-aapt2"} +ADAPTERS = set(STATIC_ADAPTERS) expected = {"all", "native", "firmware", "android", "apple", "dotnet", "wasm", "electron", "javascript", "web", "network", "java", "runtime", "symbolic", "evidence", "source", "static", "data", "security", "scripting", "malware", "decompile", "disassemble", "assemble", "debug", "memory", "forensics", "windows", "linux", "emulation", "filesystem", "formats", "hardware", "uefi", "build", "metadata", "protocol", "agent", "workflow"} items=[] for i, line in enumerate(ROWS.splitlines(),1): diff --git a/src/rei_rea/data/tools.json b/src/rei_rea/data/tools.json index 951c893..d2f1b89 100644 --- a/src/rei_rea/data/tools.json +++ b/src/rei_rea/data/tools.json @@ -98,7 +98,7 @@ "firmware" ], "homepage": "https://rizin.re/", - "integration": "discovery-only", + "integration": "static-adapter", "requires_authorization": true, "bundled": false }, @@ -115,7 +115,7 @@ "firmware" ], "homepage": "https://rada.re/", - "integration": "discovery-only", + "integration": "static-adapter", "requires_authorization": true, "bundled": false }, diff --git a/src/rei_rea/discovery.py b/src/rei_rea/discovery.py new file mode 100644 index 0000000..9b535a3 --- /dev/null +++ b/src/rei_rea/discovery.py @@ -0,0 +1,50 @@ +"""Fresh, read-only executable discovery; never starts a process.""" +from __future__ import annotations + +import os +import shutil +import sys +from concurrent.futures import ThreadPoolExecutor + + +def _windows_names() -> set[str] | None: + """Prefilter absent names with one directory scan, not thousands of stats. + + Include cwd conservatively; shutil.which retains authority over PATHEXT, + precedence and whether the current directory is permitted on this Python. + No results survive a discovery request. + """ + names: set[str] = set() + directories = dict.fromkeys([os.curdir, *os.environ.get("PATH", "").split(os.pathsep)]) + for directory in directories: + try: + with os.scandir(directory or os.curdir) as entries: + for entry in entries: + name = entry.name.casefold() + names.add(name) + for index, char in enumerate(name): + if char == '.': + names.add(name[:index]) + except PermissionError: + # A known file may be searchable even if directory listing is denied. + return None + except OSError: + continue + return names + + +def resolve_executables(executables) -> dict[str, str | None]: + """Return PATH evidence, never an installation or capability assertion.""" + candidates = _windows_names() if sys.platform == "win32" else None + unique = list(dict.fromkeys(executables)) + if sys.platform == "win32" and candidates is None: + # Restricted directory listing: preserve which() semantics while bounding + # concurrent metadata probes. This is an I/O budget, not a tool ceiling. + with ThreadPoolExecutor(max_workers=8) as pool: + return dict(zip(unique, pool.map(shutil.which, unique))) + return { + name: shutil.which(name) if ( + candidates is None or os.path.dirname(name) or name.casefold() in candidates + ) else None + for name in unique + } diff --git a/src/rei_rea/registry.py b/src/rei_rea/registry.py index 6462722..ce4eb19 100644 --- a/src/rei_rea/registry.py +++ b/src/rei_rea/registry.py @@ -11,6 +11,7 @@ import subprocess import tempfile import time +from .discovery import resolve_executables # Fixed, read-only command templates. This mapping is code-owned, not registry-provided. @@ -54,21 +55,25 @@ def load_catalog() -> list[dict]: raise RegistryError(f"Bad metadata for tool: {ident}") if tool.get("integration") not in ("static-adapter", "discovery-only"): raise RegistryError(f"Bad integration mode: {ident}") - if tool["integration"] == "static-adapter" and ident not in STATIC_ADAPTERS: - raise RegistryError(f"Tool lacks a code-owned adapter: {ident}") + if (tool["integration"] == "static-adapter") != (ident in STATIC_ADAPTERS): + raise RegistryError(f"Catalog/adapter disagreement: {ident}") seen.add(ident) return result def list_tools(*, tag: str | None = None, installed_only: bool = False) -> list[dict]: items = [] - for item in load_catalog(): - if tag and tag not in item["tags"] and tag != item["category"]: - continue - found = next((shutil.which(exe) for exe in item["executables"] if shutil.which(exe)), None) + catalog = [item for item in load_catalog() + if not tag or tag in item["tags"] or tag == item["category"]] + detected = resolve_executables(exe for item in catalog for exe in item["executables"]) + for item in catalog: + found = next((detected[exe] for exe in item["executables"] if detected[exe]), None) + adapter_exe = STATIC_ADAPTERS.get(item["id"], (None,))[0] + adapter_path = detected.get(adapter_exe) if installed_only and not found: continue - items.append({**item, "installed": bool(found), "resolved_executable": found}) + items.append({**item, "installed": bool(found), "resolved_executable": found, + "adapter_available": bool(adapter_path), "adapter_executable": adapter_path}) return items @@ -77,7 +82,7 @@ def tool_doctor(*, tag: str | None = None) -> dict: return { "catalogued": len(items), "detected_on_path": sum(x["installed"] for x in items), - "static_adapters_available": sum(x["installed"] and x["integration"] == "static-adapter" for x in items), + "static_adapters_available": sum(x["adapter_available"] for x in items), "note": "PATH discovery only. Availability, entitlement, licensing and end-to-end functionality are NOT verified.", "tools": items, } diff --git a/src/rei_rea/webui/index.html b/src/rei_rea/webui/index.html index f3006e9..176327a 100644 --- a/src/rei_rea/webui/index.html +++ b/src/rei_rea/webui/index.html @@ -19,11 +19,11 @@
ToolCategoryIntegrationStatusReference
Reading local registry…
Executable discovery is not an installation, security audit, license grant, or functional test. External tools are never launched from this dashboard. Use CLI case records and explicit adapter opt-in for additional work.
-
Rootcastle · REI REA v0.1.0 · Static triage / evidence chain / skill routes / MCP · No telemetry
+
Rootcastle · REI REA · Static triage / evidence chain / skill routes / MCP · No telemetry
diff --git a/tests/test_discovery.py b/tests/test_discovery.py new file mode 100644 index 0000000..9ab575d --- /dev/null +++ b/tests/test_discovery.py @@ -0,0 +1,65 @@ +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from rei_rea.discovery import resolve_executables, _windows_names +from rei_rea.registry import STATIC_ADAPTERS, load_catalog, tool_doctor + + +class DiscoveryTests(unittest.TestCase): + def test_permission_denied_inventory_falls_back_to_resolver(self): + with patch('rei_rea.discovery.os.scandir', side_effect=PermissionError): + self.assertIsNone(_windows_names()) + with patch('rei_rea.discovery.sys.platform', 'win32'), patch('rei_rea.discovery._windows_names', return_value=None), patch('rei_rea.discovery.shutil.which', side_effect=lambda name: '/found/'+name): + self.assertEqual(resolve_executables(['a', 'b', 'a']), {'a': '/found/a', 'b': '/found/b'}) + + def test_catalog_and_adapters_agree_in_both_directions(self): + catalog = load_catalog() + self.assertEqual({t['id'] for t in catalog if t['integration'] == 'static-adapter'}, set(STATIC_ADAPTERS)) + for tool in catalog: + if tool['id'] in STATIC_ADAPTERS: + self.assertIn(STATIC_ADAPTERS[tool['id']][0], tool['executables']) + + def test_duplicate_probes_resolved_once(self): + with patch('rei_rea.discovery.sys.platform', 'linux'), patch('rei_rea.discovery.shutil.which', return_value=None) as which: + self.assertEqual(resolve_executables(['missing', 'missing']), {'missing': None}) + which.assert_called_once_with('missing') + + def test_windows_prefilter_preserves_final_resolver_authority(self): + with patch('rei_rea.discovery.sys.platform', 'win32'), patch('rei_rea.discovery._windows_names', return_value={'tool'}), patch('rei_rea.discovery.shutil.which', return_value=None) as which: + self.assertEqual(resolve_executables(['tool', 'absent']), {'tool': None, 'absent': None}) + which.assert_called_once_with('tool') + + def test_directory_inventory_is_fresh_case_insensitive_and_handles_bad_paths(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + with patch.dict(os.environ, {'PATH': os.pathsep.join([tmp, tmp, str(root/'missing')])}): + p = root/'FreshProbeXYZ.EXE' + self.assertNotIn('freshprobexyz', _windows_names()) + p.write_bytes(b'not executed') + self.assertIn('freshprobexyz', _windows_names()) + p.unlink() + self.assertNotIn('freshprobexyz', _windows_names()) + + def test_actual_path_lookup_and_refresh(self): + with tempfile.TemporaryDirectory() as tmp: + p = Path(tmp) / ('rea_probe.exe' if os.name == 'nt' else 'rea_probe') + with patch.dict(os.environ, {'PATH': tmp}): + self.assertIsNone(resolve_executables(['rea_probe'])['rea_probe']) + p.write_bytes(b'not executed') + p.chmod(0o700) + self.assertEqual(Path(resolve_executables(['rea_probe'])['rea_probe']), p) + p.unlink() + self.assertIsNone(resolve_executables(['rea_probe'])['rea_probe']) + + def test_detected_frontend_does_not_mean_adapter_available(self): + def detect(names): + return {n: '/tools/r2' if n == 'r2' else None for n in names} + with patch('rei_rea.registry.resolve_executables', side_effect=detect): + result = tool_doctor() + tool = next(t for t in result['tools'] if t['id'] == 'radare2') + self.assertTrue(tool['installed']) + self.assertFalse(tool['adapter_available']) + self.assertEqual(result['static_adapters_available'], 0) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index d547a21..01b22a9 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -11,7 +11,7 @@ class MetadataTests(unittest.TestCase): def test_metadata_has_valid_repository_description_and_topics(self): doc = load_metadata(DEFAULT_METADATA) - self.assertEqual(doc['repository'], 'rootcastleco/Rei-Reverse-engineer-') + self.assertEqual(doc['repository'], 'rootcastleco/Rei-Reverse-engineer') self.assertGreaterEqual(len(doc['topics']), 10) self.assertIn('reverse-engineering', doc['topics']) self.assertIn('mcp-server', doc['topics']) From 09fbc7bd202f4a40aa78230e4517e7f39c4a158b Mon Sep 17 00:00:00 2001 From: Batuhan Date: Sun, 1 Nov 2026 19:19:50 +0300 Subject: [PATCH 2/2] fix: preserve default executable search when PATH is unset --- docs/STABILIZATION.md | 6 ++++-- src/rei_rea/discovery.py | 3 +++ tests/test_discovery.py | 7 +++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/STABILIZATION.md b/docs/STABILIZATION.md index d456841..b70d54d 100644 --- a/docs/STABILIZATION.md +++ b/docs/STABILIZATION.md @@ -92,9 +92,11 @@ features. The existing daily upstream review workflow stays independent. ## Local verification results -- Full suite: 59 tests in 79.342 seconds, 58 passed and one skipped (`file` absent), no failures. No overall-suite speedup is claimed. -- New discovery module: coverage.py 7.11.0 reports 100% statement/branch coverage (29 statements, 10 branches); seven focused tests. This is not whole-project coverage. +- Full suite after review correction: 60 tests in 33.735 seconds, 59 passed and one skipped (`file` absent), no failures. No overall-suite speedup is claimed. +- New discovery module: coverage.py 7.11.0 reports 100% statement/branch coverage (31 statements, 12 branches); eight focused tests. This is not whole-project coverage. - All 12 skills passed the installed skill validator. - Python wheel built locally; publication and platform-specific executables remain separate gates. - The original CLI timeout regression passes on the existing PATH without changing its 10-second deadline. - External-engine E2E, peak memory and parser performance benchmarks were not measured in this milestone. + +Review correction: an unset PATH now bypasses the directory prefilter so `shutil.which` retains its platform-default lookup. A regression distinguishes this from an explicitly empty PATH. The wheel was rebuilt after this correction. diff --git a/src/rei_rea/discovery.py b/src/rei_rea/discovery.py index 9b535a3..728f6bf 100644 --- a/src/rei_rea/discovery.py +++ b/src/rei_rea/discovery.py @@ -14,6 +14,9 @@ def _windows_names() -> set[str] | None: precedence and whether the current directory is permitted on this Python. No results survive a discovery request. """ + if "PATH" not in os.environ: + # which() uses the platform default search path when PATH is absent. + return None names: set[str] = set() directories = dict.fromkeys([os.curdir, *os.environ.get("PATH", "").split(os.pathsep)]) for directory in directories: diff --git a/tests/test_discovery.py b/tests/test_discovery.py index 9ab575d..043df45 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -9,6 +9,13 @@ class DiscoveryTests(unittest.TestCase): + def test_unset_path_retains_standard_library_default_search(self): + with patch.dict(os.environ, {}, clear=True): + self.assertIsNone(_windows_names()) + with patch('rei_rea.discovery.sys.platform', 'win32'), patch('rei_rea.discovery.shutil.which', return_value='/default/tool') as which: + self.assertEqual(resolve_executables(['tool']), {'tool': '/default/tool'}) + which.assert_called_once_with('tool') + def test_permission_denied_inventory_falls_back_to_resolver(self): with patch('rei_rea.discovery.os.scandir', side_effect=PermissionError): self.assertIsNone(_windows_names())